output-test.c

/*	Testing output of C compilers
	Copyright 2015-2019 Rivoreo

	This Source Code Form is subject to the terms of the Mozilla Public
	License, v. 2.0. If a copy of the MPL was not distributed with this
	file, You can obtain one at https://mozilla.org/MPL/2.0/.
*/

#include <sys/param.h>
#include <sys/wait.h>
#include <sys/stat.h>
#include <unistd.h>
#include <signal.h>
#include <string.h>
#include <stdlib.h>
#include <stdio.h>
#include <fcntl.h>
#include <errno.h>
#include <ctype.h>
#include <assert.h>

#ifdef __GNUC__
#define LIKELY(CONDITION) __builtin_expect((CONDITION) != 0, 1)
#define UNLIKELY(CONDITION) __builtin_expect((CONDITION), 0)
#else
#define LIKELY(CONDITION) (CONDITION)
#define UNLIKELY(CONDITION) (CONDITION)
#endif

#ifndef FD_ISZERO
static const fd_set _empty_fdset;
#define FD_ISZERO(S) (memcmp((S), &_empty_fdset, sizeof(fd_set)) == 0)
#endif

#ifdef BSD
#define setpgrp() setpgid(0, 0)
#endif

static char *generator;
static char **generator_argv;
static char *output_directory;
static unsigned int time_out = 240;

struct compiler_configuration {
	char *name;
	char **argv;
	char *output_name_suffix;
	char *launcher;
	unsigned char group;
	pid_t pid;
	int output_fd;
	char *output;
	size_t output_len;
	size_t output_max_len;
	struct compiler_configuration *next;
} *compiler_tail;

static struct compiler_configuration *create_compiler_configuration(const char *name) {
	struct compiler_configuration *node = malloc(sizeof(struct compiler_configuration));
	if(!node) return NULL;
	node->name = strdup(name);
	if(!node->name) {
		free(node);
		return NULL;
	}
	node->output_name_suffix = NULL;
	node->launcher = NULL;
	node->group = 0;
	node->pid = -1;
	node->output_fd = -1;
	node->output = NULL;
	node->output_len = 0;
	node->output_max_len = 0;
	node->next = compiler_tail;
	compiler_tail = node;
	return node;
}

static int fgetline(FILE *f, char *line, size_t len) {
	size_t i = 0;
	int c;
	while((c = fgetc(f)) != '\n') {
		if(c == EOF) {
			if(!i) return -1;
			break;
		}
		if(i >= len - 1) return -2;
		line[i++] = c;
	}
	line[i] = 0;
	return i;
}

static void command_line_to_argv(int *argc, char ***argv, char *command_line, unsigned int preallocate_count) {
	char *p = command_line;
	unsigned int i = 0, j = 0;
	int qm = 0;
	*argc = preallocate_count;
	*argv = malloc(sizeof(char *) * (preallocate_count + 1));
	if(!*argv) {
		fputs("Out of memory\n", stderr);
		exit(32);
	}
	while(command_line[i] == ' ' || command_line[i] == '	' || command_line[i] == '\n') i++;
	while(1) {
		char c = command_line[i];
		switch(c) {
			case '\\':
				c = command_line[j++] = command_line[++i];
				break;
			case '\'':
			case '"':
				if(qm == c) qm = 0;
				else if(!qm) qm = c;
				else command_line[j++] = c;
				break;
			case ' ':
			case '	':
			case '\n':	// Is this possible?
				if(qm) {
					command_line[j++] = c;
				} else {
					command_line[j++] = 0;
					*argv = realloc(*argv, sizeof(char *) * (*argc + 2));
					if(!*argv || !((*argv)[*argc] = strdup(p))) {
						fputs("Out of memory\n", stderr);
						exit(32);
					}
					(*argc)++;
					while(command_line[i] == ' ' || command_line[i] == '	' || command_line[i] == '\n') i++;
					p = command_line + i + 1;
				}
				break;
			default:
				command_line[j++] = c;
				break;
		}
		if(!c) {
			if(*p) {
				*argv = realloc(*argv, sizeof(char *) * (*argc + 2));
				if(!*argv || !((*argv)[*argc] = strdup(p))) {
					fputs("Out of memory\n", stderr);
					exit(32);
				}
				(*argc)++;
			}
			break;
		}
		i++;
	}
	(*argv)[*argc] = NULL;
}

static char **create_argv(const char *name, char *program, char *cflags, char *cflags_include,
char *cflags_warning, char *cflags_optimizing, const char *output_name_suffix, const char *output_file_option,
int output_file_option_style, char *libs) {
	if(!program) {
		fprintf(stderr, "Missing required setting 'program' in section '%s'\n",
			compiler_tail->name);
		return NULL;
	}
	if(!cflags_optimizing) {
		fprintf(stderr, "Missing required setting 'cflagsoptimizing' in section '%s'\n",
			compiler_tail->name);
		return NULL;
	}
	// <program> <cflags_include> <cflags_optimizing> <cflags_warning> <cflags> test.c <output_file_option>[ ][<output_directory>/]test-<name>-out<output_name_suffix> <libs>
	// Variable flags: cflags_include cflags_optimizing cflags_warning cflags libs
	// Fixed flag count: 3 + output_file_option_style
	size_t argv_size = (3 + output_file_option_style + 1) * sizeof(char *);
	char **argv = malloc(argv_size);
	if(!argv) {
		fputs("Out of memory\n", stderr);
		return NULL;
	}
	argv[0] = program;
	int i = 1, j;
	char **argv_part;
#define APPEND_TO_ARGV(COMMAND_LINE) do {				\
		command_line_to_argv(&j, &argv_part, (COMMAND_LINE), 0);\
		if(j) {							\
			argv_size += j * sizeof(char *);		\
			argv = realloc(argv, argv_size);		\
			if(!argv) {					\
				fputs("Out of memory\n", stderr);	\
				return NULL;				\
			}						\
			memcpy(argv + i, argv_part, j * sizeof(char *));\
			i += j;						\
		}							\
		free(argv_part);					\
	} while(0)
	APPEND_TO_ARGV(cflags_include);
	APPEND_TO_ARGV(cflags_optimizing);
	APPEND_TO_ARGV(cflags_warning);
	APPEND_TO_ARGV(cflags);
	argv[i++] = (char *)"test.c";
	size_t output_dir_len = output_directory ? strlen(output_directory) : 0;
	size_t config_name_len = strlen(name);
	size_t output_name_suffix_len = output_name_suffix ? strlen(output_name_suffix) : 0;
	char *output_name_p;
	if(output_file_option_style) {
		argv[i] = strdup(output_file_option);
		if(!argv[i]) {
			fputs("Out of memory\n", stderr);
			return NULL;
		}
		argv[++i] = malloc(output_dir_len + 1 + 5 + config_name_len + 4 + output_name_suffix_len + 1);
		if(!argv[i]) {
			fputs("Out of memory\n", stderr);
			return NULL;
		}
		output_name_p = argv[i];
	} else {
		size_t output_option_len = strlen(output_file_option);
		argv[i] = malloc(output_option_len + output_dir_len + 1 + 5 + config_name_len + 4 + output_name_suffix_len + 1);
		if(!argv[i]) {
			fputs("Out of memory\n", stderr);
			return NULL;
		}
		memcpy(argv[i], output_file_option, output_option_len);
		output_name_p = argv[i] + output_option_len;
	}
	i++;
	if(output_dir_len) {
		memcpy(output_name_p, output_directory, output_dir_len);
		output_name_p[output_dir_len++] = '/';
	}
	memcpy(output_name_p + output_dir_len, "test-", 5);
	memcpy(output_name_p + output_dir_len + 5, name, config_name_len);
	memcpy(output_name_p + output_dir_len + 5 + config_name_len, "-out", 4);
	if(output_name_suffix_len) {
		memcpy(output_name_p + output_dir_len + 5 + config_name_len + 4, output_name_suffix, output_name_suffix_len + 1);
	} else {
		output_name_p[output_dir_len + 5 + config_name_len + 4] = 0;
	}
	APPEND_TO_ARGV(libs);
#undef APPEND_TO_ARGV
	argv[i] = NULL;
	return argv;
}

static int check_groups(unsigned char *group, char **name) {
	struct {
		char *unique_name;
		unsigned int count;
	} group_counts[256];
	memset(group_counts, 0, sizeof group_counts);
	unsigned char i = 0;
	struct compiler_configuration *node = compiler_tail;
	do {
		if(group_counts[node->group].count < 2) {
			group_counts[node->group].unique_name = node->name;
			group_counts[node->group].count++;
		}
		node = node->next;
	} while(node);
	do {
		if(group_counts[i].count == 1) {
			*group = i;
			*name = group_counts[i].unique_name;
			return 0;
		}
	} while(i++ < 255);
	return 1;
}

static int read_config_file(const char *path) {
	FILE *f = fopen(path, "r");
	if(!f) {
		perror(path);
		return -1;
	}

	char *generator_flags = NULL;
	char *program;
	char *cflags;
	char *cflags_include;
	char *cflags_warning;
	char *cflags_optimizing;
	char *output_file_option;
	int output_file_option_style;
	char *libs;

	unsigned int nlines = 0;
	char line[1024];
	while(1) {
		nlines++;
		int len = fgetline(f, line, sizeof line);
		if(len == -2) {
			fprintf(stderr, "%s:%u: line too long\n", path, nlines);
			return -1;
		}
		if(len < 0) break;
		if(len == 0) continue;
		char *p = line;
		while(*p && (*p == ' ' || *p == '	')) p++;
		if(!*p) continue;
		if(*p == ';' || *p == '#') continue;
		if(*p == '[') {
			// Section begin
			char *end_p = strchr(p, ']');
			if(!end_p) {
				fprintf(stderr, "%s:%u: cannot find ending ']'\n", path, nlines);
				return -1;
			}
			*end_p = 0;
			p++;
			if(strchr(p, '/')) {
				fprintf(stderr, "%s:%u: '/' is not allowed in section name\n", path, nlines);
				return -1;
			}
			if(compiler_tail) {
				compiler_tail->argv = create_argv(compiler_tail->name, program, cflags, cflags_include,
					cflags_warning, cflags_optimizing, compiler_tail->output_name_suffix,
					output_file_option, output_file_option_style, libs);
				if(!compiler_tail->argv) return -1;

				free(cflags);
				free(cflags_warning);
				free(cflags_optimizing);
				free(output_file_option);
				free(libs);
			}
			program = NULL;
			cflags = strdup("");
			cflags_include = strdup("-Iruntime");
			cflags_warning = strdup("-w");
			//cflags_optimizing = strdup("-Os");
			cflags_optimizing = NULL;
			output_file_option = strdup("-o");
			output_file_option_style = 1;
			libs = strdup("");
			if(!cflags || !cflags_include || !cflags_warning || !output_file_option || !libs ||
			   !create_compiler_configuration(p)) {
				fputs("out of memory when parsing configuration file\n", stderr);
				return -1;
			}
		} else if(isalpha(*p)) {
			// Variable
			char *equal = strchr(p, '=');
			if(!equal) {
				fprintf(stderr, "%s:%u: cannot find '=' sign\n", path, nlines);
				return -1;
			}
			char *w = equal - 1;
			while(*w == ' ' || *w == '	') *w-- = 0;
			assert(w > p);
			char *value = equal + 1;
			while(*value && (*value == ' ' || *value == '	')) value++;
			int value_len = len - (value - line);
			if(value[value_len - 1] == '\r') value_len--;
			while(value_len > 0 && (value[value_len - 1] == ' ' || value[value_len - 1] == '	')) {
				value_len--;
			}
			value[value_len] = 0;
			if(compiler_tail) {
				// Section
				if(strcasecmp(p, "program") == 0) {
					if(!*value) {
						fprintf(stderr, "%s:%u: program cannot be empty\n", path, nlines);
						return -1;
					}
					free(program);
					program = strdup(value);
					if(!program) {
						fprintf(stderr, "%s:%u: out of memory\n", path, nlines);
						return -1;
					}
				} else if(strcasecmp(p, "cflags") == 0) {
					free(cflags);
					cflags = strdup(value);
					if(!cflags) {
						fprintf(stderr, "%s:%u: out of memory\n", path, nlines);
						return -1;
					}
				} else if(strcasecmp(p, "cflagsinclude") == 0) {
					free(cflags);
					cflags_include = strdup(value);
					if(!cflags_include) {
						fprintf(stderr, "%s:%u: out of memory\n", path, nlines);
						return -1;
					}
				} else if(strcasecmp(p, "cflagswarning") == 0) {
					free(cflags_warning);
					cflags_warning = strdup(value);
					if(!cflags_warning) {
						fprintf(stderr, "%s:%u: out of memory\n", path, nlines);
						return -1;
					}
				} else if(strcasecmp(p, "cflagsoptimizing") == 0) {
					free(cflags_optimizing);
					cflags_optimizing = strdup(value);
					if(!cflags_optimizing) {
						fprintf(stderr, "%s:%u: out of memory\n", path, nlines);
						return -1;
					}
				} else if(strcasecmp(p, "outputnamesuffix") == 0) {
					free(compiler_tail->output_name_suffix);
					if(value) {
						compiler_tail->output_name_suffix = strdup(value);
						if(!compiler_tail->output_name_suffix) {
							fprintf(stderr, "%s:%u: out of memory\n", path, nlines);
							return -1;
						}
					} else compiler_tail->output_name_suffix = NULL;
				} else if(strcasecmp(p, "outputoption") == 0) {
					if(!*value) {
						fprintf(stderr, "%s:%u: output option cannot be empty\n", path, nlines);
						return -1;
					}
					free(output_file_option);
					output_file_option = strdup(value);
					if(!output_file_option) {
						fprintf(stderr, "%s:%u: out of memory\n", path, nlines);
						return -1;
					}
				} else if(strcasecmp(p, "outputoptionstyle") == 0) {
					//output_file_option_style = *value > '0';
					output_file_option_style = atoi(value);
					if(output_file_option_style < 0) output_file_option_style = 0;
				} else if(strcasecmp(p, "libs") == 0) {
					free(libs);
					libs = strdup(value);
					if(!libs) {
						fprintf(stderr, "%s:%u: out of memory\n", path, nlines);
						return -1;
					}
				} else if(strcasecmp(p, "launcher") == 0) {
					free(compiler_tail->launcher);
					if(value) {
						compiler_tail->launcher = strdup(value);
						if(!compiler_tail->launcher) {
							fprintf(stderr, "%s:%u: out of memory\n", path, nlines);
							return -1;
						}
					} else compiler_tail->launcher = NULL;
				} else if(strcasecmp(p, "group") == 0) {
					char *end_p;
					if(!*value) {
						fprintf(stderr, "%s:%u: group cannot be empty\n", path, nlines);
						return -1;
					}
					compiler_tail->group = strtoul(value, &end_p, 0);
					if(*end_p) {
						fprintf(stderr, "%s:%u: extra characters after group number\n",
							path, nlines);
						return -1;
					}
				} else {
					fprintf(stderr, "%s:%u: Warning: ignoring unrecognized setting '%s'\n",
						path, nlines, p);
				}
			} else {
				// Global
				if(strcasecmp(p, "generator") == 0) {
					if(!*value) {
						fprintf(stderr, "%s:%u: code generator program cannot be empty\n",
							path, nlines);
						return -1;
					}
					free(generator);
					generator = strdup(value);
					if(!generator) {
						fprintf(stderr, "%s:%u: out of memory\n", path, nlines);
						return -1;
					}
				} else if(strcasecmp(p, "generatorflags") == 0) {
					free(generator_flags);
					generator_flags = strdup(value);
					if(!generator_flags) {
						fprintf(stderr, "%s:%u: out of memory\n", path, nlines);
						return -1;
					}
				} else if(strcasecmp(p, "outputdirectory") == 0) {
					free(output_directory);
					output_directory = strdup(value);
					if(!output_directory) {
						fprintf(stderr, "%s:%u: out of memory\n", path, nlines);
						return -1;
					}
				} else if(strcasecmp(p, "timeout") == 0) {
					char *end_p;
					if(!*value) {
						fprintf(stderr, "%s:%u: timeout cannot be empty\n", path, nlines);
						return -1;
					}
					time_out = strtoul(value, &end_p, 0);
					if(*end_p) {
						fprintf(stderr, "%s:%u: extra characters after timeout value is currently not supported\n",
							path, nlines);
						return -1;
					}
				} else {
					fprintf(stderr, "%s:%u: Warning: ignoring unrecognized setting '%s'\n",
						path, nlines, p);
				}
			}
		} else {
			fprintf(stderr, "%s:%u: syntax error\n", path, nlines);
			return -1;
		}
	}

	fclose(f);

	if(!generator) generator = (char *)"csmith";	// No free(3)ing 'generator' beyond this
	if(generator_flags) {
		int argc;
		command_line_to_argv(&argc, &generator_argv, generator_flags, 1);
		generator_argv[0] = generator;
		free(generator_flags);
	} else {
		generator_argv = malloc(2 * sizeof(char *));
		if(!generator_argv) {
			fputs("out of memory\n", stderr);
			return -1;
		}
		generator_argv[0] = generator;
		generator_argv[1] = NULL;
	}

	if(!compiler_tail) {
		fputs("No compiler section defined\n", stderr);
		return -1;
	}
	compiler_tail->argv = create_argv(compiler_tail->name, program, cflags, cflags_include, cflags_warning,
		cflags_optimizing, compiler_tail->output_name_suffix, output_file_option, output_file_option_style, libs);
	if(!compiler_tail->argv) return -1;

	free(cflags);
	free(cflags_warning);
	free(cflags_optimizing);
	free(output_file_option);
	free(libs);

	unsigned char group;
	char *name;
	if(!check_groups(&group, &name)) {
		fprintf(stderr, "Group %hhu contains only one compiler configuration '%s'\n"
			"At least 2 compiler configurations must be defined for each group\n",
			group, name);
		return -1;
	}
	return 0;
}

static int generate_program() {
	pid_t pid = fork();
	if(UNLIKELY(pid < 0)) {
		perror("fork");
		return -1;
	}
	if(pid) {
		int status;
		while(waitpid(pid, &status, 0) < 0) {
			if(errno == EINTR) continue;
			perror("waitpid");
			return -1;
		}
		if(WIFSIGNALED(status)) {
			fprintf(stderr, "process %d terminated by signal %d\n", (int)pid, WTERMSIG(status));
			return -1;
		}
		return WEXITSTATUS(status) ? -1 : 0;
	} else {
		close(STDOUT_FILENO);
		int fd = open("test.c", O_WRONLY | O_CREAT | O_TRUNC, 0644);
		if(fd == -1) {
			perror("test.c");
			_exit(1);
		}
		if(LIKELY(fd != STDOUT_FILENO)) {
			dup2(fd, STDOUT_FILENO);
			close(fd);
		}
		execvp(generator, generator_argv);
		perror(generator);
		_exit(127);
	}
}

static int run_test(struct compiler_configuration *c) {
	int pipe_fds[2];
	if(pipe(pipe_fds) < 0) {
		perror("pipe");
		return -1;
	}
	pid_t pid = fork();
	if(UNLIKELY(pid < 0)) {
		perror("fork");
		//close?
		return -1;
	}
	if(pid) {
		close(pipe_fds[1]);
		c->pid = pid;
		c->output_fd = pipe_fds[0];
		return pipe_fds[0];
	} else {
		close(pipe_fds[0]);
		pid_t c_pid = fork();
		if(UNLIKELY(c_pid < 0)) {
			perror("fork");
			_exit(1);
		}
		if(c_pid) {
			int status;
			while(waitpid(c_pid, &status, 0) < 0) {
				if(errno == EINTR) continue;
				perror("waitpid");
				_exit(1);
			}
			if(WIFSIGNALED(status)) {
				fprintf(stderr, "compiler '%s' process %d terminated by signal %d\n",
					c->argv[0], (int)c_pid, WTERMSIG(status));
				_exit(1);
			}
			if(WEXITSTATUS(status)) {
				fprintf(stderr, "compiler '%s' process %d exited with status %d\n",
					c->argv[0], (int)c_pid, WEXITSTATUS(status));
				_exit(1);
			}
			if(LIKELY(pipe_fds[1] != STDOUT_FILENO)) {
				dup2(pipe_fds[1], STDOUT_FILENO);
				close(pipe_fds[1]);
			}
			size_t config_name_len = strlen(c->name);
			size_t output_name_suffix_len = c->output_name_suffix ? strlen(c->output_name_suffix) : 0;
			size_t output_dir_len = output_directory ? strlen(output_directory) : 0;
			//char output_program[(output_dir_len ? output_dir_len + 1 : (c->launcher ? 0 : 2)) + 5 + config_name_len + 4 + output_name_suffix_len + 1];
			char *output_program, *p;
			if(output_dir_len) {
				output_program = malloc(output_dir_len + 1 + 5 + config_name_len + 4 + output_name_suffix_len + 1);
				memcpy(output_program, output_directory, output_dir_len);
				output_program[output_dir_len] = '/';
				p = output_program + output_dir_len + 1;
			} else {
				output_program = malloc((c->launcher ? 0 : 2) + 5 + config_name_len + 4 + output_name_suffix_len + 1);
				if(UNLIKELY(!output_program)) {
					fputs("Out of memory\n", stderr);
					_exit(1);
				}
				if(c->launcher) {
					p = output_program;
				} else {
					memcpy(output_program, "./", 2);
					p = output_program + 2;
				}
			}
			memcpy(p, "test-", 5);
			memcpy(p + 5, c->name, config_name_len);
			memcpy(p + 5 + config_name_len, "-out", 4);
			if(output_name_suffix_len) {
				memcpy(p + 5 + config_name_len + 4, c->output_name_suffix, output_name_suffix_len + 1);
			} else {
				p[5 + config_name_len + 4] = 0;
			}
			if(c->launcher) {
				execlp(c->launcher, c->launcher, output_program, NULL);
				perror(c->launcher);
			} else {
				execl(output_program, p, NULL);
				perror(output_program);
			}
			_exit(127);
		} else {
			setpgrp();
			execvp(c->argv[0], c->argv);
			perror(c->argv[0]);
			_exit(127);
		}
	}
}

static void kill_processes() {
	struct compiler_configuration *node = compiler_tail;
	while(node) {
		if(node->pid != -1) kill(node->pid, SIGTERM);
		node = node->next;
	}
}

/* Return values:
	-1	Usage error
	1	Result diffs in same group
	2	Outputs from timed out test(s)
	4	Some test(s) timed out in a group
	8	A test finishes successfully but doesn't have output
	16	A test failed (signaled or exited with non-zero status)
	32	Operational error
Any positive value descript above could be bit-wise ORed.
*/
int main(int argc, char **argv) {
	struct {
		char *output;
		size_t output_len;
		int terminated;
		int skip;
		int skip_empty;
	} output_by_group[256];
	if(argc != 2) {
		fprintf(stderr, "Usage: %s <config-file>\n", argv[0]);
		return -1;
	}
	if(read_config_file(argv[1]) < 0) {
		return 32;
	}
	struct stat st;
	if(output_directory && *output_directory) {
		if(stat(output_directory, &st) < 0) {
			if(errno != ENOENT || mkdir(output_directory, 0755) < 0) {
				perror(output_directory);
				return 32;
			}
		} else if(!S_ISDIR(st.st_mode)) {
			fprintf(stderr, "%s: %s\n", output_directory, strerror(ENOTDIR));
			return 32;
		}
	}
	int should_quit = 0;
	unsigned long int round = 1;
	do {
		if(generate_program() < 0) {
			fputs("Failed to generate program\n", stderr);
			return 32;
		}
		fprintf(stderr, "round %lu: program generated\n", round);
		fd_set fdset;
		FD_ZERO(&fdset);
		int max_fd = 0;
		struct compiler_configuration *c = compiler_tail;
		while(c) {
			int fd = run_test(c);
			if(fd == -1) {
				fprintf(stderr, "Failed to run test for '%s'\n", c->name);
				kill_processes();
				return 32;
			}
			FD_SET(fd, &fdset);
			if(fd > max_fd) max_fd = fd;
			c = c->next;
		}
		memset(output_by_group, 0, sizeof output_by_group);
		do {
			fd_set rfdset = fdset;
			struct timeval timeout = { .tv_sec = time_out };
			int n = select(max_fd + 1, &rfdset, NULL, NULL, &timeout);
			if(n < 0) {
				if(errno == EINTR) continue;
				perror("select");
				kill_processes();
				return 32;
			}
			c = compiler_tail;
			if(!n) {
				fprintf(stderr, "round %lu: timed out\n", round);
				while(c) {
					if(c->pid > 0) {
						if(kill(c->pid, 0) < 0) {
							fprintf(stderr, "kill: %d: %s\n", (int)c->pid, strerror(errno));
							kill_processes();
							return 32;
						} else {
							if(!output_by_group[c->group].skip_empty && output_by_group[c->group].terminated) {
								fprintf(stderr, "but group %hhu has terminated test(s)\n",
									c->group);
								output_by_group[c->group].skip_empty = 1;
								should_quit |= 4;
							}
							if(c->output_fd == -1) {
								fprintf(stderr, "'%s' is running but has an invalid output file descriptor, this shouldn't happen\n",
									c->name);
								//should_quit |= 32;
								kill_processes();
								return 32;
							} else {
								close(c->output_fd);
								c->output_fd = -1;
							}
							fprintf(stderr, "terminating '%s' process %d\n", c->name, (int)c->pid);
							kill(c->pid, SIGTERM);
							while(waitpid(c->pid, NULL, 0) < 0) {
								if(errno == EINTR) continue;
								perror("waitpid");
								kill_processes();
								return 32;
							}
							c->pid = -1;
							if(c->output) {
								if(c->output_len) {
									fprintf(stderr, "'%s' has output '", c->name);
									fwrite(c->output, c->output_len, 1, stderr);
									fputs("'\n", stderr);
									should_quit |= 2;
								}
								free(c->output);
								c->output = NULL;
								c->output_len = 0;
								c->output_max_len = 0;
							}
						}
					}
					c = c->next;
				}
				break;
			}
			do {
				if(c->output_fd != -1 && FD_ISSET(c->output_fd, &rfdset)) {
					n--;
					if(c->output_len + 4 > c->output_max_len) {
						c->output_max_len += 24;
						c->output = realloc(c->output, c->output_max_len);
						if(UNLIKELY(!c->output)) {
							fprintf(stderr, "round %lu: %s: out of memory\n",
								round, c->name);
							kill_processes();
							return 32;
						}
					}
					int s;
					do {
						s = read(c->output_fd, c->output + c->output_len,
							c->output_max_len - c->output_len);
					} while(s < 0 && errno == EINTR);
					if(s < 1) {
						int e = s < 0 ? errno : 0;
						fprintf(stderr, "round %lu: %s: ", round, c->name);
						if(e) fprintf(stderr, "read error, %s; ", strerror(e));
						if(c->output_len) {
							if(c->output[c->output_len - 1] == '\n') c->output_len--;
							if(c->output_len && c->output[c->output_len - 1] == '\r') c->output_len--;
							fputc('\'', stderr);
							fwrite(c->output, c->output_len, 1, stderr);
							fputs("', ", stderr);
						} else {
							fputs("output nothing, ", stderr);
						}
						if(LIKELY(c->output_len < c->output_max_len)) {
							c->output_max_len = c->output_len;
							c->output = realloc(c->output, c->output_max_len);
						}
						close(c->output_fd);
						FD_CLR(c->output_fd, &fdset);
						c->output_fd = -1;
						kill(c->pid, SIGTERM);
						int status;
						while(waitpid(c->pid, &status, 0) < 0) {
							if(errno == EINTR) continue;
							perror("waitpid");
							kill_processes();
							return 32;
						}
						c->pid = -1;
						if(WIFSIGNALED(status)) {
							fprintf(stderr, "terminated by signal %d\n", WTERMSIG(status));
							should_quit |= 16;
						} else {
							status = WEXITSTATUS(status);
							fprintf(stderr, "exited with status %d\n", status);
							if(status) should_quit |= 16;
						}
						output_by_group[c->group].terminated = 1;
					} else {
						c->output_len += s;
					}
				}
				c = c->next;
			} while(c && n > 0);
			if(UNLIKELY(n)) {
				fprintf(stderr, "Internal error: failed to check all file descriptors after select (n = %d)\n", n);
				kill_processes();
				return 32;
			}
		} while(!FD_ISZERO(&fdset));
		c = compiler_tail;
		while(c) {
			if(!output_by_group[c->group].skip && output_by_group[c->group].terminated) {
				if(c->output_len) {
					if(output_by_group[c->group].output_len) {
						if(output_by_group[c->group].output_len != c->output_len ||
						   memcmp(output_by_group[c->group].output, c->output, c->output_len)) {
							fprintf(stderr, "round %lu: group %hhu has different outputs\n",
								round, c->group);
							output_by_group[c->group].skip = 1;
							should_quit |= 1;
						}
					} else {
						output_by_group[c->group].output = c->output;
						output_by_group[c->group].output_len = c->output_len;
					}
				} else if(!output_by_group[c->group].skip_empty) {
					fprintf(stderr, "round %lu: %s, (group %hhu) doesn't have output\n",
						round, c->name, c->group);
					should_quit |= 8;
				}
			}
			c = c->next;
		}
		c = compiler_tail;
		while(c) {
			assert(c->output_fd == -1);
			assert(c->pid == -1);
			free(c->output);
			c->output = NULL;
			c->output_len = 0;
			c->output_max_len = 0;
			c = c->next;
		}
		round++;
	} while(!should_quit);
	return should_quit;
}