xbps/bin/xbps-fbulk/main.c

/*
 * Copyright (c) 2010 The DragonFly Project.  All rights reserved.
 *
 * This code is derived from software contributed to The DragonFly Project
 * by Matthew Dillon <dillon@backplane.com>
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 *
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in
 *    the documentation and/or other materials provided with the
 *    distribution.
 * 3. Neither the name of The DragonFly Project nor the names of its
 *    contributors may be used to endorse or promote products derived
 *    from this software without specific, prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 */
/*
 * This is a derived version of DragonFly's BSD "fastbulk", adapted for xbps
 * by Juan RP <xtraeme@voidlinux.eu>.
 *
 * This program iterates all srcpkgs directories, runs './xbps-src show-build-deps',
 * and builds a dependency tree on the fly.
 *
 * As the dependency tree is being built, terminal dependencies are built
 * and packaged on the fly.
 *
 * As these builds complete additional dependencies may be satisfied and be
 * added to the build order. Ultimately the entire tree is built.
 *
 * Only one attempt is made to build any given package, no matter how many
 * other packages depend on it.
 */
#define _DEFAULT_SOURCE
#define _BSD_SOURCE
#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <string.h>
#include <dirent.h>
#include <assert.h>
#include <libgen.h>
#include <limits.h>
#include <errno.h>
#include <fcntl.h>
#include <sys/file.h>
#include <sys/stat.h>
#include <sys/wait.h>
#include <getopt.h>

#include <xbps.h>

struct item;

struct depn {
	struct depn *dnext;
	struct item *item;
};

struct item {
	enum { XWAITING, XDEPFAIL, XBUILD, XRUN, XDONE, XBROKEN } status;
	struct item *hnext;	/* ItemHash next */
	struct item *bnext;	/* BuildList/RunList next */
	struct depn *dbase;	/* packages depending on us */
	char *pkgn;		/* package name */
	char *emsg;		/* error message */
	int dcount;		/* build completion for our dependencies */
	int xcode;		/* exit code from build */
	pid_t pid;		/* running build */
};

#define ITHSIZE	1024
#define ITHMASK	(ITHSIZE - 1)

static struct item *ItemHash[ITHSIZE];
static struct item *BuildList;
static struct item **BuildListP = &BuildList;
static struct item *RunList;

int NParallel = 1;
int VerboseOpt;
int NRunning;
char *LogDir;
char *TargetArch;

/*
 * Item hashing and dependency helper routines, called during the
 * directory scan.
 */
static int
itemhash(const char *pkgn)
{
	int hv = 0xA1B5F342;
	int i;

	assert(pkgn);

	for (i = 0; pkgn[i]; ++i)
		hv = (hv << 5) ^ (hv >> 23) ^ pkgn[i];

	return hv & ITHMASK;
}

static struct item *
lookupItem(const char *pkgn)
{
	struct item *item;

	assert(pkgn);

	for (item = ItemHash[itemhash(pkgn)]; item; item = item->hnext) {
		if (strcmp(pkgn, item->pkgn) == 0)
			return item;
	}
	return NULL;
}

static struct item *
addItem(const char *pkgn)
{
	struct item **itemp;
	struct item *item = calloc(sizeof(*item), 1);

	assert(pkgn);
	assert(item);

	itemp = &ItemHash[itemhash(pkgn)];
	item->status = XWAITING;
	item->hnext = *itemp;
	item->pkgn = strdup(pkgn);
	*itemp = item;

	return item;
}

static void __attribute__((noreturn))
usage(const char *progname)
{
	fprintf(stderr, "%s [-a targetarch] [-h] [-j parallel] [-l logdir] [-V]"
	    "/path/to/void-packages [pkg pkgN]\n", progname);
	exit(EXIT_FAILURE);
}

/*
 * Add the item to the build request list.  This routine is called
 * after all build dependencies have been satisfied for the item.
 * runBuilds() will pick items off of BuildList to keep the parallel
 * build pipeline full.
 */
static void
addBuild(struct item *item)
{
	assert(item);

	printf("BuildOrder %s\n", item->pkgn);
	*BuildListP = item;
	BuildListP = &item->bnext;
	item->status = XBUILD;
}

/*
 * Process the build completion for an item.
 */
static void
processCompletion(struct item *item)
{
	struct depn *depn;
	struct item *xitem;
	const char *logdir;
	char *logpath1;
	char *logpath2;
	char *logpath3;
	FILE *fp;

	assert(item);
	/*
	 * If XRUN we have to move the logfile to the correct directory.
	 * (If XDEPFAIL the logfile is already in the correct directory).
	 */
	if (item->status == XRUN) {
		logpath1 = xbps_xasprintf("%s/run/%s", LogDir, item->pkgn);
		if (!item->xcode)
			logdir = "good";
		else if (item->xcode == 2)
			logdir = "skipped";
		else
			logdir = "bad";
		logpath2 = xbps_xasprintf("%s/%s/%s", LogDir, logdir, item->pkgn);
		(void)rename(logpath1, logpath2);
		free(logpath1);
		free(logpath2);
	}
	/*
	 * If XBROKEN, "xbps-src show-build-deps" returned an error, perhaps
	 * because the pkg is currently broken or cannot be packaged for the
	 * target architecture, just set it as skipped.
	 */
	if (item->status == XBROKEN) {
		logpath1 = xbps_xasprintf("%s/skipped/%s", LogDir, item->pkgn);
		fp = fopen(logpath1, "a");
		fprintf(fp, "%s", item->emsg);
		fclose(fp);
		free(logpath1);
		free(item->emsg);
	}

	printf("Finish %-3d %s\n", item->xcode, item->pkgn);
	assert(item->status == XRUN || item->status == XBROKEN || item->status == XDEPFAIL);
	item->status = XDONE;

	for (depn = item->dbase; depn; depn = depn->dnext) {
		xitem = depn->item;
		assert(xitem->dcount > 0);
		--xitem->dcount;
		if (xitem->status == XWAITING || xitem->status == XDEPFAIL || xitem->status == XBROKEN) {
			/*
			 * If our build went well add items dependent
			 * on us to the build, otherwise fail the items
			 * dependent on us.
			 */
			if (item->xcode) {
				xitem->xcode = item->xcode;
				xitem->status = XDEPFAIL;
				logpath3 = xbps_xasprintf("%s/deps/%s", LogDir, xitem->pkgn);
				fp = fopen(logpath3, "a");
				fprintf(fp, "%s\n", item->pkgn);
				fclose(fp);
				free(logpath3);
			}
			if (xitem->dcount == 0) {
				if (xitem->status == XWAITING)
					addBuild(xitem);
				else
					processCompletion(xitem);
			}
		} else if (xitem->status == XDONE && xitem->xcode) {
			/*
			 * The package depending on us has already run
			 * (this case should not occur).
			 *
			 * Add this dependency failure to its log file
			 * (which has already been renamed).
			 */
			logpath3 = xbps_xasprintf("%s/deps/%s", LogDir, xitem->pkgn);
			fp = fopen(logpath3, "a");
			fprintf(fp, "%s\n", item->pkgn);
			fclose(fp);
			free(logpath3);
		}
	}
}

/*
 * Wait for a running build to finish and process its completion.
 * Return the build or NULL if no builds are pending.
 *
 * The caller should call runBuilds() in the loop to keep the build
 * pipeline full until there is nothing left in the build list.
 */
static struct item *
waitRunning(int flags)
{
	struct item *item;
	struct item **itemp;
	pid_t pid;
	int status;

	if (RunList == NULL)
		return NULL;

	while ((pid = wait3(&status, flags, NULL)) < 0 && flags == 0)
		;

	/*
	 * NOTE! The pid may be associated with one of our popen()'s
	 *	 so just ignore it if we cannot find it.
	 */
	if (pid > 0) {
		status = WEXITSTATUS(status);
		itemp = &RunList;
		while ((item = *itemp) != NULL) {
			if (item->pid == pid)
				break;
			itemp = &item->bnext;
		}
		if (item) {
			*itemp = item->bnext;
			item->bnext = NULL;
			item->xcode = status;
			--NRunning;
			processCompletion(item);
		}
	} else {
		item = NULL;
	}
	return item;
}

/*
 * Start new builds from the build list and handle build completions,
 * which can potentialy add new items to the build list.
 *
 * This routine will maintain up to NParallel builds.  A new build is
 * only started once its dependencies have completed successfully so
 * when the bulk build starts it typically takes a little while before
 * fastbulk can keep the parallel pipeline full.
 */
static void
runBuilds(const char *bpath)
{
	struct item *item;
	char *logpath;
	FILE *fp;
	int fd;

	assert(bpath);
	/*
	 * Try to maintain up to NParallel builds
	 */
	while (NRunning < NParallel && BuildList) {
		item = BuildList;
		if ((BuildList = item->bnext) == NULL)
			BuildListP = &BuildList;

		printf("BuildStart %s\n", item->pkgn);

		/*
		 * When [re]running a build remove any bad log from prior
		 * attempts.
		 */
		logpath = xbps_xasprintf("%s/bad/%s", LogDir, item->pkgn);
		(void)remove(logpath);
		free(logpath);

		logpath = xbps_xasprintf("%s/run/%s", LogDir, item->pkgn);
		item->status = XRUN;

		item->pid = fork();
		if (item->pid == 0) {
			/*
			 * Child process - setup the log file and build
			 */
			if (chdir(bpath) < 0)
				_exit(99);

			fd = open(logpath, O_RDWR|O_CREAT|O_TRUNC, 0666);
			if (fd != 1)
				dup2(fd, 1);
			if (fd != 2)
				dup2(fd, 2);
			if (fd != 1 && fd != 2)
				close(fd);
			fd = open("/dev/null", O_RDWR);
			if (fd != 0) {
				dup2(fd, 0);
				close(fd);
			}
			/* build the current pkg! */
			if (TargetArch != NULL)
				execl("./xbps-src", "./xbps-src", "-a", TargetArch,
					"-E", "-N", "-t", "pkg", item->pkgn, NULL);
			else
				execl("./xbps-src", "./xbps-src",
					"-E", "-N", "-t", "pkg", item->pkgn, NULL);

			_exit(99);
		} else if (item->pid < 0) {
			/*
			 * Parent fork() failed, log the problem and
			 * do completion processing.
			 */
			item->xcode = -98;
			fp = fopen(logpath, "a");
			fprintf(fp, "xfbulk: Unable to fork/exec xbps-src\n");
			fclose(fp);
			processCompletion(item);
		} else {
			/*
			 * Parent is now tracking the running child,
			 * add the item to the RunList.
			 */
			item->bnext = RunList;
			RunList = item;
			++NRunning;
		}
		free(logpath);
	}

	/*
	 * Process any completed builds (non-blocking)
	 */
	while (waitRunning(WNOHANG) != NULL)
		;
}

/*
 * Add a reverse dependency from the deepest point (xitem) to the
 * packages that depend on xitem (item in this case).
 *
 * Caller will check dcount after it is through adding dependencies.
 */
static void
addDepn(struct item *item, struct item *xitem)
{
	struct depn *depn = calloc(sizeof(*depn), 1);
	char *logpath3;
	FILE *fp;

	assert(item);
	assert(xitem);
	assert(depn);

	if (VerboseOpt)
		printf("%s: added dependency: %s\n", item->pkgn, xitem->pkgn);

	depn->item = item;
	depn->dnext = xitem->dbase;
	xitem->dbase = depn;
	if (xitem->status == XDONE) {
		if (xitem->xcode) {
			assert(item->status == XWAITING ||
			       item->status == XDEPFAIL);
			item->xcode = xitem->xcode;
			item->status = XDEPFAIL;
			logpath3 = xbps_xasprintf("%s/deps/%s", LogDir, item->pkgn);
			fp = fopen(logpath3, "a");
			fprintf(fp, "%s\n", xitem->pkgn);
			fclose(fp);
			free(logpath3);
		}
	} else {
		++item->dcount;
	}
}

/*
 * Recursively execute './xbps-src show-build-deps' to calculate all required
 * dependencies.
 */
static struct item *
ordered_depends(const char *bpath, const char *pkgn)
{
	struct item *item, *xitem;
	char buf[1024];
	FILE *fp;
	char *cmd;

	assert(bpath);
	assert(pkgn);

	item = addItem(pkgn);
	/*
	 * Retrieve and process dependencies recursively.  Note that
	 * addDepn() can modify item's status.
	 *
	 * Guard the recursion by bumping dcount to prevent the item
	 * from being processed for completion while we are still adding
	 * its dependencies.  This would normally not occur but it can
	 * if a pkg has a broken dependency loop.
	 */
	++item->dcount;

	if (VerboseOpt)
		printf("%s: collecting build dependencies...\n", pkgn);

	if (TargetArch != NULL)
		cmd = xbps_xasprintf("%s/xbps-src -a %s show-build-deps %s 2>&1", bpath, TargetArch, pkgn);
	else
		cmd = xbps_xasprintf("%s/xbps-src show-build-deps %s 2>&1", bpath, pkgn);

	fp = popen(cmd, "r");
	while (fgets(buf, sizeof(buf), fp) != NULL) {
		size_t len;

		if (strncmp(buf, "=> ERROR", 8) == 0) {
			/* ignore pkgs returning errors */
			item->emsg = strdup(buf);
			item->status = XBROKEN;
			item->xcode = EXIT_FAILURE;
			break;
		} else if (strncmp(buf, "=>", 2) == 0) {
			/* ignore xbps-src msgs */
			continue;
		}
		len = strlen(buf);
		if (len && buf[len-1] == '\n')
			buf[--len] = 0;

		if (VerboseOpt)
			printf("%s: depends on %s\n", pkgn, buf);

		xitem = lookupItem(buf);
		if (xitem == NULL)
			xitem = ordered_depends(bpath, buf);
		addDepn(item, xitem);
	}
	pclose(fp);
	free(cmd);
	--item->dcount;

	/*
	 * If the item has no dependencies left either add it to the
	 * build list or do completion processing (i.e. if some of the
	 * dependencies failed).
	 */
	if (item->dcount == 0) {
		switch (item->status) {
		case XWAITING:
			addBuild(item);
			break;
		case XBROKEN:
		case XDEPFAIL:
			processCompletion(item);
			break;
		default:
			assert(0);
			/* NOT REACHED */
			break;
		}
	} else {
		if (VerboseOpt)
			printf("Deferred package: %s\n", item->pkgn);
	}
	runBuilds(bpath);
	return item;
}

int
main(int argc, char **argv)
{
	DIR *dir;
	struct dirent *den;
	struct stat st;
	const char *progname = argv[0];
	char *bpath, *rpath, *minit, *tmp, cwd[PATH_MAX-1];
	size_t blen;
	int ch;
	const struct option longopts[] = {
		{ NULL, 0, NULL, 0 }
	};

	while ((ch = getopt_long(argc, argv, "a:hj:l:vV", longopts, NULL)) != -1) {
		switch (ch) {
		case 'a':
			TargetArch = optarg;
			break;
		case 'j':
			NParallel = strtol(optarg, NULL, 0);
			break;
		case 'v':
			VerboseOpt = 1;
			break;
		case 'l':
			LogDir = optarg;
			break;
		case 'V':
			printf("%s\n", XBPS_RELVER);
			exit(EXIT_SUCCESS);
		case 'h':
		default:
			usage(progname);
			/* NOT REACHED */
		}
	}
	argc -= optind;
	argv += optind;

	if (argc < 1) {
		usage(progname);
		/* NOT REACHED */
	}

	if ((bpath = realpath(argv[0], NULL)) == NULL)
		exit(EXIT_FAILURE);

	blen = strlen(bpath) + strlen("/srcpkgs") + 1;
	rpath = malloc(blen);
	assert(rpath);
	snprintf(rpath, blen, "%s/srcpkgs", bpath);

	minit = xbps_xasprintf("%s/masterdir/.xbps_chroot_init", bpath);
	if (access(rpath, R_OK) == -1) {
		fprintf(stderr, "ERROR: %s/masterdir wasn't initialized, "
		    "run binary-bootstrap first.\n", bpath);
		exit(EXIT_FAILURE);
	}
	free(minit);
	/*
	 * Create LogDir and its subdirs.
	 */
	if (getcwd(cwd, sizeof(cwd)) == NULL)
		exit(EXIT_FAILURE);

	if (LogDir == NULL) {
		tmp = xbps_xasprintf("%s/log.%u", cwd, (unsigned)getpid());
	} else {
		tmp = strdup(LogDir);
	}
	assert(tmp);
	if (xbps_mkpath(tmp, 0755) != 0) {
		fprintf(stderr, "ERROR: failed to create LogDir %s: %s\n", tmp, strerror(errno));
		exit(EXIT_FAILURE);
	}
	LogDir = realpath(tmp, NULL);
	assert(tmp);
	free(tmp);

	tmp = xbps_xasprintf("%s/run", LogDir);
	if (xbps_mkpath(tmp, 0755) != 0) {
		fprintf(stderr, "ERROR: failed to create run LogDir %s: %s\n", tmp, strerror(errno));
		exit(EXIT_FAILURE);
	}
	free(tmp);
	tmp = xbps_xasprintf("%s/good", LogDir);
	if (xbps_mkpath(tmp, 0755) != 0) {
		fprintf(stderr, "ERROR: failed to create good LogDir %s: %s\n", tmp, strerror(errno));
		exit(EXIT_FAILURE);
	}
	free(tmp);
	tmp = xbps_xasprintf("%s/bad", LogDir);
	if (xbps_mkpath(tmp, 0755) != 0) {
		fprintf(stderr, "ERROR: failed to create bad LogDir %s: %s\n", tmp, strerror(errno));
		exit(EXIT_FAILURE);
	}
	free(tmp);
	tmp = xbps_xasprintf("%s/skipped", LogDir);
	if (xbps_mkpath(tmp, 0755) != 0) {
		fprintf(stderr, "ERROR: failed to create skipped LogDir %s: %s\n", tmp, strerror(errno));
		exit(EXIT_FAILURE);
	}
	free(tmp);
	tmp = xbps_xasprintf("%s/deps", LogDir);
	if (xbps_mkpath(tmp, 0755) != 0) {
		fprintf(stderr, "ERROR: failed to create deps LogDir %s: %s\n", tmp, strerror(errno));
		exit(EXIT_FAILURE);
	}
	free(tmp);

	/*
	 * Process all directories in void-packages/srcpkgs, excluding symlinks
	 * (subpackages).
	 */
	assert(chdir(rpath) == 0);

	if ((dir = opendir(rpath)) != NULL) {
		while ((den = readdir(dir)) != NULL) {
			char *xpath;
			bool found = false;

			if (den->d_name[0] == '.')
				continue;

			if (lstat(den->d_name, &st) == -1)
				continue;

			if (!S_ISDIR(st.st_mode))
				continue;

			if (argc == 1) {
				/* process all pkgs */
				found = true;
			}
			/* only process pkgs specified as arguments */
			for (int i = 1; i < argc; i++) {
				if (strcmp(argv[i], den->d_name) == 0) {
					found = true;
					break;
				}
			}
			if (!found)
				continue;

			xpath = xbps_xasprintf("%s/template", den->d_name);

			if (lookupItem(den->d_name) == NULL &&
			    stat(xpath, &st) == 0)
				ordered_depends(bpath, den->d_name);

			free(xpath);
		}
		(void)closedir(dir);
	}
	/*
	 * Wait for all current builds to finish running, keep the pipeline
	 * full until both the BuildList and RunList have been exhausted.
	 */
	free(rpath);
	runBuilds(bpath);
	while (waitRunning(0) != NULL)
		runBuilds(bpath);

	exit(EXIT_SUCCESS);
}