xbps-fbulk: misc changes (v3).

- Re-add original behaviour now I fixed the real issue for missing
  logs. Keep the build/run pipeline full as soon as possible.
- Added -s, --system. System build mode. To only build pkgs that
  are installed manually in your system.
- Added long options; sync usage.
- Restrict max jobs to ncores; there are issues with shared data,
  and until they are resolved this is the only way to make it work
  reliably.

bin/xbps-fbulk/main.c

@@ -38,7 +38,7 @@
  * This program iterates all srcpkgs directories, runs './xbps-src show-build-deps',
  * and builds a dependency tree on the fly.
  *
- * When the dependency tree is built, terminal dependencies are built
+ * As the dependency tree is being built, terminal dependencies are built
  * and packaged on the fly.
  *
  * As these builds complete additional dependencies may be satisfied and be
@@ -100,7 +100,6 @@ int NRunning;
 unsigned int NBuilt = 0;
 unsigned int NFinished = 0;
 unsigned int NChecked = 0;
-unsigned int NSkipped = 0;
 unsigned int NTotal = 0;
 char *LogDir;
 
@@ -135,8 +134,14 @@ addItem(const char *pkgn)
 static void __attribute__((noreturn))
 usage(const char *progname)
 {
-	fprintf(stderr, "%s [-h] [-j procs] [-l logdir] [-V]"
+	fprintf(stderr, "%s [OPTIONS] /path/to/void-packages [pkg pkg+N]\n"
-	    " /path/to/void-packages [pkg pkgN]\n", progname);
+			"OPTIONS\n"
+			" -j, --jobs <N>       Number of parallel builds\n"
+			" -l, --logdir <path>  Path to store logs\n"
+		        " -s, --system         System rebuild mode\n"
+			" -V, --verbose        Enable verbose mode\n"
+			" -v, --version        Show XBPS version\n"
+			" -h, --help           Show usage\n\n", progname);
 	exit(EXIT_FAILURE);
 }
 
@@ -171,7 +176,7 @@ processCompletion(struct item *item)
 	assert(item);
 	/*
 	 * If XRUN we have to move the logfile to the correct directory.
-	 * (If XDEPFAIL the log is at correct location).
+	 * (If XDEPFAIL the logfile is already in the correct directory).
 	 */
 	if (item->status == XRUN) {
 		logpath = xbps_xasprintf("%s/run/%s.txt", LogDir, item->pkgn);
@@ -192,9 +197,18 @@ processCompletion(struct item *item)
 		free(logpath2);
 	}
 
+	/*
+	 * Make sure that item has already run (XRUN) or
+	 * failed due to dependencies (XDEPFAIL).
+	 *
+	 * When XWAITING the item is waiting for its dependencies.
+	 * When XBUILD the item is in the build list.
+	 */
 	assert(item->status == XRUN || item->status == XDEPFAIL);
-	item->status = XDONE;
 
+	/*
+	 * Process reverse dependencies for the item.
+	 */
 	for (depn = item->dbase; depn; depn = depn->dnext) {
 		xitem = depn->item;
 		assert(xitem->dcount > 0);
@@ -224,10 +238,13 @@ processCompletion(struct item *item)
 				fclose(fp);
 				free(logpath);
 				processCompletion(xitem);
-				++NSkipped;
 			}
 		}
 	}
+	/*
+	 * Item has been processed successfully.
+	 */
+	item->status = XDONE;
 	++NFinished;
 
 	printf("[%u/%u] Finished %s (PID: %u RET: %d)\n",
@@ -255,6 +272,10 @@ waitRunning(int flags)
 	while (((pid = waitpid(0, &status, flags)) < 0) && !flags)
 		;
 
+	/*
+	 * NOTE! The pid may be associated with one of our popen()'s
+	 *       so just ignore it if we cannot find it.
+	 */
 	if (pid > 0) {
 		status = WEXITSTATUS(status);
 		itemp = &RunList;
@@ -270,6 +291,8 @@ waitRunning(int flags)
 			--NRunning;
 			processCompletion(item);
 		}
+	} else {
+		item = NULL;
 	}
 	return item;
 }
@@ -278,8 +301,10 @@ waitRunning(int flags)
  * Start new builds from the build list and handle build completions,
  * which can potentialy add new items to the build list.
  *
- * This routine will maintain up to NParallel builds. A new build is
+ * This routine will maintain up to NParallel builds.  A new build is
- * only started once its dependencies have been completed successfully.
+ * only started once its dependencies have completed successfully so
+ * when the bulk build starts it typically takes a little while before
+ * xbps-fbulk can keep the parallel pipeline full.
  */
 static void
 runBuilds(const char *bpath)
@@ -322,7 +347,6 @@ runBuilds(const char *bpath)
 			if (chdir(bpath) < 0)
 				_exit(99);
 
-
 			fd = open(logpath, O_RDWR|O_CREAT|O_TRUNC, 0666);
 			if (fd != 1)
 				dup2(fd, 1);
@@ -360,7 +384,7 @@ runBuilds(const char *bpath)
 			++NRunning;
 			++NBuilt;
 			printf("[%u/%u] Building %s (PID: %u)\n",
-			     NBuilt+NSkipped, NTotal, item->pkgn, item->pid);
+			     NBuilt, NTotal, item->pkgn, item->pid);
 		}
 		free(logpath);
 	}
@@ -381,6 +405,8 @@ static void
 addDepn(struct item *item, struct item *xitem)
 {
 	struct depn *depn = malloc(sizeof(struct depn));
+	FILE *fp;
+	char *logpath;
 
 	assert(item);
 	assert(xitem);
@@ -389,7 +415,27 @@ addDepn(struct item *item, struct item *xitem)
 	depn->item = item;
 	depn->dnext = xitem->dbase;
 	xitem->dbase = depn;
-	++item->dcount;
+	if (xitem->status == XDONE) {
+		if (xitem->xcode) {
+			/*
+			 * If reverse dependency has failed,
+			 * current item also failed!
+			 */
+			assert(item->status == XWAITING ||
+			       item->status == XDEPFAIL);
+			item->xcode = xitem->xcode;
+			item->status = XDEPFAIL;
+			logpath = xbps_xasprintf("%s/deps/%s.txt",
+			    LogDir, item->pkgn);
+			fp = fopen(logpath, "a");
+			fprintf(fp, "%s\n", xitem->pkgn);
+			fclose(fp);
+			free(logpath);
+			++NBuilt;
+		}
+	} else {
+		++item->dcount;
+	}
 }
 
 /*
@@ -408,7 +454,8 @@ ordered_depends(const char *bpath, const char *pkgn)
 
 	item = addItem(pkgn);
 	/*
-	 * Retrieve and process dependencies recursively.
+	 * Retrieve and process dependencies recursively.  Note that
+	 * addDepn() can modify item's status.
 	 */
 	++NChecked;
 	printf("[%u] Checking %s\n", NChecked, item->pkgn);
@@ -456,20 +503,59 @@ ordered_depends(const char *bpath, const char *pkgn)
 	pclose(fp);
 	++NTotal;
 	/*
-	 * If the item has no dependencies left add it to the build list.
+	 * If the item has no dependencies left either add it to the
+	 * build list or do completion processing (i.e. if some of the
+	 * dependencies failed).
 	 */
 	if (item->dcount == 0) {
-		addBuild(item);
+		switch (item->status) {
+		case XWAITING:
+			addBuild(item);
+			break;
+		case XDEPFAIL:
+			processCompletion(item);
+			break;
+		default:
+			/*
+			 * Might happen due to excessive NParallel jobs!
+			 * Error out because this is critical.
+			 */
+			printf("%s: item->xcode %d item->status %d\n",
+			    item->pkgn, item->xcode, item->status);
+			assert(0);
+			break;
+		}
 	} else {
 		if (VerboseOpt)
 			printf("Deferred package: %s\n", item->pkgn);
 	}
+	runBuilds(bpath);
 	return item;
 }
 
+static int
+pkgdb_get_pkgs_cb(struct xbps_handle *xhp UNUSED,
+		xbps_object_t obj, const char *key UNUSED,
+		void *arg, bool *done UNUSED)
+{
+	xbps_array_t *array = arg;
+	const char *pkgname;
+	bool automatic = false;
+
+	xbps_dictionary_get_bool(obj, "automatic-install", &automatic);
+	if (automatic)
+		return 0;
+
+	xbps_dictionary_get_cstring_nocopy(obj, "pkgname", &pkgname);
+	xbps_array_add_cstring_nocopy(*array, pkgname);
+	return 0;
+}
+
 int
 main(int argc, char **argv)
 {
+	struct xbps_handle xh = {0};
+	xbps_array_t array;
 	DIR *dir;
 	struct dirent *den;
 	struct stat st;
@@ -477,13 +563,23 @@ main(int argc, char **argv)
 	const char *logdirs[] = { "good", "bad", "run", "deps", "skipped" };
 	char *bpath, *rpath, *tmp, cwd[PATH_MAX];
 	size_t blen;
-	int ch;
+	int ch, NCores, rv;
+	bool RebuildSystem = false;
 	const struct option longopts[] = {
+		{ "system", no_argument, NULL, 's' },
+		{ "jobs", required_argument, NULL, 'j' },
+		{ "logdir", required_argument, NULL, 'l' },
+		{ "verbose", no_argument, NULL, 'v' },
+		{ "version", no_argument, NULL, 'V' },
+		{ "help", no_argument, NULL, 'h' },
 		{ NULL, 0, NULL, 0 }
 	};
 
-	while ((ch = getopt_long(argc, argv, "hj:l:vV", longopts, NULL)) != -1) {
+	while ((ch = getopt_long(argc, argv, "hj:l:svV", longopts, NULL)) != -1) {
 		switch (ch) {
+		case 's':
+			RebuildSystem = true;
+			break;
 		case 'j':
 			NParallel = strtol(optarg, NULL, 0);
 			break;
@@ -510,6 +606,16 @@ main(int argc, char **argv)
 		/* NOT REACHED */
 	}
 
+	/*
+	 * FIXME
+	 * Limit NParallel to max cores, due to program design
+	 * this won't work when it's higher, and we'd need to
+	 * synchronize shared data!
+	 */
+	NCores = (int)sysconf(_SC_NPROCESSORS_ONLN);
+	if (NParallel > NCores)
+		NParallel = NCores;
+
 	/*
 	 * Check masterdir is properly initialized.
 	 */
@@ -560,6 +666,34 @@ main(int argc, char **argv)
 		free(tmp);
 	}
 
+	/*
+	 * RebuildSystem: only rebuild packages that were installed
+	 * manually.
+	 */
+	if (RebuildSystem) {
+		rv = xbps_init(&xh);
+		if (rv != 0) {
+			fprintf(stderr, "ERROR: failed to initialize libxbps: %s", strerror(rv));
+			exit(EXIT_FAILURE);
+		}
+		array = xbps_array_create();
+		rv = xbps_pkgdb_foreach_cb_multi(&xh, pkgdb_get_pkgs_cb, &array);
+		if (rv != 0) {
+			fprintf(stderr, "ERROR: xbps_pkgdb_foreach_cb_multi: %s", strerror(rv));
+			exit(EXIT_FAILURE);
+		}
+		for (unsigned int i = 0; i < xbps_array_count(array); i++) {
+			const char *pkgname = NULL;
+
+			xbps_array_get_cstring_nocopy(array, i, &pkgname);
+			if (pkgname && !lookupItem(pkgname)) {
+				ordered_depends(bpath, pkgname);
+			}
+		}
+		xbps_end(&xh);
+		goto start;
+	}
+
 	/*
 	 * Generate dependency tree. This is done in two steps to know how
 	 * many packages will be built.
@@ -606,9 +740,10 @@ main(int argc, char **argv)
 		}
 		(void)closedir(dir);
 	}
+start:
 	/*
-	 * Start building collected packages, keep the pipeline full
+	 * Wait for all current builds to finish running, keep the pipeline
-	 * until both the BuildList and RunList have been exhausted.
+	 * full until both the BuildList and RunList have been exhausted.
 	 */
 	free(rpath);
 	runBuilds(bpath);

bin/xbps-fbulk/xbps-fbulk.1

@@ -1,4 +1,4 @@
-.Dd April 14, 2020
+.Dd April 20, 2020
 .Dt XBPS-FBULK 1
 .Sh NAME
 .Nm xbps-fbulk
@@ -7,7 +7,7 @@
 .Nm xbps-fbulk
 .Op OPTIONS
 .Ar /path/to/void-packages
-.Op pkgN pkgN+1 ...
+.Op pkgN pkg+N ...
 .Sh DESCRIPTION
 The
 .Nm
@@ -19,22 +19,32 @@ arguments, and then runs
 .Ar 'xbps-src show-build-deps'
 to build a dependency tree on the fly.
 .Pp
-When the dependency tree is built, terminal dependencies are built
+As the dependency tree is built, terminal dependencies are built
 and packaged on the fly.
 .Pp
-As these builds complete additional dependencies may be satisfied and be
+As these builds complete, additional dependencies may be satisfied and be
 added to the build order. Ultimately the entire tree is built.
 .Pp
 Only one attempt is made to build any given package, no matter how many
 other packages depend on it.
+.Pp
+When using
+.Ar system mode
+only installed packages that are in manual mode (see
+.Xr xbps-pkgdb 1)
+will be processed.
+This is useful to keep up a running system up-to-date.
 .Sh OPTIONS
 .Bl -tag -width -x
-.It Fl j Ar X
+.It Fl j, Fl -jobs Ar X
 Set number of parallel builds running at the same time. By default set to 1.
-.It Fl l Ar logdir
+.It Fl l, Fl -logdir  Ar logdir
 Set the log directory. By default set to `fbulk-log.<pid>`.
 .It Fl d, Fl -debug
 Enables extra debugging shown to stderr.
+.It Fl s, Fl -system
+System build mode. If set, only packages that were installed manually
+in your system will be processed.
 .It Fl h, Fl -help
 Show the help message.
 .It Fl v, Fl -verbose
@@ -53,7 +63,7 @@ Packages that failed to build.
 .It Ar logdir/skipped
 Packages that were not built because they had to be skipped (unsupported architecture, broken or restricted).
 .It Ar logdir/deps
-Packages that were not built due to missing dependencies.
+Packages that were not built due to failed or missing dependencies.
 .El
 .Sh NOTES
 The