From 1f4cfc377bfa18b9c86661c083466e7aa0a3ed83 Mon Sep 17 00:00:00 2001 From: Juan RP Date: Mon, 20 Apr 2020 18:50:11 +0200 Subject: [PATCH] xbps-fbulk: misc changes (v3). - Re-add original behaviour now I fixed the real issue for missing logs. Keep the build/run pipeline full as soon as possible. - Added -s, --system. System build mode. To only build pkgs that are installed manually in your system. - Added long options; sync usage. - Restrict max jobs to ncores; there are issues with shared data, and until they are resolved this is the only way to make it work reliably. --- bin/xbps-fbulk/main.c | 173 ++++++++++++++++++++++++++++++++---- bin/xbps-fbulk/xbps-fbulk.1 | 24 +++-- 2 files changed, 171 insertions(+), 26 deletions(-) diff --git a/bin/xbps-fbulk/main.c b/bin/xbps-fbulk/main.c index 1ed1a01f..829163c8 100644 --- a/bin/xbps-fbulk/main.c +++ b/bin/xbps-fbulk/main.c @@ -38,7 +38,7 @@ * This program iterates all srcpkgs directories, runs './xbps-src show-build-deps', * and builds a dependency tree on the fly. * - * When the dependency tree is built, terminal dependencies are built + * As the dependency tree is being built, terminal dependencies are built * and packaged on the fly. * * As these builds complete additional dependencies may be satisfied and be @@ -100,7 +100,6 @@ int NRunning; unsigned int NBuilt = 0; unsigned int NFinished = 0; unsigned int NChecked = 0; -unsigned int NSkipped = 0; unsigned int NTotal = 0; char *LogDir; @@ -135,8 +134,14 @@ addItem(const char *pkgn) static void __attribute__((noreturn)) usage(const char *progname) { - fprintf(stderr, "%s [-h] [-j procs] [-l logdir] [-V]" - " /path/to/void-packages [pkg pkgN]\n", progname); + fprintf(stderr, "%s [OPTIONS] /path/to/void-packages [pkg pkg+N]\n" + "OPTIONS\n" + " -j, --jobs Number of parallel builds\n" + " -l, --logdir Path to store logs\n" + " -s, --system System rebuild mode\n" + " -V, --verbose Enable verbose mode\n" + " -v, --version Show XBPS version\n" + " -h, --help Show usage\n\n", progname); exit(EXIT_FAILURE); } @@ -171,7 +176,7 @@ processCompletion(struct item *item) assert(item); /* * If XRUN we have to move the logfile to the correct directory. - * (If XDEPFAIL the log is at correct location). + * (If XDEPFAIL the logfile is already in the correct directory). */ if (item->status == XRUN) { logpath = xbps_xasprintf("%s/run/%s.txt", LogDir, item->pkgn); @@ -192,9 +197,18 @@ processCompletion(struct item *item) free(logpath2); } + /* + * Make sure that item has already run (XRUN) or + * failed due to dependencies (XDEPFAIL). + * + * When XWAITING the item is waiting for its dependencies. + * When XBUILD the item is in the build list. + */ assert(item->status == XRUN || item->status == XDEPFAIL); - item->status = XDONE; + /* + * Process reverse dependencies for the item. + */ for (depn = item->dbase; depn; depn = depn->dnext) { xitem = depn->item; assert(xitem->dcount > 0); @@ -224,10 +238,13 @@ processCompletion(struct item *item) fclose(fp); free(logpath); processCompletion(xitem); - ++NSkipped; } } } + /* + * Item has been processed successfully. + */ + item->status = XDONE; ++NFinished; printf("[%u/%u] Finished %s (PID: %u RET: %d)\n", @@ -255,6 +272,10 @@ waitRunning(int flags) while (((pid = waitpid(0, &status, flags)) < 0) && !flags) ; + /* + * NOTE! The pid may be associated with one of our popen()'s + * so just ignore it if we cannot find it. + */ if (pid > 0) { status = WEXITSTATUS(status); itemp = &RunList; @@ -270,6 +291,8 @@ waitRunning(int flags) --NRunning; processCompletion(item); } + } else { + item = NULL; } return item; } @@ -278,8 +301,10 @@ waitRunning(int flags) * Start new builds from the build list and handle build completions, * which can potentialy add new items to the build list. * - * This routine will maintain up to NParallel builds. A new build is - * only started once its dependencies have been completed successfully. + * This routine will maintain up to NParallel builds. A new build is + * only started once its dependencies have completed successfully so + * when the bulk build starts it typically takes a little while before + * xbps-fbulk can keep the parallel pipeline full. */ static void runBuilds(const char *bpath) @@ -322,7 +347,6 @@ runBuilds(const char *bpath) if (chdir(bpath) < 0) _exit(99); - fd = open(logpath, O_RDWR|O_CREAT|O_TRUNC, 0666); if (fd != 1) dup2(fd, 1); @@ -360,7 +384,7 @@ runBuilds(const char *bpath) ++NRunning; ++NBuilt; printf("[%u/%u] Building %s (PID: %u)\n", - NBuilt+NSkipped, NTotal, item->pkgn, item->pid); + NBuilt, NTotal, item->pkgn, item->pid); } free(logpath); } @@ -381,6 +405,8 @@ static void addDepn(struct item *item, struct item *xitem) { struct depn *depn = malloc(sizeof(struct depn)); + FILE *fp; + char *logpath; assert(item); assert(xitem); @@ -389,7 +415,27 @@ addDepn(struct item *item, struct item *xitem) depn->item = item; depn->dnext = xitem->dbase; xitem->dbase = depn; - ++item->dcount; + if (xitem->status == XDONE) { + if (xitem->xcode) { + /* + * If reverse dependency has failed, + * current item also failed! + */ + assert(item->status == XWAITING || + item->status == XDEPFAIL); + item->xcode = xitem->xcode; + item->status = XDEPFAIL; + logpath = xbps_xasprintf("%s/deps/%s.txt", + LogDir, item->pkgn); + fp = fopen(logpath, "a"); + fprintf(fp, "%s\n", xitem->pkgn); + fclose(fp); + free(logpath); + ++NBuilt; + } + } else { + ++item->dcount; + } } /* @@ -408,7 +454,8 @@ ordered_depends(const char *bpath, const char *pkgn) item = addItem(pkgn); /* - * Retrieve and process dependencies recursively. + * Retrieve and process dependencies recursively. Note that + * addDepn() can modify item's status. */ ++NChecked; printf("[%u] Checking %s\n", NChecked, item->pkgn); @@ -456,20 +503,59 @@ ordered_depends(const char *bpath, const char *pkgn) pclose(fp); ++NTotal; /* - * If the item has no dependencies left add it to the build list. + * If the item has no dependencies left either add it to the + * build list or do completion processing (i.e. if some of the + * dependencies failed). */ if (item->dcount == 0) { - addBuild(item); + switch (item->status) { + case XWAITING: + addBuild(item); + break; + case XDEPFAIL: + processCompletion(item); + break; + default: + /* + * Might happen due to excessive NParallel jobs! + * Error out because this is critical. + */ + printf("%s: item->xcode %d item->status %d\n", + item->pkgn, item->xcode, item->status); + assert(0); + break; + } } else { if (VerboseOpt) printf("Deferred package: %s\n", item->pkgn); } + runBuilds(bpath); return item; } +static int +pkgdb_get_pkgs_cb(struct xbps_handle *xhp UNUSED, + xbps_object_t obj, const char *key UNUSED, + void *arg, bool *done UNUSED) +{ + xbps_array_t *array = arg; + const char *pkgname; + bool automatic = false; + + xbps_dictionary_get_bool(obj, "automatic-install", &automatic); + if (automatic) + return 0; + + xbps_dictionary_get_cstring_nocopy(obj, "pkgname", &pkgname); + xbps_array_add_cstring_nocopy(*array, pkgname); + return 0; +} + int main(int argc, char **argv) { + struct xbps_handle xh = {0}; + xbps_array_t array; DIR *dir; struct dirent *den; struct stat st; @@ -477,13 +563,23 @@ main(int argc, char **argv) const char *logdirs[] = { "good", "bad", "run", "deps", "skipped" }; char *bpath, *rpath, *tmp, cwd[PATH_MAX]; size_t blen; - int ch; + int ch, NCores, rv; + bool RebuildSystem = false; const struct option longopts[] = { + { "system", no_argument, NULL, 's' }, + { "jobs", required_argument, NULL, 'j' }, + { "logdir", required_argument, NULL, 'l' }, + { "verbose", no_argument, NULL, 'v' }, + { "version", no_argument, NULL, 'V' }, + { "help", no_argument, NULL, 'h' }, { NULL, 0, NULL, 0 } }; - while ((ch = getopt_long(argc, argv, "hj:l:vV", longopts, NULL)) != -1) { + while ((ch = getopt_long(argc, argv, "hj:l:svV", longopts, NULL)) != -1) { switch (ch) { + case 's': + RebuildSystem = true; + break; case 'j': NParallel = strtol(optarg, NULL, 0); break; @@ -510,6 +606,16 @@ main(int argc, char **argv) /* NOT REACHED */ } + /* + * FIXME + * Limit NParallel to max cores, due to program design + * this won't work when it's higher, and we'd need to + * synchronize shared data! + */ + NCores = (int)sysconf(_SC_NPROCESSORS_ONLN); + if (NParallel > NCores) + NParallel = NCores; + /* * Check masterdir is properly initialized. */ @@ -560,6 +666,34 @@ main(int argc, char **argv) free(tmp); } + /* + * RebuildSystem: only rebuild packages that were installed + * manually. + */ + if (RebuildSystem) { + rv = xbps_init(&xh); + if (rv != 0) { + fprintf(stderr, "ERROR: failed to initialize libxbps: %s", strerror(rv)); + exit(EXIT_FAILURE); + } + array = xbps_array_create(); + rv = xbps_pkgdb_foreach_cb_multi(&xh, pkgdb_get_pkgs_cb, &array); + if (rv != 0) { + fprintf(stderr, "ERROR: xbps_pkgdb_foreach_cb_multi: %s", strerror(rv)); + exit(EXIT_FAILURE); + } + for (unsigned int i = 0; i < xbps_array_count(array); i++) { + const char *pkgname = NULL; + + xbps_array_get_cstring_nocopy(array, i, &pkgname); + if (pkgname && !lookupItem(pkgname)) { + ordered_depends(bpath, pkgname); + } + } + xbps_end(&xh); + goto start; + } + /* * Generate dependency tree. This is done in two steps to know how * many packages will be built. @@ -606,9 +740,10 @@ main(int argc, char **argv) } (void)closedir(dir); } +start: /* - * Start building collected packages, keep the pipeline full - * until both the BuildList and RunList have been exhausted. + * Wait for all current builds to finish running, keep the pipeline + * full until both the BuildList and RunList have been exhausted. */ free(rpath); runBuilds(bpath); diff --git a/bin/xbps-fbulk/xbps-fbulk.1 b/bin/xbps-fbulk/xbps-fbulk.1 index cf7ff5ae..81f5c540 100644 --- a/bin/xbps-fbulk/xbps-fbulk.1 +++ b/bin/xbps-fbulk/xbps-fbulk.1 @@ -1,4 +1,4 @@ -.Dd April 14, 2020 +.Dd April 20, 2020 .Dt XBPS-FBULK 1 .Sh NAME .Nm xbps-fbulk @@ -7,7 +7,7 @@ .Nm xbps-fbulk .Op OPTIONS .Ar /path/to/void-packages -.Op pkgN pkgN+1 ... +.Op pkgN pkg+N ... .Sh DESCRIPTION The .Nm @@ -19,22 +19,32 @@ arguments, and then runs .Ar 'xbps-src show-build-deps' to build a dependency tree on the fly. .Pp -When the dependency tree is built, terminal dependencies are built +As the dependency tree is built, terminal dependencies are built and packaged on the fly. .Pp -As these builds complete additional dependencies may be satisfied and be +As these builds complete, additional dependencies may be satisfied and be added to the build order. Ultimately the entire tree is built. .Pp Only one attempt is made to build any given package, no matter how many other packages depend on it. +.Pp +When using +.Ar system mode +only installed packages that are in manual mode (see +.Xr xbps-pkgdb 1) +will be processed. +This is useful to keep up a running system up-to-date. .Sh OPTIONS .Bl -tag -width -x -.It Fl j Ar X +.It Fl j, Fl -jobs Ar X Set number of parallel builds running at the same time. By default set to 1. -.It Fl l Ar logdir +.It Fl l, Fl -logdir Ar logdir Set the log directory. By default set to `fbulk-log.`. .It Fl d, Fl -debug Enables extra debugging shown to stderr. +.It Fl s, Fl -system +System build mode. If set, only packages that were installed manually +in your system will be processed. .It Fl h, Fl -help Show the help message. .It Fl v, Fl -verbose @@ -53,7 +63,7 @@ Packages that failed to build. .It Ar logdir/skipped Packages that were not built because they had to be skipped (unsupported architecture, broken or restricted). .It Ar logdir/deps -Packages that were not built due to missing dependencies. +Packages that were not built due to failed or missing dependencies. .El .Sh NOTES The