xbps-fbulk: misc changes (v3).
- Re-add original behaviour now I fixed the real issue for missing logs. Keep the build/run pipeline full as soon as possible. - Added -s, --system. System build mode. To only build pkgs that are installed manually in your system. - Added long options; sync usage. - Restrict max jobs to ncores; there are issues with shared data, and until they are resolved this is the only way to make it work reliably.
This commit is contained in:
parent
a1d66032bd
commit
1f4cfc377b
@ -38,7 +38,7 @@
|
||||
* This program iterates all srcpkgs directories, runs './xbps-src show-build-deps',
|
||||
* and builds a dependency tree on the fly.
|
||||
*
|
||||
* When the dependency tree is built, terminal dependencies are built
|
||||
* As the dependency tree is being built, terminal dependencies are built
|
||||
* and packaged on the fly.
|
||||
*
|
||||
* As these builds complete additional dependencies may be satisfied and be
|
||||
@ -100,7 +100,6 @@ int NRunning;
|
||||
unsigned int NBuilt = 0;
|
||||
unsigned int NFinished = 0;
|
||||
unsigned int NChecked = 0;
|
||||
unsigned int NSkipped = 0;
|
||||
unsigned int NTotal = 0;
|
||||
char *LogDir;
|
||||
|
||||
@ -135,8 +134,14 @@ addItem(const char *pkgn)
|
||||
static void __attribute__((noreturn))
|
||||
usage(const char *progname)
|
||||
{
|
||||
fprintf(stderr, "%s [-h] [-j procs] [-l logdir] [-V]"
|
||||
" /path/to/void-packages [pkg pkgN]\n", progname);
|
||||
fprintf(stderr, "%s [OPTIONS] /path/to/void-packages [pkg pkg+N]\n"
|
||||
"OPTIONS\n"
|
||||
" -j, --jobs <N> Number of parallel builds\n"
|
||||
" -l, --logdir <path> Path to store logs\n"
|
||||
" -s, --system System rebuild mode\n"
|
||||
" -V, --verbose Enable verbose mode\n"
|
||||
" -v, --version Show XBPS version\n"
|
||||
" -h, --help Show usage\n\n", progname);
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
|
||||
@ -171,7 +176,7 @@ processCompletion(struct item *item)
|
||||
assert(item);
|
||||
/*
|
||||
* If XRUN we have to move the logfile to the correct directory.
|
||||
* (If XDEPFAIL the log is at correct location).
|
||||
* (If XDEPFAIL the logfile is already in the correct directory).
|
||||
*/
|
||||
if (item->status == XRUN) {
|
||||
logpath = xbps_xasprintf("%s/run/%s.txt", LogDir, item->pkgn);
|
||||
@ -192,9 +197,18 @@ processCompletion(struct item *item)
|
||||
free(logpath2);
|
||||
}
|
||||
|
||||
/*
|
||||
* Make sure that item has already run (XRUN) or
|
||||
* failed due to dependencies (XDEPFAIL).
|
||||
*
|
||||
* When XWAITING the item is waiting for its dependencies.
|
||||
* When XBUILD the item is in the build list.
|
||||
*/
|
||||
assert(item->status == XRUN || item->status == XDEPFAIL);
|
||||
item->status = XDONE;
|
||||
|
||||
/*
|
||||
* Process reverse dependencies for the item.
|
||||
*/
|
||||
for (depn = item->dbase; depn; depn = depn->dnext) {
|
||||
xitem = depn->item;
|
||||
assert(xitem->dcount > 0);
|
||||
@ -224,10 +238,13 @@ processCompletion(struct item *item)
|
||||
fclose(fp);
|
||||
free(logpath);
|
||||
processCompletion(xitem);
|
||||
++NSkipped;
|
||||
}
|
||||
}
|
||||
}
|
||||
/*
|
||||
* Item has been processed successfully.
|
||||
*/
|
||||
item->status = XDONE;
|
||||
++NFinished;
|
||||
|
||||
printf("[%u/%u] Finished %s (PID: %u RET: %d)\n",
|
||||
@ -255,6 +272,10 @@ waitRunning(int flags)
|
||||
while (((pid = waitpid(0, &status, flags)) < 0) && !flags)
|
||||
;
|
||||
|
||||
/*
|
||||
* NOTE! The pid may be associated with one of our popen()'s
|
||||
* so just ignore it if we cannot find it.
|
||||
*/
|
||||
if (pid > 0) {
|
||||
status = WEXITSTATUS(status);
|
||||
itemp = &RunList;
|
||||
@ -270,6 +291,8 @@ waitRunning(int flags)
|
||||
--NRunning;
|
||||
processCompletion(item);
|
||||
}
|
||||
} else {
|
||||
item = NULL;
|
||||
}
|
||||
return item;
|
||||
}
|
||||
@ -278,8 +301,10 @@ waitRunning(int flags)
|
||||
* Start new builds from the build list and handle build completions,
|
||||
* which can potentialy add new items to the build list.
|
||||
*
|
||||
* This routine will maintain up to NParallel builds. A new build is
|
||||
* only started once its dependencies have been completed successfully.
|
||||
* This routine will maintain up to NParallel builds. A new build is
|
||||
* only started once its dependencies have completed successfully so
|
||||
* when the bulk build starts it typically takes a little while before
|
||||
* xbps-fbulk can keep the parallel pipeline full.
|
||||
*/
|
||||
static void
|
||||
runBuilds(const char *bpath)
|
||||
@ -322,7 +347,6 @@ runBuilds(const char *bpath)
|
||||
if (chdir(bpath) < 0)
|
||||
_exit(99);
|
||||
|
||||
|
||||
fd = open(logpath, O_RDWR|O_CREAT|O_TRUNC, 0666);
|
||||
if (fd != 1)
|
||||
dup2(fd, 1);
|
||||
@ -360,7 +384,7 @@ runBuilds(const char *bpath)
|
||||
++NRunning;
|
||||
++NBuilt;
|
||||
printf("[%u/%u] Building %s (PID: %u)\n",
|
||||
NBuilt+NSkipped, NTotal, item->pkgn, item->pid);
|
||||
NBuilt, NTotal, item->pkgn, item->pid);
|
||||
}
|
||||
free(logpath);
|
||||
}
|
||||
@ -381,6 +405,8 @@ static void
|
||||
addDepn(struct item *item, struct item *xitem)
|
||||
{
|
||||
struct depn *depn = malloc(sizeof(struct depn));
|
||||
FILE *fp;
|
||||
char *logpath;
|
||||
|
||||
assert(item);
|
||||
assert(xitem);
|
||||
@ -389,7 +415,27 @@ addDepn(struct item *item, struct item *xitem)
|
||||
depn->item = item;
|
||||
depn->dnext = xitem->dbase;
|
||||
xitem->dbase = depn;
|
||||
++item->dcount;
|
||||
if (xitem->status == XDONE) {
|
||||
if (xitem->xcode) {
|
||||
/*
|
||||
* If reverse dependency has failed,
|
||||
* current item also failed!
|
||||
*/
|
||||
assert(item->status == XWAITING ||
|
||||
item->status == XDEPFAIL);
|
||||
item->xcode = xitem->xcode;
|
||||
item->status = XDEPFAIL;
|
||||
logpath = xbps_xasprintf("%s/deps/%s.txt",
|
||||
LogDir, item->pkgn);
|
||||
fp = fopen(logpath, "a");
|
||||
fprintf(fp, "%s\n", xitem->pkgn);
|
||||
fclose(fp);
|
||||
free(logpath);
|
||||
++NBuilt;
|
||||
}
|
||||
} else {
|
||||
++item->dcount;
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
@ -408,7 +454,8 @@ ordered_depends(const char *bpath, const char *pkgn)
|
||||
|
||||
item = addItem(pkgn);
|
||||
/*
|
||||
* Retrieve and process dependencies recursively.
|
||||
* Retrieve and process dependencies recursively. Note that
|
||||
* addDepn() can modify item's status.
|
||||
*/
|
||||
++NChecked;
|
||||
printf("[%u] Checking %s\n", NChecked, item->pkgn);
|
||||
@ -456,20 +503,59 @@ ordered_depends(const char *bpath, const char *pkgn)
|
||||
pclose(fp);
|
||||
++NTotal;
|
||||
/*
|
||||
* If the item has no dependencies left add it to the build list.
|
||||
* If the item has no dependencies left either add it to the
|
||||
* build list or do completion processing (i.e. if some of the
|
||||
* dependencies failed).
|
||||
*/
|
||||
if (item->dcount == 0) {
|
||||
addBuild(item);
|
||||
switch (item->status) {
|
||||
case XWAITING:
|
||||
addBuild(item);
|
||||
break;
|
||||
case XDEPFAIL:
|
||||
processCompletion(item);
|
||||
break;
|
||||
default:
|
||||
/*
|
||||
* Might happen due to excessive NParallel jobs!
|
||||
* Error out because this is critical.
|
||||
*/
|
||||
printf("%s: item->xcode %d item->status %d\n",
|
||||
item->pkgn, item->xcode, item->status);
|
||||
assert(0);
|
||||
break;
|
||||
}
|
||||
} else {
|
||||
if (VerboseOpt)
|
||||
printf("Deferred package: %s\n", item->pkgn);
|
||||
}
|
||||
runBuilds(bpath);
|
||||
return item;
|
||||
}
|
||||
|
||||
static int
|
||||
pkgdb_get_pkgs_cb(struct xbps_handle *xhp UNUSED,
|
||||
xbps_object_t obj, const char *key UNUSED,
|
||||
void *arg, bool *done UNUSED)
|
||||
{
|
||||
xbps_array_t *array = arg;
|
||||
const char *pkgname;
|
||||
bool automatic = false;
|
||||
|
||||
xbps_dictionary_get_bool(obj, "automatic-install", &automatic);
|
||||
if (automatic)
|
||||
return 0;
|
||||
|
||||
xbps_dictionary_get_cstring_nocopy(obj, "pkgname", &pkgname);
|
||||
xbps_array_add_cstring_nocopy(*array, pkgname);
|
||||
return 0;
|
||||
}
|
||||
|
||||
int
|
||||
main(int argc, char **argv)
|
||||
{
|
||||
struct xbps_handle xh = {0};
|
||||
xbps_array_t array;
|
||||
DIR *dir;
|
||||
struct dirent *den;
|
||||
struct stat st;
|
||||
@ -477,13 +563,23 @@ main(int argc, char **argv)
|
||||
const char *logdirs[] = { "good", "bad", "run", "deps", "skipped" };
|
||||
char *bpath, *rpath, *tmp, cwd[PATH_MAX];
|
||||
size_t blen;
|
||||
int ch;
|
||||
int ch, NCores, rv;
|
||||
bool RebuildSystem = false;
|
||||
const struct option longopts[] = {
|
||||
{ "system", no_argument, NULL, 's' },
|
||||
{ "jobs", required_argument, NULL, 'j' },
|
||||
{ "logdir", required_argument, NULL, 'l' },
|
||||
{ "verbose", no_argument, NULL, 'v' },
|
||||
{ "version", no_argument, NULL, 'V' },
|
||||
{ "help", no_argument, NULL, 'h' },
|
||||
{ NULL, 0, NULL, 0 }
|
||||
};
|
||||
|
||||
while ((ch = getopt_long(argc, argv, "hj:l:vV", longopts, NULL)) != -1) {
|
||||
while ((ch = getopt_long(argc, argv, "hj:l:svV", longopts, NULL)) != -1) {
|
||||
switch (ch) {
|
||||
case 's':
|
||||
RebuildSystem = true;
|
||||
break;
|
||||
case 'j':
|
||||
NParallel = strtol(optarg, NULL, 0);
|
||||
break;
|
||||
@ -510,6 +606,16 @@ main(int argc, char **argv)
|
||||
/* NOT REACHED */
|
||||
}
|
||||
|
||||
/*
|
||||
* FIXME
|
||||
* Limit NParallel to max cores, due to program design
|
||||
* this won't work when it's higher, and we'd need to
|
||||
* synchronize shared data!
|
||||
*/
|
||||
NCores = (int)sysconf(_SC_NPROCESSORS_ONLN);
|
||||
if (NParallel > NCores)
|
||||
NParallel = NCores;
|
||||
|
||||
/*
|
||||
* Check masterdir is properly initialized.
|
||||
*/
|
||||
@ -560,6 +666,34 @@ main(int argc, char **argv)
|
||||
free(tmp);
|
||||
}
|
||||
|
||||
/*
|
||||
* RebuildSystem: only rebuild packages that were installed
|
||||
* manually.
|
||||
*/
|
||||
if (RebuildSystem) {
|
||||
rv = xbps_init(&xh);
|
||||
if (rv != 0) {
|
||||
fprintf(stderr, "ERROR: failed to initialize libxbps: %s", strerror(rv));
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
array = xbps_array_create();
|
||||
rv = xbps_pkgdb_foreach_cb_multi(&xh, pkgdb_get_pkgs_cb, &array);
|
||||
if (rv != 0) {
|
||||
fprintf(stderr, "ERROR: xbps_pkgdb_foreach_cb_multi: %s", strerror(rv));
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
for (unsigned int i = 0; i < xbps_array_count(array); i++) {
|
||||
const char *pkgname = NULL;
|
||||
|
||||
xbps_array_get_cstring_nocopy(array, i, &pkgname);
|
||||
if (pkgname && !lookupItem(pkgname)) {
|
||||
ordered_depends(bpath, pkgname);
|
||||
}
|
||||
}
|
||||
xbps_end(&xh);
|
||||
goto start;
|
||||
}
|
||||
|
||||
/*
|
||||
* Generate dependency tree. This is done in two steps to know how
|
||||
* many packages will be built.
|
||||
@ -606,9 +740,10 @@ main(int argc, char **argv)
|
||||
}
|
||||
(void)closedir(dir);
|
||||
}
|
||||
start:
|
||||
/*
|
||||
* Start building collected packages, keep the pipeline full
|
||||
* until both the BuildList and RunList have been exhausted.
|
||||
* Wait for all current builds to finish running, keep the pipeline
|
||||
* full until both the BuildList and RunList have been exhausted.
|
||||
*/
|
||||
free(rpath);
|
||||
runBuilds(bpath);
|
||||
|
@ -1,4 +1,4 @@
|
||||
.Dd April 14, 2020
|
||||
.Dd April 20, 2020
|
||||
.Dt XBPS-FBULK 1
|
||||
.Sh NAME
|
||||
.Nm xbps-fbulk
|
||||
@ -7,7 +7,7 @@
|
||||
.Nm xbps-fbulk
|
||||
.Op OPTIONS
|
||||
.Ar /path/to/void-packages
|
||||
.Op pkgN pkgN+1 ...
|
||||
.Op pkgN pkg+N ...
|
||||
.Sh DESCRIPTION
|
||||
The
|
||||
.Nm
|
||||
@ -19,22 +19,32 @@ arguments, and then runs
|
||||
.Ar 'xbps-src show-build-deps'
|
||||
to build a dependency tree on the fly.
|
||||
.Pp
|
||||
When the dependency tree is built, terminal dependencies are built
|
||||
As the dependency tree is built, terminal dependencies are built
|
||||
and packaged on the fly.
|
||||
.Pp
|
||||
As these builds complete additional dependencies may be satisfied and be
|
||||
As these builds complete, additional dependencies may be satisfied and be
|
||||
added to the build order. Ultimately the entire tree is built.
|
||||
.Pp
|
||||
Only one attempt is made to build any given package, no matter how many
|
||||
other packages depend on it.
|
||||
.Pp
|
||||
When using
|
||||
.Ar system mode
|
||||
only installed packages that are in manual mode (see
|
||||
.Xr xbps-pkgdb 1)
|
||||
will be processed.
|
||||
This is useful to keep up a running system up-to-date.
|
||||
.Sh OPTIONS
|
||||
.Bl -tag -width -x
|
||||
.It Fl j Ar X
|
||||
.It Fl j, Fl -jobs Ar X
|
||||
Set number of parallel builds running at the same time. By default set to 1.
|
||||
.It Fl l Ar logdir
|
||||
.It Fl l, Fl -logdir Ar logdir
|
||||
Set the log directory. By default set to `fbulk-log.<pid>`.
|
||||
.It Fl d, Fl -debug
|
||||
Enables extra debugging shown to stderr.
|
||||
.It Fl s, Fl -system
|
||||
System build mode. If set, only packages that were installed manually
|
||||
in your system will be processed.
|
||||
.It Fl h, Fl -help
|
||||
Show the help message.
|
||||
.It Fl v, Fl -verbose
|
||||
@ -53,7 +63,7 @@ Packages that failed to build.
|
||||
.It Ar logdir/skipped
|
||||
Packages that were not built because they had to be skipped (unsupported architecture, broken or restricted).
|
||||
.It Ar logdir/deps
|
||||
Packages that were not built due to missing dependencies.
|
||||
Packages that were not built due to failed or missing dependencies.
|
||||
.El
|
||||
.Sh NOTES
|
||||
The
|
||||
|
Loading…
Reference in New Issue
Block a user