xbps-repo: made 'genindex' code run ~60% faster than before!
Rather than externalizing the package index plist file to storage every time a package is registered, only do it one time once all packages have been processed. Creating an index with 700 pkgs now takes 14s in a VM, where before it took 39s. As consequence of this, the 'add-pkgidx' target is no longer relevant or useful, remove it. --HG-- extra : convert_revision : xtraeme%40gmail.com-20091118053439-ct1chn4hdeir1pdo
This commit is contained in:
Juan RP
parent
cc6417b9bf
commit
f859663a4b
@ -73,15 +73,15 @@ out:
|
||||
}
|
||||
|
||||
int
|
||||
xbps_repo_addpkg_index(const char *pkgdir, const char *file)
|
||||
xbps_repo_addpkg_index(prop_dictionary_t idxdict, const char *file)
|
||||
{
|
||||
prop_dictionary_t newpkgd, idxdict, curpkgd;
|
||||
prop_dictionary_t newpkgd, curpkgd;
|
||||
prop_array_t pkgar;
|
||||
struct archive *ar;
|
||||
struct archive_entry *entry;
|
||||
struct stat st;
|
||||
const char *pkgname, *version, *regver;
|
||||
char *sha256, *plist, *filen = NULL, *tmpfilen = NULL;
|
||||
char *sha256, *filen = NULL, *tmpfilen = NULL;
|
||||
int rv = 0;
|
||||
|
||||
assert(file != NULL);
|
||||
@ -112,19 +112,12 @@ xbps_repo_addpkg_index(const char *pkgdir, const char *file)
|
||||
goto out1;
|
||||
}
|
||||
|
||||
/* Get existing or create repo index dictionary */
|
||||
idxdict = repoidx_getdict(pkgdir);
|
||||
if (idxdict == NULL) {
|
||||
/* Get package array in repo index file */
|
||||
pkgar = prop_dictionary_get(idxdict, "packages");
|
||||
if (pkgar == NULL) {
|
||||
rv = errno;
|
||||
goto out1;
|
||||
}
|
||||
plist = xbps_get_pkg_index_plist(pkgdir);
|
||||
if (plist == NULL) {
|
||||
prop_dictionary_remove(idxdict, "packages");
|
||||
rv = ENOMEM;
|
||||
goto out2;
|
||||
}
|
||||
|
||||
/*
|
||||
* Open the binary package and read the props.plist
|
||||
* into a buffer.
|
||||
@ -156,10 +149,11 @@ xbps_repo_addpkg_index(const char *pkgdir, const char *file)
|
||||
prop_dictionary_get_cstring_nocopy(curpkgd,
|
||||
"version", ®ver);
|
||||
if (xbps_cmpver(version, regver) <= 0) {
|
||||
printf("Skipping %s. Version %s already "
|
||||
"registered.\n", filen, regver);
|
||||
printf("W: skipping %s. %s-%s already "
|
||||
"registered.\n", filen, pkgname, regver);
|
||||
prop_object_release(newpkgd);
|
||||
archive_read_data_skip(ar);
|
||||
rv = EEXIST;
|
||||
break;
|
||||
}
|
||||
/*
|
||||
@ -199,32 +193,19 @@ xbps_repo_addpkg_index(const char *pkgdir, const char *file)
|
||||
/*
|
||||
* Add dictionary into the index and update package count.
|
||||
*/
|
||||
pkgar = prop_dictionary_get(idxdict, "packages");
|
||||
if (pkgar == NULL) {
|
||||
prop_object_release(newpkgd);
|
||||
rv = errno;
|
||||
break;
|
||||
}
|
||||
if (!xbps_add_obj_to_array(pkgar, newpkgd)) {
|
||||
prop_object_release(newpkgd);
|
||||
rv = EINVAL;
|
||||
break;
|
||||
}
|
||||
|
||||
prop_dictionary_set_uint64(idxdict, "total-pkgs",
|
||||
prop_array_count(pkgar));
|
||||
if (!prop_dictionary_externalize_to_file(idxdict, plist)) {
|
||||
rv = errno;
|
||||
break;
|
||||
}
|
||||
printf("Registered %s-%s in package index.\n",
|
||||
pkgname, version);
|
||||
printf("\033[1A\033[K");
|
||||
break;
|
||||
}
|
||||
|
||||
free(plist);
|
||||
out2:
|
||||
prop_object_release(idxdict);
|
||||
out1:
|
||||
archive_read_finish(ar);
|
||||
out:
|
||||
@ -236,16 +217,32 @@ out:
|
||||
int
|
||||
xbps_repo_genindex(const char *pkgdir)
|
||||
{
|
||||
prop_dictionary_t idxdict = NULL;
|
||||
struct dirent *dp;
|
||||
DIR *dirp;
|
||||
struct utsname un;
|
||||
char *binfile, *path;
|
||||
uint64_t npkgcnt = 0;
|
||||
char *binfile, *path, *plist;
|
||||
size_t i;
|
||||
int rv = 0;
|
||||
bool foundpkg = false;
|
||||
bool registered_newpkgs = false, foundpkg = false;
|
||||
|
||||
if (uname(&un) == -1)
|
||||
return errno;
|
||||
|
||||
/*
|
||||
* Create or read existing package index plist file.
|
||||
*/
|
||||
idxdict = repoidx_getdict(pkgdir);
|
||||
if (idxdict == NULL)
|
||||
return errno;
|
||||
|
||||
plist = xbps_get_pkg_index_plist(pkgdir);
|
||||
if (plist == NULL) {
|
||||
prop_object_release(idxdict);
|
||||
return errno;
|
||||
}
|
||||
|
||||
/*
|
||||
* Iterate over the known architecture directories to find
|
||||
* binary packages.
|
||||
@ -256,8 +253,10 @@ xbps_repo_genindex(const char *pkgdir)
|
||||
continue;
|
||||
|
||||
path = xbps_xasprintf("%s/%s", pkgdir, archdirs[i]);
|
||||
if (path == NULL)
|
||||
if (path == NULL) {
|
||||
prop_object_release(idxdict);
|
||||
return errno;
|
||||
}
|
||||
|
||||
dirp = opendir(path);
|
||||
if (dirp == NULL) {
|
||||
@ -279,22 +278,48 @@ xbps_repo_genindex(const char *pkgdir)
|
||||
if (binfile == NULL) {
|
||||
(void)closedir(dirp);
|
||||
free(path);
|
||||
prop_object_release(idxdict);
|
||||
free(plist);
|
||||
return errno;
|
||||
}
|
||||
rv = xbps_repo_addpkg_index(pkgdir, binfile);
|
||||
rv = xbps_repo_addpkg_index(idxdict, binfile);
|
||||
free(binfile);
|
||||
if (rv != 0) {
|
||||
if (rv == EEXIST)
|
||||
continue;
|
||||
else if (rv != 0) {
|
||||
(void)closedir(dirp);
|
||||
free(path);
|
||||
prop_object_release(idxdict);
|
||||
free(plist);
|
||||
return rv;
|
||||
}
|
||||
registered_newpkgs = true;
|
||||
}
|
||||
(void)closedir(dirp);
|
||||
free(path);
|
||||
}
|
||||
|
||||
if (foundpkg == false)
|
||||
if (foundpkg == false) {
|
||||
/* No packages were found in directory */
|
||||
rv = ENOENT;
|
||||
} else {
|
||||
/*
|
||||
* Show total count registered packages.
|
||||
*/
|
||||
prop_dictionary_get_uint64(idxdict, "total-pkgs", &npkgcnt);
|
||||
printf("%ju packages registered in package index.\n", npkgcnt);
|
||||
/*
|
||||
* Don't write plist file if no packages were registered.
|
||||
*/
|
||||
if (registered_newpkgs == false)
|
||||
return rv;
|
||||
/*
|
||||
* If any package was registered in package index, write
|
||||
* plist file to storage.
|
||||
*/
|
||||
if (!prop_dictionary_externalize_to_file(idxdict, plist))
|
||||
rv = errno;
|
||||
}
|
||||
|
||||
return rv;
|
||||
}
|
||||
|
||||
@ -27,6 +27,6 @@
|
||||
#define _XBPS_REPO_INDEX_H_
|
||||
|
||||
int xbps_repo_genindex(const char *);
|
||||
int xbps_repo_addpkg_index(const char *, const char *);
|
||||
int xbps_repo_addpkg_index(prop_dictionary_t, const char *);
|
||||
|
||||
#endif /* !_XBPS_REPO_INDEX_H_ */
|
||||
|
||||
@ -49,10 +49,9 @@ usage(void)
|
||||
{
|
||||
printf("Usage: xbps-repo [options] [action] [arguments]\n\n"
|
||||
" Available actions:\n"
|
||||
" add, add-pkgidx, genindex, list, remove, search, show\n"
|
||||
" add, genindex, list, remove, search, show\n"
|
||||
" Actions with arguments:\n"
|
||||
" add\t\t<URI>\n"
|
||||
" add-pkgidx\t<repo> <binpkg>\n"
|
||||
" genindex\t<path>\n"
|
||||
" remove\t<URI>\n"
|
||||
" search\t<string>\n"
|
||||
@ -69,7 +68,6 @@ usage(void)
|
||||
" $ xbps-repo remove /path/to/directory\n"
|
||||
" $ xbps-repo search klibc\n"
|
||||
" $ xbps-repo show klibc\n"
|
||||
" $ xbps-repo add-pkgidx /pkgdir /pkgdir/noarch/foo.xbps\n"
|
||||
" $ xbps-repo genindex /pkgdir\n");
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
@ -147,6 +145,7 @@ add_repository(const char *uri, bool remote)
|
||||
if (!sanitize_localpath(idxstr, uri))
|
||||
return errno;
|
||||
|
||||
printf("Fetching remote package index at %s...\n", uri);
|
||||
rv = xbps_sync_repository_pkg_index(idxstr);
|
||||
if (rv != 0)
|
||||
return rv;
|
||||
@ -301,15 +300,6 @@ main(int argc, char **argv)
|
||||
rv = xbps_repo_genindex(argv[1]);
|
||||
exit(rv);
|
||||
|
||||
} else if (strcasecmp(argv[0], "add-pkgidx") == 0) {
|
||||
/* Add a binary package into a package repository. */
|
||||
if (argc != 3)
|
||||
usage();
|
||||
|
||||
rv = xbps_repo_addpkg_index(argv[1], argv[2]);
|
||||
if (rv != 0)
|
||||
exit(rv);
|
||||
|
||||
} else if (strcasecmp(argv[0], "sync") == 0) {
|
||||
/* Syncs the pkg index file from a remote repo */
|
||||
if (argc != 2)
|
||||
|
||||
@ -43,16 +43,12 @@ Please note that all targets are *case insensitive*.
|
||||
Local (by specifying a *directory*) and remote (by specifiying an
|
||||
*HTTP/HTTPS/FTP URL*) repositories can be specified.
|
||||
|
||||
*add-pkgidx* '/path/to/local/repo' '/path/to/local/repo/arch/binpkg'::
|
||||
Adds the binary package 'binpkg' as specified in the second argument
|
||||
into the package index of a local repository as specified in the
|
||||
first argument. Please note that 'binpkg' must also live in the same
|
||||
local repository, you need to specify the full path to the file.
|
||||
|
||||
*genindex* '/path/to/local/repo'::
|
||||
Generates a package index for a local repository as specified in
|
||||
its argument. The same argument should be used for the *add*
|
||||
and *remove* targets.
|
||||
its argument. It will look for archives with *.xbps* extension
|
||||
and will only add it into the index if version is newer than the one
|
||||
available in the index. The same argument should be used for the *add*
|
||||
and *remove* targets.
|
||||
|
||||
*list*::
|
||||
Lists all currently registered repositories in repository pool.
|
||||
|
||||
Loading…
Reference in New Issue
Block a user