From f859663a4b3d3b8685ba3778d932974b29e1cf4a Mon Sep 17 00:00:00 2001 From: Juan RP Date: Wed, 18 Nov 2009 06:34:39 +0100 Subject: [PATCH] xbps-repo: made 'genindex' code run ~60% faster than before! Rather than externalizing the package index plist file to storage every time a package is registered, only do it one time once all packages have been processed. Creating an index with 700 pkgs now takes 14s in a VM, where before it took 39s. As consequence of this, the 'add-pkgidx' target is no longer relevant or useful, remove it. --HG-- extra : convert_revision : xtraeme%40gmail.com-20091118053439-ct1chn4hdeir1pdo --- bin/xbps-repo/index.c | 95 ++++++++++++++++++++++------------- bin/xbps-repo/index.h | 2 +- bin/xbps-repo/main.c | 14 +----- bin/xbps-repo/xbps-repo.8.txt | 12 ++--- 4 files changed, 67 insertions(+), 56 deletions(-) diff --git a/bin/xbps-repo/index.c b/bin/xbps-repo/index.c index 1ff75a60..cef87d98 100644 --- a/bin/xbps-repo/index.c +++ b/bin/xbps-repo/index.c @@ -73,15 +73,15 @@ out: } int -xbps_repo_addpkg_index(const char *pkgdir, const char *file) +xbps_repo_addpkg_index(prop_dictionary_t idxdict, const char *file) { - prop_dictionary_t newpkgd, idxdict, curpkgd; + prop_dictionary_t newpkgd, curpkgd; prop_array_t pkgar; struct archive *ar; struct archive_entry *entry; struct stat st; const char *pkgname, *version, *regver; - char *sha256, *plist, *filen = NULL, *tmpfilen = NULL; + char *sha256, *filen = NULL, *tmpfilen = NULL; int rv = 0; assert(file != NULL); @@ -112,19 +112,12 @@ xbps_repo_addpkg_index(const char *pkgdir, const char *file) goto out1; } - /* Get existing or create repo index dictionary */ - idxdict = repoidx_getdict(pkgdir); - if (idxdict == NULL) { + /* Get package array in repo index file */ + pkgar = prop_dictionary_get(idxdict, "packages"); + if (pkgar == NULL) { rv = errno; goto out1; } - plist = xbps_get_pkg_index_plist(pkgdir); - if (plist == NULL) { - prop_dictionary_remove(idxdict, "packages"); - rv = ENOMEM; - goto out2; - } - /* * Open the binary package and read the props.plist * into a buffer. @@ -156,10 +149,11 @@ xbps_repo_addpkg_index(const char *pkgdir, const char *file) prop_dictionary_get_cstring_nocopy(curpkgd, "version", ®ver); if (xbps_cmpver(version, regver) <= 0) { - printf("Skipping %s. Version %s already " - "registered.\n", filen, regver); + printf("W: skipping %s. %s-%s already " + "registered.\n", filen, pkgname, regver); prop_object_release(newpkgd); archive_read_data_skip(ar); + rv = EEXIST; break; } /* @@ -199,32 +193,19 @@ xbps_repo_addpkg_index(const char *pkgdir, const char *file) /* * Add dictionary into the index and update package count. */ - pkgar = prop_dictionary_get(idxdict, "packages"); - if (pkgar == NULL) { - prop_object_release(newpkgd); - rv = errno; - break; - } if (!xbps_add_obj_to_array(pkgar, newpkgd)) { prop_object_release(newpkgd); rv = EINVAL; break; } - prop_dictionary_set_uint64(idxdict, "total-pkgs", prop_array_count(pkgar)); - if (!prop_dictionary_externalize_to_file(idxdict, plist)) { - rv = errno; - break; - } printf("Registered %s-%s in package index.\n", pkgname, version); + printf("\033[1A\033[K"); break; } - free(plist); -out2: - prop_object_release(idxdict); out1: archive_read_finish(ar); out: @@ -236,16 +217,32 @@ out: int xbps_repo_genindex(const char *pkgdir) { + prop_dictionary_t idxdict = NULL; struct dirent *dp; DIR *dirp; struct utsname un; - char *binfile, *path; + uint64_t npkgcnt = 0; + char *binfile, *path, *plist; size_t i; int rv = 0; - bool foundpkg = false; + bool registered_newpkgs = false, foundpkg = false; if (uname(&un) == -1) return errno; + + /* + * Create or read existing package index plist file. + */ + idxdict = repoidx_getdict(pkgdir); + if (idxdict == NULL) + return errno; + + plist = xbps_get_pkg_index_plist(pkgdir); + if (plist == NULL) { + prop_object_release(idxdict); + return errno; + } + /* * Iterate over the known architecture directories to find * binary packages. @@ -256,8 +253,10 @@ xbps_repo_genindex(const char *pkgdir) continue; path = xbps_xasprintf("%s/%s", pkgdir, archdirs[i]); - if (path == NULL) + if (path == NULL) { + prop_object_release(idxdict); return errno; + } dirp = opendir(path); if (dirp == NULL) { @@ -279,22 +278,48 @@ xbps_repo_genindex(const char *pkgdir) if (binfile == NULL) { (void)closedir(dirp); free(path); + prop_object_release(idxdict); + free(plist); return errno; } - rv = xbps_repo_addpkg_index(pkgdir, binfile); + rv = xbps_repo_addpkg_index(idxdict, binfile); free(binfile); - if (rv != 0) { + if (rv == EEXIST) + continue; + else if (rv != 0) { (void)closedir(dirp); free(path); + prop_object_release(idxdict); + free(plist); return rv; } + registered_newpkgs = true; } (void)closedir(dirp); free(path); } - if (foundpkg == false) + if (foundpkg == false) { + /* No packages were found in directory */ rv = ENOENT; + } else { + /* + * Show total count registered packages. + */ + prop_dictionary_get_uint64(idxdict, "total-pkgs", &npkgcnt); + printf("%ju packages registered in package index.\n", npkgcnt); + /* + * Don't write plist file if no packages were registered. + */ + if (registered_newpkgs == false) + return rv; + /* + * If any package was registered in package index, write + * plist file to storage. + */ + if (!prop_dictionary_externalize_to_file(idxdict, plist)) + rv = errno; + } return rv; } diff --git a/bin/xbps-repo/index.h b/bin/xbps-repo/index.h index 43844d11..6a53c55b 100644 --- a/bin/xbps-repo/index.h +++ b/bin/xbps-repo/index.h @@ -27,6 +27,6 @@ #define _XBPS_REPO_INDEX_H_ int xbps_repo_genindex(const char *); -int xbps_repo_addpkg_index(const char *, const char *); +int xbps_repo_addpkg_index(prop_dictionary_t, const char *); #endif /* !_XBPS_REPO_INDEX_H_ */ diff --git a/bin/xbps-repo/main.c b/bin/xbps-repo/main.c index 1ab55374..18b96683 100644 --- a/bin/xbps-repo/main.c +++ b/bin/xbps-repo/main.c @@ -49,10 +49,9 @@ usage(void) { printf("Usage: xbps-repo [options] [action] [arguments]\n\n" " Available actions:\n" - " add, add-pkgidx, genindex, list, remove, search, show\n" + " add, genindex, list, remove, search, show\n" " Actions with arguments:\n" " add\t\t\n" - " add-pkgidx\t \n" " genindex\t\n" " remove\t\n" " search\t\n" @@ -69,7 +68,6 @@ usage(void) " $ xbps-repo remove /path/to/directory\n" " $ xbps-repo search klibc\n" " $ xbps-repo show klibc\n" - " $ xbps-repo add-pkgidx /pkgdir /pkgdir/noarch/foo.xbps\n" " $ xbps-repo genindex /pkgdir\n"); exit(EXIT_FAILURE); } @@ -147,6 +145,7 @@ add_repository(const char *uri, bool remote) if (!sanitize_localpath(idxstr, uri)) return errno; + printf("Fetching remote package index at %s...\n", uri); rv = xbps_sync_repository_pkg_index(idxstr); if (rv != 0) return rv; @@ -301,15 +300,6 @@ main(int argc, char **argv) rv = xbps_repo_genindex(argv[1]); exit(rv); - } else if (strcasecmp(argv[0], "add-pkgidx") == 0) { - /* Add a binary package into a package repository. */ - if (argc != 3) - usage(); - - rv = xbps_repo_addpkg_index(argv[1], argv[2]); - if (rv != 0) - exit(rv); - } else if (strcasecmp(argv[0], "sync") == 0) { /* Syncs the pkg index file from a remote repo */ if (argc != 2) diff --git a/bin/xbps-repo/xbps-repo.8.txt b/bin/xbps-repo/xbps-repo.8.txt index 839e6f03..3eb2c471 100644 --- a/bin/xbps-repo/xbps-repo.8.txt +++ b/bin/xbps-repo/xbps-repo.8.txt @@ -43,16 +43,12 @@ Please note that all targets are *case insensitive*. Local (by specifying a *directory*) and remote (by specifiying an *HTTP/HTTPS/FTP URL*) repositories can be specified. -*add-pkgidx* '/path/to/local/repo' '/path/to/local/repo/arch/binpkg':: - Adds the binary package 'binpkg' as specified in the second argument - into the package index of a local repository as specified in the - first argument. Please note that 'binpkg' must also live in the same - local repository, you need to specify the full path to the file. - *genindex* '/path/to/local/repo':: Generates a package index for a local repository as specified in - its argument. The same argument should be used for the *add* - and *remove* targets. + its argument. It will look for archives with *.xbps* extension + and will only add it into the index if version is newer than the one + available in the index. The same argument should be used for the *add* + and *remove* targets. *list*:: Lists all currently registered repositories in repository pool.