xbps-repo: made 'genindex' code run ~60% faster than before!

Rather than externalizing the package index plist file to storage every
time a package is registered, only do it one time once all packages
have been processed. Creating an index with 700 pkgs now takes 14s
in a VM, where before it took 39s. As consequence of this, the 'add-pkgidx'
target is no longer relevant or useful, remove it.

--HG--
extra : convert_revision : xtraeme%40gmail.com-20091118053439-ct1chn4hdeir1pdo

bin/xbps-repo/index.c

@@ -73,15 +73,15 @@ out:
 }
 
 int
-xbps_repo_addpkg_index(const char *pkgdir, const char *file)
+xbps_repo_addpkg_index(prop_dictionary_t idxdict, const char *file)
 {
-	prop_dictionary_t newpkgd, idxdict, curpkgd;
+	prop_dictionary_t newpkgd, curpkgd;
 	prop_array_t pkgar;
 	struct archive *ar;
 	struct archive_entry *entry;
 	struct stat st;
 	const char *pkgname, *version, *regver;
-	char *sha256, *plist, *filen = NULL, *tmpfilen = NULL;
+	char *sha256, *filen = NULL, *tmpfilen = NULL;
 	int rv = 0;
 
 	assert(file != NULL);
@@ -112,19 +112,12 @@ xbps_repo_addpkg_index(const char *pkgdir, const char *file)
 		goto out1;
 	}
 
-	/* Get existing or create repo index dictionary */
-	idxdict = repoidx_getdict(pkgdir);
-	if (idxdict == NULL) {
+	/* Get package array in repo index file */
+	pkgar = prop_dictionary_get(idxdict, "packages");
+	if (pkgar == NULL) {
 		rv = errno;
 		goto out1;
 	}
-	plist = xbps_get_pkg_index_plist(pkgdir);
-	if (plist == NULL) {
-		prop_dictionary_remove(idxdict, "packages");
-		rv = ENOMEM;
-		goto out2;
-	}
-
 	/*
 	 * Open the binary package and read the props.plist
 	 * into a buffer.
@@ -156,10 +149,11 @@ xbps_repo_addpkg_index(const char *pkgdir, const char *file)
 			prop_dictionary_get_cstring_nocopy(curpkgd,
 			    "version", &regver);
 			if (xbps_cmpver(version, regver) <= 0) {
-				printf("Skipping %s. Version %s already "
-				    "registered.\n", filen, regver);
+				printf("W: skipping %s. %s-%s already "
+				    "registered.\n", filen, pkgname, regver);
 				prop_object_release(newpkgd);
 				archive_read_data_skip(ar);
+				rv = EEXIST;
 				break;
 			}
 			/*
@@ -199,32 +193,19 @@ xbps_repo_addpkg_index(const char *pkgdir, const char *file)
 		/*
 		 * Add dictionary into the index and update package count.
 		 */
-		pkgar = prop_dictionary_get(idxdict, "packages");
-		if (pkgar == NULL) {
-			prop_object_release(newpkgd);
-			rv = errno;
-			break;
-		}
 		if (!xbps_add_obj_to_array(pkgar, newpkgd)) {
 			prop_object_release(newpkgd);
 			rv = EINVAL;
 			break;
 		}
-
 		prop_dictionary_set_uint64(idxdict, "total-pkgs",
 		    prop_array_count(pkgar));
-		if (!prop_dictionary_externalize_to_file(idxdict, plist)) {
-			rv = errno;
-			break;
-		}
 		printf("Registered %s-%s in package index.\n",
 		    pkgname, version);
+		printf("\033[1A\033[K");
 		break;
 	}
 
-	free(plist);
-out2:
-	prop_object_release(idxdict);
 out1:
 	archive_read_finish(ar);
 out:
@@ -236,16 +217,32 @@ out:
 int
 xbps_repo_genindex(const char *pkgdir)
 {
+	prop_dictionary_t idxdict = NULL;
 	struct dirent *dp;
 	DIR *dirp;
 	struct utsname un;
-	char *binfile, *path;
+	uint64_t npkgcnt = 0;
+	char *binfile, *path, *plist;
 	size_t i;
 	int rv = 0;
-	bool foundpkg = false;
+	bool registered_newpkgs = false, foundpkg = false;
 
 	if (uname(&un) == -1)
 		return errno;
+
+	/*
+	 * Create or read existing package index plist file.
+	 */
+	idxdict = repoidx_getdict(pkgdir);
+	if (idxdict == NULL)
+		return errno;
+
+	plist = xbps_get_pkg_index_plist(pkgdir);
+	if (plist == NULL) {
+		prop_object_release(idxdict);
+		return errno;
+	}
+
 	/*
 	 * Iterate over the known architecture directories to find
 	 * binary packages.
@@ -256,8 +253,10 @@ xbps_repo_genindex(const char *pkgdir)
 			continue;
 
 		path = xbps_xasprintf("%s/%s", pkgdir, archdirs[i]);
-		if (path == NULL)
+		if (path == NULL) {
+			prop_object_release(idxdict);
 			return errno;
+		}
 
 		dirp = opendir(path);
 		if (dirp == NULL) {
@@ -279,22 +278,48 @@ xbps_repo_genindex(const char *pkgdir)
 			if (binfile == NULL) {
 				(void)closedir(dirp);
 				free(path);
+				prop_object_release(idxdict);
+				free(plist);
 				return errno;
 			}
-			rv = xbps_repo_addpkg_index(pkgdir, binfile);
+			rv = xbps_repo_addpkg_index(idxdict, binfile);
 			free(binfile);
-			if (rv != 0) {
+			if (rv == EEXIST)
+				continue;
+			else if (rv != 0) {
 				(void)closedir(dirp);
 				free(path);
+				prop_object_release(idxdict);
+				free(plist);
 				return rv;
 			}
+			registered_newpkgs = true;
 		}
 		(void)closedir(dirp);
 		free(path);
 	}
 
-	if (foundpkg == false)
+	if (foundpkg == false) {
+		/* No packages were found in directory */
 		rv = ENOENT;
+	} else {
+		/*
+		 * Show total count registered packages.
+		 */
+		prop_dictionary_get_uint64(idxdict, "total-pkgs", &npkgcnt);
+		printf("%ju packages registered in package index.\n", npkgcnt);
+		/*
+		 * Don't write plist file if no packages were registered.
+		 */
+		if (registered_newpkgs == false)
+			return rv;
+		/*
+		 * If any package was registered in package index, write
+		 * plist file to storage.
+		 */
+		if (!prop_dictionary_externalize_to_file(idxdict, plist))
+			rv = errno;
+	}
 
 	return rv;
 }

bin/xbps-repo/index.h

@@ -27,6 +27,6 @@
 #define _XBPS_REPO_INDEX_H_
 
 int	xbps_repo_genindex(const char *);
-int	xbps_repo_addpkg_index(const char *, const char *);
+int	xbps_repo_addpkg_index(prop_dictionary_t, const char *);
 
 #endif /* !_XBPS_REPO_INDEX_H_ */

bin/xbps-repo/main.c

@@ -49,10 +49,9 @@ usage(void)
 {
 	printf("Usage: xbps-repo [options] [action] [arguments]\n\n"
 	" Available actions:\n"
-        "    add, add-pkgidx, genindex, list, remove, search, show\n"
+        "    add, genindex, list, remove, search, show\n"
 	" Actions with arguments:\n"
 	"    add\t\t<URI>\n"
-	"    add-pkgidx\t<repo> <binpkg>\n"
 	"    genindex\t<path>\n"
 	"    remove\t<URI>\n"
 	"    search\t<string>\n"
@@ -69,7 +68,6 @@ usage(void)
 	"    $ xbps-repo remove /path/to/directory\n"
 	"    $ xbps-repo search klibc\n"
 	"    $ xbps-repo show klibc\n"
-	"    $ xbps-repo add-pkgidx /pkgdir /pkgdir/noarch/foo.xbps\n"
 	"    $ xbps-repo genindex /pkgdir\n");
 	exit(EXIT_FAILURE);
 }
@@ -147,6 +145,7 @@ add_repository(const char *uri, bool remote)
 		if (!sanitize_localpath(idxstr, uri))
 			return errno;
 
+		printf("Fetching remote package index at %s...\n", uri);
 		rv = xbps_sync_repository_pkg_index(idxstr);
 		if (rv != 0)
 			return rv;
@@ -301,15 +300,6 @@ main(int argc, char **argv)
 		rv = xbps_repo_genindex(argv[1]);
 		exit(rv);
 
-	} else if (strcasecmp(argv[0], "add-pkgidx") == 0) {
-		/* Add a binary package into a package repository. */
-		if (argc != 3)
-			usage();
-
-		rv = xbps_repo_addpkg_index(argv[1], argv[2]);
-		if (rv != 0)
-			exit(rv);
-
 	} else if (strcasecmp(argv[0], "sync") == 0) {
 		/* Syncs the pkg index file from a remote repo */
 		if (argc != 2)

bin/xbps-repo/xbps-repo.8.txt

@@ -43,16 +43,12 @@ Please note that all targets are *case insensitive*.
 	Local (by specifying a *directory*) and remote (by specifiying an
 	*HTTP/HTTPS/FTP URL*) repositories can be specified.
 
-*add-pkgidx* '/path/to/local/repo' '/path/to/local/repo/arch/binpkg'::
-	Adds the binary package 'binpkg' as specified in the second argument
-	into the package index of a local repository as specified in the
-	first argument. Please note that 'binpkg' must also live in the same
-	local repository, you need to specify the full path to the file.
-
 *genindex* '/path/to/local/repo'::
 	Generates a package index for a local repository as specified in
-	its argument. The same argument should be used for the *add*
-	and *remove* targets.
+	its argument. It will look for archives with *.xbps* extension
+	and will only add it into the index if version is newer than the one
+	available in the index. The same argument should be used for the *add*
+	and *remove* targets. 
 
 *list*::
 	Lists all currently registered repositories in repository pool.