Fixed issue 21 "xbps-repo find-files is awfully slow" reported by ojab.

NEWS

@@ -1,5 +1,13 @@
 xbps-0.12.0 (???):
 
+ * xbps-repo(8): the 'genindex' target now creates a plist file to cache
+   all files provided by binary packages in a repository. This makes
+   the 'find-files' target marginally faster, because this avoids having
+   to fetch data from network in remote repositories. The new file
+   'index-files.plist' will be downloaded every time the 'sync' target
+   is issued but only iff the file has been modified.
+   This fixes Issue 21 "`xbps-repo find-files` is awfully slow".
+
  * It's now possible to fetch files thru xbps_fetch_file() from HTTP
    servers that don't set the Content-Length header. This fixes issue 19:
    "xbps-src cannot fetch source with unknown Content-Length" reported

bin/xbps-repo/Makefile

@@ -3,6 +3,7 @@ TOPDIR = ../..
 
 BIN =	xbps-repo
 OBJS =	main.o index.o show.o find-files.o list.o
+OBJS += index-files.o
 OBJS += ../xbps-bin/fetch_cb.o ../xbps-bin/util.o
 OBJS += ../xbps-bin/state_cb.o ../xbps-bin/list.o
 MAN = $(BIN).8

bin/xbps-repo/defs.h

@@ -1,5 +1,5 @@
 /*-
- * Copyright (c) 2009-2011 Juan Romero Pardines.
+ * Copyright (c) 2009-2012 Juan Romero Pardines.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -41,6 +41,9 @@ struct repo_search_data {
 /* From index.c */
 int	repo_genindex(const char *);
 
+/* From index-files.c */
+int	repo_genindex_files(const char *);
+
 /* From find-files.c */
 int	repo_find_files_in_packages(int, char **);
 

bin/xbps-repo/find-files.c

@@ -1,5 +1,5 @@
 /*-
- * Copyright (c) 2010 Juan Romero Pardines.
+ * Copyright (c) 2010-2012 Juan Romero Pardines.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -37,27 +37,27 @@ struct ffdata {
 	char **patterns;
 };
 
-static int
+static void
-match_files_by_pattern(prop_dictionary_t pkg_filesd,
+match_files_by_pattern(prop_dictionary_t pkg_filesd, struct ffdata *ffd)
-		       prop_dictionary_keysym_t key,
-		       struct ffdata *ffd,
-		       const char *pkgver)
 {
 	prop_object_iterator_t iter;
-	prop_array_t array;
+	prop_array_t array, allkeys;
 	prop_object_t obj;
-	const char *keyname, *filestr, *typestr;
+	prop_dictionary_keysym_t key;
-	int i;
+	const char *keyname, *filestr, *typestr, *pkgver;
+	size_t i;
+	int x;
 
+	allkeys = prop_dictionary_all_keys(pkg_filesd);
+	for (i = 0; i < prop_array_count(allkeys); i++) {
+		key = prop_array_get(allkeys, i);
 		keyname = prop_dictionary_keysym_cstring_nocopy(key);
 		array = prop_dictionary_get_keysym(pkg_filesd, key);
 		if (prop_object_type(array) != PROP_TYPE_ARRAY)
-		return 0;
+			break;
 
 		if (strcmp(keyname, "files") == 0)
 			typestr = "regular file";
-	else if (strcmp(keyname, "dirs") == 0)
-		typestr = "directory";
 		else if (strcmp(keyname, "links") == 0)
 			typestr = "link";
 		else
@@ -66,71 +66,52 @@ match_files_by_pattern(prop_dictionary_t pkg_filesd,
 		iter = prop_array_iterator(array);
 		while ((obj = prop_object_iterator_next(iter))) {
 			prop_dictionary_get_cstring_nocopy(obj, "file", &filestr);
-		for (i = 1; i < ffd->npatterns; i++) {
+			for (x = 1; x < ffd->npatterns; x++) {
-			if ((strcmp(filestr, ffd->patterns[i]) == 0) ||
+				if ((strcmp(filestr, ffd->patterns[x]) == 0) ||
-			    (strstr(filestr, ffd->patterns[i])) ||
+				    (strstr(filestr, ffd->patterns[x])) ||
 				     (xbps_pkgpattern_match(filestr,
-						   ffd->patterns[i]) == 1))
+				      ffd->patterns[x]) == 1)) {
-				printf(" %s: %s (%s)\n", pkgver, filestr, typestr);
+					prop_dictionary_get_cstring_nocopy(
+					    pkg_filesd, "pkgver", &pkgver);
+					printf(" %s: %s (%s)\n",
+					    pkgver, filestr, typestr);
+				}
 			}
 		}
 		prop_object_iterator_release(iter);
-	return 0;
+	}
+	prop_object_release(allkeys);
 }
 
 static int
 find_files_in_package(struct repository_pool_index *rpi, void *arg, bool *done)
 {
-	prop_dictionary_t pkg_filesd;
+	prop_dictionary_t idxfilesd;
 	prop_array_t files_keys;
-	prop_object_t obj;
-	prop_object_iterator_t iter;
 	struct ffdata *ffd = arg;
-	const char *pkgname, *pkgver;
+	char *plist;
-	char *url;
+	unsigned int i;
-	int rv = 0;
-	unsigned int i, count;
 
 	(void)done;
 
-	iter = xbps_array_iter_from_dict(rpi->rpi_repod, "packages");
-	if (iter == NULL)
-		return -1;
-
 	printf("Looking in repository '%s', please wait...\n", rpi->rpi_uri);
-	while ((obj = prop_object_iterator_next(iter))) {
+	plist = xbps_pkg_index_files_plist(rpi->rpi_uri);
-		url = xbps_path_from_repository_uri(obj, rpi->rpi_uri);
+	if (plist == NULL)
-		if (url == NULL) {
+		return ENOMEM;
-			rv = -1;
+
-			break;
+	idxfilesd = prop_dictionary_internalize_from_zfile(plist);
+	if (idxfilesd == NULL) {
+		free(plist);
+		return errno;
 	}
-		prop_dictionary_get_cstring_nocopy(obj, "pkgname", &pkgname);
+	free(plist);
-		prop_dictionary_get_cstring_nocopy(obj, "pkgver", &pkgver);
+
-		pkg_filesd = xbps_dictionary_metadata_plist_by_url(url,
+	files_keys = prop_dictionary_get(idxfilesd, "packages");
-		    XBPS_PKGFILES);
+	for (i = 0; i < prop_array_count(files_keys); i++)
-		free(url);
+		match_files_by_pattern(prop_array_get(files_keys, i), ffd);
-		if (pkg_filesd == NULL) {
+
-			xbps_error_printf("xbps-repo: couldn't read '%s' "
+	prop_object_release(idxfilesd);
-			    "from '%s (%s)': %s\n", XBPS_PKGFILES, pkgname,
+	return 0;
-			    rpi->rpi_uri, strerror(errno));
-			rv = -1;
-			break;
-		}
-		files_keys = prop_dictionary_all_keys(pkg_filesd);
-		count = prop_array_count(files_keys);
-		for (i = 0; i < count; i++) {
-			rv = match_files_by_pattern(pkg_filesd,
-			    prop_array_get(files_keys, i), ffd, pkgver);
-			if (rv == -1)
-				break;
-		}
-		prop_object_release(files_keys);
-		prop_object_release(pkg_filesd);
-		if (rv == -1)
-			break;
-	}
-	prop_object_iterator_release(iter);
-	return rv;
 }
 
 int

bin/xbps-repo/index-files.c

@@ -0,0 +1,196 @@
+/*-
+ * Copyright (c) 2012 Juan Romero Pardines.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <stdbool.h>
+#include <errno.h>
+
+#include <xbps_api.h>
+#include "defs.h"
+
+struct index_files_data {
+	prop_array_t idxfiles;
+	const char *pkgdir;
+};
+
+static int
+genindex_files_cb(prop_object_t obj, void *arg, bool *done)
+{
+	prop_dictionary_t pkg_filesd, pkgd;
+	prop_array_t array;
+	struct index_files_data *ifd = arg;
+	const char *binpkg, *pkgver;
+	char *file;
+	bool found = false;
+
+	(void)done;
+
+	prop_dictionary_get_cstring_nocopy(obj, "filename", &binpkg);
+	prop_dictionary_get_cstring_nocopy(obj, "pkgver", &pkgver);
+
+	file = xbps_xasprintf("%s/%s", ifd->pkgdir, binpkg);
+	if (file == NULL)
+		return ENOMEM;
+
+	/* internalize files.plist from binary package archive */
+	pkg_filesd = xbps_dictionary_metadata_plist_by_url(file, XBPS_PKGFILES);
+	if (pkg_filesd == NULL) {
+		free(file);
+		return EINVAL;
+	}
+	free(file);
+
+	/* create pkg dictionary */
+	pkgd = prop_dictionary_create();
+	if (pkgd == NULL) {
+		prop_object_release(pkg_filesd);
+		return ENOMEM;
+	}
+	/* add conf_files array in pkgd */
+	array = prop_dictionary_get(pkg_filesd, "conf_files");
+	if (array != NULL && prop_array_count(array)) {
+		found = true;
+		if (!prop_dictionary_set(pkgd, "conf_files", array)) {
+			prop_object_release(pkgd);
+			prop_object_release(pkg_filesd);
+			return EINVAL;
+		}
+	}
+	/* add files array in pkgd */
+	array = prop_dictionary_get(pkg_filesd, "files");
+	if (array != NULL && prop_array_count(array)) {
+		found = true;
+		if (!prop_dictionary_set(pkgd, "files", array)) {
+			prop_object_release(pkgd);
+			prop_object_release(pkg_filesd);
+			return EINVAL;
+		}
+	}
+	/* add links array in pkgd */
+	array = prop_dictionary_get(pkg_filesd, "links");
+	if (array != NULL && prop_array_count(array)) {
+		found = true;
+		if (!prop_dictionary_set(pkgd, "links", array)) {
+			prop_object_release(pkgd);
+			prop_object_release(pkg_filesd);
+			return EINVAL;
+		}
+	}
+	prop_object_release(pkg_filesd);
+	if (!found) {
+		prop_object_release(pkgd);
+		return 0;
+	}
+	/* pkgver obj in pkgd */
+	if (!prop_dictionary_set_cstring(pkgd, "pkgver", pkgver)) {
+		prop_object_release(pkgd);
+		return EINVAL;
+	}
+
+	/* add pkgd into provided array */
+	if (!prop_array_add(ifd->idxfiles, pkgd)) {
+		prop_object_release(pkgd);
+		return EINVAL;
+	}
+	prop_object_release(pkgd);
+
+	return 0;
+}
+
+/*
+ * Create the index files cache for all packages in repository.
+ */
+int
+repo_genindex_files(const char *pkgdir)
+{
+	prop_dictionary_t idxdict, idxfilesd;
+	struct index_files_data *ifd;
+	char *plist, *files_plist;
+	int rv;
+
+	plist = xbps_pkg_index_plist(pkgdir);
+	if (plist == NULL)
+		return ENOMEM;
+
+	/* internalize repository index plist */
+	idxdict = prop_dictionary_internalize_from_zfile(plist);
+	if (idxdict == NULL) {
+		free(plist);
+		return errno;
+	}
+
+	ifd = malloc(sizeof(*ifd));
+	if (ifd == NULL) {
+		prop_object_release(idxdict);
+		free(plist);
+		return ENOMEM;
+	}
+	ifd->pkgdir = pkgdir;
+	ifd->idxfiles = prop_array_create();
+
+	printf("Creating repository's index files cache...\n");
+
+	/* iterate over index.plist packages array */
+	rv = xbps_callback_array_iter_in_dict(idxdict,
+	    "packages", genindex_files_cb, ifd);
+	prop_object_release(idxdict);
+	free(plist);
+	if (rv != 0) {
+		prop_object_release(ifd->idxfiles);
+		free(ifd);
+		return rv;
+	}
+	idxfilesd = prop_dictionary_create();
+	/* add array into the index-files dictionary */
+	if (!prop_dictionary_set(idxfilesd, "packages", ifd->idxfiles)) {
+		prop_object_release(ifd->idxfiles);
+		prop_object_release(idxfilesd);
+		free(ifd);
+		return EINVAL;
+	}
+	files_plist = xbps_pkg_index_files_plist(pkgdir);
+	if (files_plist == NULL) {
+		prop_object_release(ifd->idxfiles);
+		prop_object_release(idxfilesd);
+		free(ifd);
+		return ENOMEM;
+	}
+	/* externalize index-files dictionary to the plist file */
+	if (!prop_dictionary_externalize_to_zfile(idxfilesd, files_plist)) {
+		free(files_plist);
+		prop_object_release(ifd->idxfiles);
+		prop_object_release(idxfilesd);
+		free(ifd);
+		return errno;
+	}
+	free(files_plist);
+	prop_object_release(idxfilesd);
+	prop_object_release(ifd->idxfiles);
+	free(ifd);
+
+	return 0;
+}

bin/xbps-repo/main.c

@@ -1,5 +1,5 @@
 /*-
- * Copyright (c) 2008-2011 Juan Romero Pardines.
+ * Copyright (c) 2008-2012 Juan Romero Pardines.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -239,6 +239,9 @@ main(int argc, char **argv)
 			usage(xhp);
 
 		rv = repo_genindex(argv[1]);
+		if (rv == 0)
+			rv = repo_genindex_files(argv[1]);
+
 	} else if (strcasecmp(argv[0], "sync") == 0) {
 		/* Syncs the pkg index for all registered remote repos */
 		if (argc != 1)

include/xbps_api.h

@@ -56,7 +56,7 @@
  */
 #define XBPS_PKGINDEX_VERSION	"1.3"
 
-#define XBPS_API_VERSION	"20120105"
+#define XBPS_API_VERSION	"20120115"
 #define XBPS_VERSION		"0.12"
 
 /**
@@ -103,6 +103,12 @@
  */
 #define XBPS_PKGINDEX		"index.plist"
 
+/**
+ * @def XBPS_PKGINDEX_FILES
+ * Filename for the repository package index files property list.
+ */
+#define XBPS_PKGINDEX_FILES	"index-files.plist"
+
 /**
  * @def XBPS_SYSCONF_PATH
  * Default configuration PATH to find XBPS_CONF_PLIST.
@@ -1450,12 +1456,13 @@ prop_dictionary_t
  * by the \a uri argument (if necessary).
  *
  * @param[in] uri URI to a remote repository.
+ * @param[in] plistf Plist file to sync.
  *
  * @return -1 on error (errno is set appropiately), 0 if transfer was
  * not necessary (local/remote size/mtime matched) or 1 if
  * downloaded successfully.
  */
-int xbps_repository_sync_pkg_index(const char *uri);
+int xbps_repository_sync_pkg_index(const char *uri, const char *plistf);
 
 /*@}*/
 
@@ -1692,6 +1699,18 @@ char *xbps_path_from_repository_uri(prop_dictionary_t pkgd, const char *repoloc)
  */
 char *xbps_pkg_index_plist(const char *uri);
 
+/**
+ * Returns the full path to a repository package index files plist file,
+ * as specified by \a uri.
+ *
+ * @param[in] uri Repository URI.
+ *
+ * @return A pointer to a malloc(3)ed string, NULL otherwise and
+ * errno is set appropiately. The pointer should be free(3)d when it's
+ * no longer needded.
+ */
+char *xbps_pkg_index_files_plist(const char *uri);
+
 /**
  * Gets the name of a package string. Package strings are composed
  * by a @<pkgname@>/@<version@> pair and separated by the <em>minus</em>

lib/repository_pool.c

@@ -196,9 +196,20 @@ xbps_repository_pool_sync(const struct xbps_handle *xhp)
 			continue;
 		}
 		/*
-		 * Fetch repository index file.
+		 * Fetch repository index.plist.
 		 */
-		rv = xbps_repository_sync_pkg_index(repouri);
+		rv = xbps_repository_sync_pkg_index(repouri, XBPS_PKGINDEX);
+		if (rv == -1) {
+			xbps_dbg_printf("[rpool] `%s' failed to fetch: %s\n",
+			    repouri, fetchLastErrCode == 0 ?
+			    strerror(errno) : xbps_fetch_error_string());
+			continue;
+		}
+		/*
+		 * Fetch repository index-files.plist.
+		 */
+		rv = xbps_repository_sync_pkg_index(repouri,
+		    XBPS_PKGINDEX_FILES);
 		if (rv == -1) {
 			xbps_dbg_printf("[rpool] `%s' failed to fetch: %s\n",
 			    repouri, fetchLastErrCode == 0 ?

lib/repository_sync_index.c

@@ -86,7 +86,7 @@ xbps_get_remote_repo_string(const char *uri)
  * size and/or mtime match) and 1 if downloaded successfully.
  */
 int
-xbps_repository_sync_pkg_index(const char *uri)
+xbps_repository_sync_pkg_index(const char *uri, const char *plistf)
 {
 	prop_dictionary_t tmpd;
 	struct xbps_handle *xhp;
@@ -132,7 +132,7 @@ xbps_repository_sync_pkg_index(const char *uri)
 	/*
 	 * Remote repository index.plist full URL.
 	 */
-	rpidx = xbps_xasprintf("%s/%s", uri, XBPS_PKGINDEX);
+	rpidx = xbps_xasprintf("%s/%s", uri, plistf);
 	if (rpidx == NULL) {
 		rv = -1;
 		goto out;
@@ -141,7 +141,7 @@ xbps_repository_sync_pkg_index(const char *uri)
 	 * Save temporary file in XBPS_META_PATH, and rename if it
 	 * was downloaded successfully.
 	 */
-	tmp_metafile = xbps_xasprintf("%s/%s", metadir, XBPS_PKGINDEX);
+	tmp_metafile = xbps_xasprintf("%s/%s", metadir, plistf);
 	if (tmp_metafile == NULL) {
 		rv = -1;
 		goto out;
@@ -200,7 +200,7 @@ xbps_repository_sync_pkg_index(const char *uri)
 	}
 	prop_object_release(tmpd);
 
-	lrepofile = xbps_xasprintf("%s/%s", lrepodir, XBPS_PKGINDEX);
+	lrepofile = xbps_xasprintf("%s/%s", lrepodir, plistf);
 	if (lrepofile == NULL) {
 		rv = -1;
 		goto out;

lib/util.c

@@ -1,5 +1,5 @@
 /*-
- * Copyright (c) 2008-2011 Juan Romero Pardines.
+ * Copyright (c) 2008-2012 Juan Romero Pardines.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -201,7 +201,7 @@ xbps_pkgpattern_version(const char *pkg)
 }
 
 static char *
-get_pkg_index_remote_plist(const char *uri)
+get_pkg_index_remote_plist(const char *uri, const char *plistf)
 {
 	struct xbps_handle *xhp;
 	char *uri_fixed, *repodir;
@@ -215,11 +215,11 @@ get_pkg_index_remote_plist(const char *uri)
 
 	if (strcmp(xhp->rootdir, "/") == 0) {
 		repodir = xbps_xasprintf("/%s/%s/%s",
-		    XBPS_META_PATH, uri_fixed, XBPS_PKGINDEX);
+		    XBPS_META_PATH, uri_fixed, plistf);
 	} else {
 		repodir = xbps_xasprintf("%s/%s/%s/%s",
 		    xhp->rootdir,
-		    XBPS_META_PATH, uri_fixed, XBPS_PKGINDEX);
+		    XBPS_META_PATH, uri_fixed, plistf);
 	}
 	free(uri_fixed);
 	return repodir;
@@ -231,11 +231,21 @@ xbps_pkg_index_plist(const char *uri)
 	assert(uri != NULL);
 
 	if (xbps_check_is_repository_uri_remote(uri))
-		return get_pkg_index_remote_plist(uri);
+		return get_pkg_index_remote_plist(uri, XBPS_PKGINDEX);
 
 	return xbps_xasprintf("%s/%s", uri, XBPS_PKGINDEX);
 }
 
+char *
+xbps_pkg_index_files_plist(const char *uri)
+{
+	assert(uri != NULL);
+	if (xbps_check_is_repository_uri_remote(uri))
+		return get_pkg_index_remote_plist(uri, XBPS_PKGINDEX_FILES);
+
+	return xbps_xasprintf("%s/%s", uri, XBPS_PKGINDEX_FILES);
+}
+
 char *
 xbps_path_from_repository_uri(prop_dictionary_t pkg_repod, const char *repoloc)
 {