Get rid of repodata index-files; the archive is now 8x smaller.

See the NEWS file for more information.

bin/xbps-rindex/defs.h

@@ -81,6 +81,6 @@ int	sign_repo(struct xbps_handle *, const char *, const char *,
 
 /* From repoflush.c */
 bool	repodata_flush(struct xbps_handle *, const char *,
-		xbps_dictionary_t, xbps_dictionary_t, xbps_dictionary_t);
+		xbps_dictionary_t, xbps_dictionary_t);
 
 #endif /* !_XBPS_RINDEX_DEFS_H_ */

bin/xbps-rindex/index-add.c

@@ -40,14 +40,12 @@
 int
 index_add(struct xbps_handle *xhp, int args, int argmax, char **argv, bool force)
 {
-	xbps_array_t array, pkg_files, pkg_links, pkg_cffiles;
-	xbps_dictionary_t idx, idxmeta, idxfiles, binpkgd, pkg_filesd, curpkgd;
-	xbps_object_t obj, fileobj;
+	xbps_dictionary_t idx, idxmeta, binpkgd, curpkgd;
 	struct xbps_repo *repo = NULL;
 	struct stat st;
 	char *tmprepodir = NULL, *repodir = NULL;
 	int rv = 0, ret = 0;
-	bool flush = false, found = false;
+	bool flush = false;
 
 	assert(argv);
 	/*
@@ -64,15 +62,12 @@ index_add(struct xbps_handle *xhp, int args, int argmax, char **argv, bool force
 		rv = -1;
 		goto out;
 	}
-	if (repo && repo->idx) {
-		xbps_repo_open_idxfiles(repo);
+	if (repo) {
 		idx = xbps_dictionary_copy(repo->idx);
 		idxmeta = xbps_dictionary_copy(repo->idxmeta);
-		idxfiles = xbps_dictionary_copy(repo->idxfiles);
 	} else {
 		idx = xbps_dictionary_create();
 		idxmeta = NULL;
-		idxfiles = xbps_dictionary_create();
 	}
 	/*
 	 * Process all packages specified in argv.
@@ -153,7 +148,6 @@ index_add(struct xbps_handle *xhp, int args, int argmax, char **argv, bool force
 			 */
 			printf("index: removed obsolete entry `%s' (%s).\n", opkgver, oarch);
 			xbps_dictionary_remove(idx, pkgname);
-			xbps_dictionary_remove(idxfiles, opkgver);
 			free(opkgver);
 			free(oarch);
 		}
@@ -205,87 +199,19 @@ index_add(struct xbps_handle *xhp, int args, int argmax, char **argv, bool force
 		printf("index: added `%s' (%s).\n", pkgver, arch);
 		xbps_object_release(binpkgd);
 		free(pkgname);
-		/*
-		 * Add new pkg dictionary into the index-files.
-		 */
-		found = false;
-		pkg_filesd = xbps_get_pkg_plist_from_binpkg(pkg, "./files.plist");
-		if (pkg_filesd == NULL) {
-			free(pkgver);
-			rv = EINVAL;
-			goto out;
-		}
-
-		pkg_cffiles = xbps_dictionary_get(pkg_filesd, "conf_files");
-		if (xbps_array_count(pkg_cffiles))
-			found = true;
-		else
-			pkg_cffiles = NULL;
-
-		pkg_files = xbps_dictionary_get(pkg_filesd, "files");
-		if (xbps_array_count(pkg_files))
-			found = true;
-		else
-			pkg_files = NULL;
-
-		pkg_links = xbps_dictionary_get(pkg_filesd, "links");
-		if (xbps_array_count(pkg_links))
-			found = true;
-		else
-			pkg_links = NULL;
-
-		/* If pkg does not contain any file, ignore it */
-		if (!found) {
-			xbps_object_release(pkg_filesd);
-			free(pkgver);
-			continue;
-		}
-		/* create pkg files array */
-		array = xbps_array_create();
-		assert(array);
-
-		/* add conf_files in pkg files array */
-		if (pkg_cffiles != NULL) {
-			for (unsigned int x = 0; x < xbps_array_count(pkg_cffiles); x++) {
-				obj = xbps_array_get(pkg_cffiles, x);
-				fileobj = xbps_dictionary_get(obj, "file");
-				xbps_array_add(array, fileobj);
-			}
-		}
-		/* add files array in pkg files array */
-		if (pkg_files != NULL) {
-			for (unsigned int x = 0; x < xbps_array_count(pkg_files); x++) {
-				obj = xbps_array_get(pkg_files, x);
-				fileobj = xbps_dictionary_get(obj, "file");
-				xbps_array_add(array, fileobj);
-			}
-		}
-		/* add links array in pkg files array */
-		if (pkg_links != NULL) {
-			for (unsigned int x = 0; x < xbps_array_count(pkg_links); x++) {
-				obj = xbps_array_get(pkg_links, x);
-				fileobj = xbps_dictionary_get(obj, "file");
-				xbps_array_add(array, fileobj);
-			}
-		}
-		/* add pkg files array into index-files */
-		xbps_dictionary_set(idxfiles, pkgver, array);
-		xbps_object_release(array);
-		xbps_object_release(pkg_filesd);
 		free(pkgver);
 	}
 	/*
 	 * Generate repository data files.
 	 */
 	if (flush) {
-		if (!repodata_flush(xhp, repodir, idx, idxfiles, idxmeta)) {
+		if (!repodata_flush(xhp, repodir, idx, idxmeta)) {
 			fprintf(stderr, "%s: failed to write repodata: %s\n",
 			    _XBPS_RINDEX, strerror(errno));
 			goto out;
 		}
 	}
 	printf("index: %u packages registered.\n", xbps_dictionary_count(idx));
-	printf("index-files: %u packages registered.\n", xbps_dictionary_count(idxfiles));
 
 out:
 	if (repo)

bin/xbps-rindex/index-clean.c

@@ -85,32 +85,6 @@ idx_cleaner_cb(struct xbps_handle *xhp,
 	return 0;
 }
 
-static int
-idxfiles_cleaner_cb(struct xbps_handle *xhp _unused, xbps_object_t obj _unused,
-		const char *key, void *arg, bool *done _unused)
-{
-	xbps_dictionary_t pkg;
-	struct cbdata *cbd = arg;
-	char *pkgname;
-	const char *pkgver;
-
-	/* Find out entries on index-files that aren't registered on index */
-	if ((pkgname = xbps_pkg_name(key)) == NULL) {
-		xbps_dbg_printf(xhp, "%s: invalid entry found on index-files: %s\n", cbd->repourl, key);
-		return 0;
-	}
-	if ((pkg = xbps_dictionary_get(cbd->idx, pkgname))) {
-		xbps_dictionary_get_cstring_nocopy(pkg, "pkgver", &pkgver);
-		if (strcmp(pkgver, key)) {
-			pthread_mutex_lock(&cbd->mtx);
-			xbps_array_add_cstring_nocopy(cbd->result, key);
-			pthread_mutex_unlock(&cbd->mtx);
-		}
-	}
-	free(pkgname);
-
-	return 0;
-}
 /*
  * Removes stalled pkg entries in repository's XBPS_REPOIDX file, if any
  * binary package cannot be read (unavailable, not enough perms, etc).
@@ -119,7 +93,7 @@ int
 index_clean(struct xbps_handle *xhp, const char *repodir)
 {
 	xbps_array_t allkeys;
-	xbps_dictionary_t idx = NULL, idxmeta = NULL, idxfiles = NULL;
+	xbps_dictionary_t idx = NULL, idxmeta = NULL;
 	struct xbps_repo *repo;
 	struct cbdata cbd;
 	int rv = 0;
@@ -135,11 +109,9 @@ index_clean(struct xbps_handle *xhp, const char *repodir)
 		    _XBPS_RINDEX, strerror(errno));
 		return rv;
 	}
-	xbps_repo_open_idxfiles(repo);
 	idx = xbps_dictionary_copy(repo->idx);
 	idxmeta = xbps_dictionary_copy(repo->idxmeta);
-	idxfiles = xbps_dictionary_copy(repo->idxfiles);
-	if (idx == NULL || idxfiles == NULL) {
+	if (idx == NULL) {
 		fprintf(stderr, "%s: incomplete repository data file!\n", _XBPS_RINDEX);
 		rv = EINVAL;
 		goto out;
@@ -159,41 +131,20 @@ index_clean(struct xbps_handle *xhp, const char *repodir)
 		char *keyname = NULL, *pkgname = NULL;
 
 		xbps_array_get_cstring(cbd.result, x, &keyname);
-		printf("index-files: removed entry %s\n", keyname);
 		printf("index: removed entry %s\n", keyname);
 		pkgname = xbps_pkg_name(keyname);
 		assert(pkgname);
-		xbps_dictionary_remove(idxfiles, keyname);
 		xbps_dictionary_remove(idx, pkgname);
 		free(pkgname);
 		free(keyname);
 		flush = true;
 	}
-	/*
-	 * Second pass: find out obsolete entries on index-files.
-	 */
-	xbps_object_release(cbd.result);
-	xbps_object_release(allkeys);
-	cbd.idx = idx;
-	cbd.result = xbps_array_create();
-	allkeys = xbps_dictionary_all_keys(idxfiles);
-	(void)xbps_array_foreach_cb_multi(xhp, allkeys, idxfiles, idxfiles_cleaner_cb, &cbd);
-	for (unsigned int x = 0; x < xbps_array_count(cbd.result); x++) {
-		char *keyname = NULL;
-
-		xbps_array_get_cstring(cbd.result, x, &keyname);
-		printf("index-files: removed entry %s\n", keyname);
-		xbps_dictionary_remove(idxfiles, keyname);
-		free(keyname);
-		flush = true;
-	}
-
 	pthread_mutex_destroy(&cbd.mtx);
 	xbps_object_release(cbd.result);
 	xbps_object_release(allkeys);
 
 	if (flush) {
-		if (!repodata_flush(xhp, repodir, idx, idxfiles, idxmeta)) {
+		if (!repodata_flush(xhp, repodir, idx, idxmeta)) {
 			rv = errno;
 			fprintf(stderr, "failed to write repodata: %s\n",
 			    strerror(errno));
@@ -202,8 +153,6 @@ index_clean(struct xbps_handle *xhp, const char *repodir)
 	}
 	printf("index: %u packages registered.\n",
 			xbps_dictionary_count(idx));
-	printf("index-files: %u packages registered.\n",
-			xbps_dictionary_count(idxfiles));
 
 out:
 	xbps_repo_close(repo, true);
@@ -211,8 +160,6 @@ out:
 		xbps_object_release(idx);
 	if (idxmeta)
 		xbps_object_release(idxmeta);
-	if (idxfiles)
-		xbps_object_release(idxfiles);
 
 	return rv;
 }

bin/xbps-rindex/repoflush.c

@@ -1,5 +1,5 @@
 /*-
- * Copyright (c) 2013 Juan Romero Pardines.
+ * Copyright (c) 2013-2014 Juan Romero Pardines.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -39,8 +39,7 @@
 
 bool
 repodata_flush(struct xbps_handle *xhp, const char *repodir,
-	xbps_dictionary_t idx, xbps_dictionary_t idxfiles,
-	xbps_dictionary_t meta)
+	xbps_dictionary_t idx, xbps_dictionary_t meta)
 {
 	struct archive *ar;
 	char *repofile, *tname, *buf;
@@ -82,15 +81,6 @@ repodata_flush(struct xbps_handle *xhp, const char *repodir,
 	if (rv != 0)
 		return false;
 
-	/* XBPS_REPOIDX_FILES */
-	buf = xbps_dictionary_externalize(idxfiles);
-	assert(buf);
-	rv = xbps_archive_append_buf(ar, buf, strlen(buf),
-	    XBPS_REPOIDX_FILES, 0644, "root", "root");
-	free(buf);
-	if (rv != 0)
-		return false;
-
 	/* Write data to tempfile and rename */
 	archive_write_finish(ar);
 	fdatasync(repofd);

bin/xbps-rindex/sign.c

@@ -153,7 +153,6 @@ sign_repo(struct xbps_handle *xhp, const char *repodir,
 		rv = EINVAL;
 		goto out;
 	}
-	xbps_repo_open_idxfiles(repo);
 	/*
 	 * If privkey not set, default to ~/.ssh/id_rsa.
 	 */
@@ -286,7 +285,7 @@ sign_repo(struct xbps_handle *xhp, const char *repodir,
 	xbps_object_release(data);
 	data = NULL;
 
-	if (!repodata_flush(xhp, repodir, repo->idx, repo->idxfiles, meta)) {
+	if (!repodata_flush(xhp, repodir, repo->idx, meta)) {
 		fprintf(stderr, "failed to write repodata: %s\n", strerror(errno));
 		goto out;
 	}