Welcome pkgdb-0.38.

NEWS

@@ -1,5 +1,17 @@
 xbps-0.38 (???):
 
+ * pkgdb-0.38 format: the pkgdb plist file now contains all pkg metadata objects
+   plus the pkgdb specific ones (automatic-install, install-date, metafile-sha256)
+   with the pkg files list being the exception. The pkgdb code will automatically
+   convert the pkgdb files to the new format the first time it's accesed.
+
+   This effectively implements a two phase upgrade, the first time to upgrade the
+   pkgdb format and the second one to execute the real operation.
+
+   The pkg metadata files stored in metadir (/var/db/xbps) now just contains its
+   pkg files list objects, everything else is stored in pkgdb. This allows libxbps
+   to speed up the process of checking for unmet shared libraries, among other things.
+
  * libxbps: added support to check pkg shlib-{provides,requires} objects for
    all pkgs in transaction when installing or updating packages. This will help
    to avoid broken pkgs due to unmatched shlibs.

bin/xbps-dgraph/main.c

@@ -508,7 +508,7 @@ main(int argc, char **argv)
 	/*
 	 * Internalize the plist file of the target installed package.
 	 */
-	plistd = xbps_pkgdb_get_pkg_metadata(&xh, argv[0]);
+	plistd = xbps_pkgdb_get_pkg(&xh, argv[0]);
 	if (plistd == NULL)
 		die("cannot internalize %s metadata file", argv[0]);
 

bin/xbps-install/state_cb.c

@@ -59,6 +59,9 @@ state_cb(const struct xbps_state_cb_data *xscd, void *cbdata _unused)
 	case XBPS_STATE_TRANS_CONFIGURE:
 		printf("\n[*] Configuring unpacked packages\n");
 		break;
+	case XBPS_STATE_PKGDB:
+		printf("[*] pkgdb upgrade in progress, please wait...\n");
+		break;
 	case XBPS_STATE_REPOSYNC:
 		printf("[*] Updating `%s' ...\n", xscd->arg);
 		break;
@@ -128,6 +131,10 @@ state_cb(const struct xbps_state_cb_data *xscd, void *cbdata _unused)
 			    xscd->xhp->rootdir);
 		}
 		break;
+	case XBPS_STATE_PKGDB_DONE:
+		printf("The pkgdb file has been upgraded successfully, please reexec "
+		    "the command again.\n");
+		break;
 	case XBPS_STATE_REPO_KEY_IMPORT:
 		printf("%s\n", xscd->desc);
 		printf("Fingerprint: %s\n", xscd->arg);

bin/xbps-install/transaction.c

@@ -281,9 +281,17 @@ exec_transaction(struct xbps_handle *xhp, int maxcols, bool yes, bool drun)
 	if ((rv = xbps_transaction_prepare(xhp)) != 0) {
 		if (rv == ENODEV) {
 			array = xbps_dictionary_get(xhp->transd, "missing_deps");
-			/* missing packages */
+			if (xbps_array_count(array)) {
+				/* missing dependencies */
 				print_array(array);
-			fprintf(stderr, "Transaction aborted due to missing packages.\n");
+				fprintf(stderr, "Transaction aborted due to unresolved dependencies.\n");
+			}
+			array = xbps_dictionary_get(xhp->transd, "missing_shlibs");
+			if (xbps_array_count(array)) {
+				/* missing shlibs */
+				print_array(array);
+				fprintf(stderr, "Transaction aborted due to unresolved shlibs.\n");
+			}
 			goto out;
 		} else if (rv == EAGAIN) {
 			/* conflicts */

bin/xbps-pkgdb/Makefile

@@ -5,6 +5,5 @@ BIN =	xbps-pkgdb
 OBJS =	main.o check.o check_pkg_files.o
 OBJS +=	check_pkg_rundeps.o
 OBJS +=	check_pkg_symlinks.o check_pkg_unneeded.o
-OBJS +=	convert.o
 
 include $(TOPDIR)/mk/prog.mk

bin/xbps-pkgdb/check.c

@@ -72,12 +72,12 @@ check_pkg_integrity(struct xbps_handle *xhp,
 		    xbps_dictionary_t pkgd,
 		    const char *pkgname)
 {
-	xbps_dictionary_t opkgd, propsd;
+	xbps_dictionary_t opkgd, filesd;
 	const char *sha256;
 	char *buf;
 	int rv = 0;
 
-	propsd = opkgd = NULL;
+	filesd = opkgd = NULL;
 
 	/* find real pkg by name */
 	opkgd = pkgd;
@@ -89,17 +89,17 @@ check_pkg_integrity(struct xbps_handle *xhp,
 		}
 	}
 	/*
-	 * Check for props.plist metadata file.
+	 * Check for pkg files metadata file.
 	 */
-	buf = xbps_xasprintf("%s/.%s.plist",  xhp->metadir, pkgname);
+	buf = xbps_xasprintf("%s/.%s-files.plist",  xhp->metadir, pkgname);
-	propsd = xbps_dictionary_internalize_from_file(buf);
+	filesd = xbps_dictionary_internalize_from_file(buf);
 	free(buf);
-	if (propsd == NULL) {
+	if (filesd == NULL) {
-		xbps_error_printf("%s: unexistent metafile!\n", pkgname);
+		xbps_error_printf("%s: unexistent files metafile!\n", pkgname);
 		return EINVAL;
-	} else if (xbps_dictionary_count(propsd) == 0) {
+	} else if (xbps_dictionary_count(filesd) == 0) {
 		xbps_error_printf("%s: incomplete metadata file.\n", pkgname);
-		xbps_object_release(propsd);
+		xbps_object_release(filesd);
 		return 1;
 	}
 	/*
@@ -107,12 +107,12 @@ check_pkg_integrity(struct xbps_handle *xhp,
 	 */
 	xbps_dictionary_get_cstring_nocopy(opkgd, "metafile-sha256", &sha256);
 	if (sha256 != NULL) {
-		buf = xbps_xasprintf("%s/.%s.plist",
+		buf = xbps_xasprintf("%s/.%s-files.plist",
 		    xhp->metadir, pkgname);
 		rv = xbps_file_hash_check(buf, sha256);
 		free(buf);
 		if (rv == ERANGE) {
-			xbps_object_release(propsd);
+			xbps_object_release(filesd);
 			fprintf(stderr, "%s: metadata file has been "
 			    "modified!\n", pkgname);
 			return 1;
@@ -130,12 +130,12 @@ do {								\
 } while (0)
 
 	/* Execute pkg checks */
-	RUN_PKG_CHECK(xhp, files, propsd);
+	RUN_PKG_CHECK(xhp, files, filesd);
-	RUN_PKG_CHECK(xhp, symlinks, propsd);
+	RUN_PKG_CHECK(xhp, symlinks, filesd);
-	RUN_PKG_CHECK(xhp, rundeps, propsd);
+	RUN_PKG_CHECK(xhp, rundeps, opkgd);
 	RUN_PKG_CHECK(xhp, unneeded, opkgd);
 
-	xbps_object_release(propsd);
+	xbps_object_release(filesd);
 
 #undef RUN_PKG_CHECK
 

bin/xbps-pkgdb/convert.c

@@ -1,117 +0,0 @@
-/*-
- * Copyright (c) 2013 Juan Romero Pardines.
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
- * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
- * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
- * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
- * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
- * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
- * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */
-
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <sys/mman.h>
-#include <fcntl.h>
-#include <stdio.h>
-#include <stdbool.h>
-#include <stdlib.h>
-#include <string.h>
-#include <errno.h>
-#include <unistd.h>
-#include <assert.h>
-
-#include <xbps.h>
-#include "defs.h"
-
-/*
- * Convert pkgdb format to 0.21.
- */
-static void
-pkgdb_format_021(struct xbps_handle *xhp, const char *plist_new)
-{
-	xbps_array_t array, rdeps;
-	xbps_dictionary_t pkgdb, pkgd;
-	char *pkgname, *plist;
-
-	plist = xbps_xasprintf("%s/pkgdb.plist", xhp->metadir);
-	if (access(plist, R_OK) == -1) {
-		if (errno == ENOENT) {
-			/* missing file, no conversion needed */
-			free(plist);
-			return;
-		}
-		xbps_error_printf("cannot read %s: %s\n",
-		    plist, strerror(errno));
-		exit(EXIT_FAILURE);
-	}
-
-	array = xbps_array_internalize_from_zfile(plist);
-	if (xbps_object_type(array) != XBPS_TYPE_ARRAY) {
-		xbps_error_printf("unknown object type for %s\n",
-		    plist, strerror(errno));
-		exit(EXIT_FAILURE);
-	}
-
-	pkgdb = xbps_dictionary_create();
-	assert(pkgdb);
-
-	for (unsigned int i = 0; i < xbps_array_count(array); i++) {
-		pkgd = xbps_array_get(array, i);
-		xbps_dictionary_get_cstring(pkgd, "pkgname", &pkgname);
-		rdeps = xbps_dictionary_get(pkgd, "run_depends");
-		/* remove unneeded objs */
-		xbps_dictionary_remove(pkgd, "pkgname");
-		xbps_dictionary_remove(pkgd, "version");
-		if (xbps_array_count(rdeps) == 0)
-			xbps_dictionary_remove(pkgd, "run_depends");
-
-		xbps_dictionary_set(pkgdb, pkgname, pkgd);
-		free(pkgname);
-	}
-
-	if (xbps_array_count(array) != xbps_dictionary_count(pkgdb)) {
-		xbps_error_printf("failed conversion! unmatched obj count "
-		    "(got %zu, need %zu)\n", xbps_dictionary_count(pkgdb),
-		    xbps_array_count(array));
-		exit(EXIT_FAILURE);
-	}
-
-	if (!xbps_dictionary_externalize_to_file(pkgdb, plist_new)) {
-		xbps_error_printf("failed to write %s: %s\n",
-		    plist_new, strerror(errno));
-		exit(EXIT_FAILURE);
-	}
-
-	xbps_object_release(array);
-	xbps_object_release(pkgdb);
-	free(plist);
-
-	printf("Conversion to 0.21 pkgdb format successfully\n");
-}
-
-void
-convert_pkgdb_format(struct xbps_handle *xhp)
-{
-	char *plist;
-
-	plist = xbps_xasprintf("%s/%s", xhp->metadir, XBPS_PKGDB);
-	if ((access(plist, R_OK) == -1) && (errno == ENOENT))
-		pkgdb_format_021(xhp, plist);
-
-	free(plist);
-}

bin/xbps-pkgdb/main.c

@@ -151,9 +151,10 @@ main(int argc, char **argv)
 		fprintf(stderr, "failed to lock pkgdb: %s\n", strerror(rv));
 		exit(EXIT_FAILURE);
 	}
-	if (update_format)
+	if (update_format) {
-		convert_pkgdb_format(&xh);
+		/* nothing to do; xbps_pkgdb_lock() runs the conversion for us */
-	else if (instmode) {
+		goto out;
+	} else if (instmode) {
 		if (argc == optind) {
 			fprintf(stderr,
 			    "xbps-pkgdb: missing PKGNAME argument\n");
@@ -181,6 +182,7 @@ main(int argc, char **argv)
 		}
 	}
 
+out:
 	xbps_pkgdb_unlock(&xh);
 	exit(rv ? EXIT_FAILURE : EXIT_SUCCESS);
 }

bin/xbps-query/ownedby.c

@@ -103,7 +103,7 @@ ownedby_pkgdb_cb(struct xbps_handle *xhp,
 
 	xbps_dictionary_get_cstring_nocopy(obj, "pkgver", &pkgver);
 
-	pkgmetad = xbps_pkgdb_get_pkg_metadata(xhp, pkgver);
+	pkgmetad = xbps_pkgdb_get_pkg_files(xhp, pkgver);
 	assert(pkgmetad);
 
 	files_keys = xbps_dictionary_all_keys(pkgmetad);

bin/xbps-query/show-info-files.c

@@ -240,27 +240,12 @@ show_pkg_info_from_metadir(struct xbps_handle *xhp,
 			   const char *pkg,
 			   const char *option)
 {
-	xbps_array_t allkeys;
+	xbps_dictionary_t d;
-	xbps_object_t obj, obj2;
-	xbps_dictionary_t d, pkgdb_d;
-	const char *key;
 
-	pkgdb_d = xbps_pkgdb_get_pkg(xhp, pkg);
+	d = xbps_pkgdb_get_pkg(xhp, pkg);
-	if (pkgdb_d == NULL)
-		return ENOENT;
-
-	d = xbps_pkgdb_get_pkg_metadata(xhp, pkg);
 	if (d == NULL)
 		return ENOENT;
 
-	allkeys = xbps_dictionary_all_keys(pkgdb_d);
-	for (unsigned int i = 0; i < xbps_array_count(allkeys); i++) {
-		obj = xbps_array_get(allkeys, i);
-		obj2 = xbps_dictionary_get_keysym(pkgdb_d, obj);
-		key = xbps_dictionary_keysym_cstring_nocopy(obj);
-		xbps_dictionary_set(d, key, obj2);
-	}
-
 	if (option == NULL)
 		show_pkg_info(d);
 	else
@@ -275,7 +260,7 @@ show_pkg_files_from_metadir(struct xbps_handle *xhp, const char *pkg)
 	xbps_dictionary_t d;
 	int rv = 0;
 
-	d = xbps_pkgdb_get_pkg_metadata(xhp, pkg);
+	d = xbps_pkgdb_get_pkg_files(xhp, pkg);
 	if (d == NULL)
 		return ENOENT;
 

include/xbps.h.in

@@ -48,7 +48,7 @@
  *
  * This header documents the full API for the XBPS Library.
  */
-#define XBPS_API_VERSION	"20140906"
+#define XBPS_API_VERSION	"20140909"
 
 #ifndef XBPS_VERSION
  #define XBPS_VERSION		"UNSET"
@@ -128,7 +128,7 @@
  * @def XBPS_PKGDB
  * Filename for the package database.
  */
-#define XBPS_PKGDB		"pkgdb-0.21.plist"
+#define XBPS_PKGDB		"pkgdb-0.38.plist"
 
 /**
  * @def XBPS_PKGPROPS
@@ -291,6 +291,8 @@ extern "C" {
  * - XBPS_STATE_SHOW_INSTALL_MSG: package must show a post-install message.
  * - XBPS_STATE_SHOW_REMOVE_MSG: package must show a pre-remove message.
  * - XBPS_STATE_UNPACK_FILE_PRESERVED: package unpack preserved a file.
+ * - XBPS_STATE_PKGDB: pkgdb upgrade in progress.
+ * - XBPS_STATE_PKGDB_DONE: pkgdb has been upgraded successfully.
  */
 typedef enum xbps_state {
 	XBPS_STATE_UNKNOWN = 0,
@@ -333,7 +335,9 @@ typedef enum xbps_state {
 	XBPS_STATE_INVALID_DEP,
 	XBPS_STATE_SHOW_INSTALL_MSG,
 	XBPS_STATE_SHOW_REMOVE_MSG,
-	XBPS_STATE_UNPACK_FILE_PRESERVED
+	XBPS_STATE_UNPACK_FILE_PRESERVED,
+	XBPS_STATE_PKGDB,
+	XBPS_STATE_PKGDB_DONE
 } xbps_state_t;
 
 /**
@@ -517,7 +521,6 @@ struct xbps_handle {
 	/**
 	 * @private
 	 */
-	xbps_dictionary_t pkg_metad;
 	xbps_dictionary_t pkgdb_revdeps;
 	xbps_dictionary_t vpkgd;
 	/**
@@ -851,14 +854,14 @@ xbps_dictionary_t xbps_pkgdb_get_virtualpkg(struct xbps_handle *xhp,
 					    const char *pkg);
 
 /**
- * Returns the package dictionary with all metadata info for \a pkg.
+ * Returns the package dictionary with all files for \a pkg.
  *
  * @param[in] xhp The pointer to the xbps_handle struct.
  * @param[in] pkg Package expression to match.
  *
- * @return The matching package metadata dictionary, NULL otherwise.
+ * @return The matching package dictionary, NULL otherwise.
  */
-xbps_dictionary_t xbps_pkgdb_get_pkg_metadata(struct xbps_handle *xhp,
+xbps_dictionary_t xbps_pkgdb_get_pkg_files(struct xbps_handle *xhp,
 					   const char *pkg);
 
 /**

include/xbps_api_impl.h

@@ -106,6 +106,7 @@ int HIDDEN dewey_match(const char *, const char *);
  */
 int HIDDEN xbps_pkgdb_init(struct xbps_handle *);
 void HIDDEN xbps_pkgdb_release(struct xbps_handle *);
+int HIDDEN xbps_pkgdb_conversion(struct xbps_handle *);
 
 /**
  * @private
@@ -153,6 +154,7 @@ int HIDDEN xbps_cb_message(struct xbps_handle *, xbps_dictionary_t, const char *
  */
 int HIDDEN xbps_entry_is_a_conf_file(xbps_dictionary_t, const char *);
 int HIDDEN xbps_entry_install_conf_file(struct xbps_handle *,
+					xbps_dictionary_t,
 					xbps_dictionary_t,
 					struct archive_entry *,
 					const char *,

lib/Makefile

@@ -36,7 +36,7 @@ EXTOBJS = external/dewey.o external/fexec.o external/mkpath.o
 OBJS = package_configure.o package_config_files.o package_orphans.o
 OBJS += package_remove.o package_find_obsoletes.o package_state.o
 OBJS += package_unpack.o package_register.o package_script.o verifysig.o
-OBJS += package_msg.o transaction_shlibs.o
+OBJS += package_msg.o pkgdb_conversion.o transaction_shlibs.o
 OBJS += transaction_commit.o transaction_package_replace.o
 OBJS += transaction_dictionary.o transaction_sortdeps.o transaction_ops.o
 OBJS += transaction_revdeps.o pubkey2fp.o

lib/package_config_files.c

@@ -59,26 +59,26 @@ xbps_entry_is_a_conf_file(xbps_dictionary_t filesd,
  */
 int HIDDEN
 xbps_entry_install_conf_file(struct xbps_handle *xhp,
-			     xbps_dictionary_t filesd,
+			     xbps_dictionary_t binpkg_filesd,
+			     xbps_dictionary_t pkg_filesd,
 			     struct archive_entry *entry,
 			     const char *entry_pname,
 			     const char *pkgver,
 			     const char *pkgname)
 {
-	xbps_dictionary_t forigd;
 	xbps_object_t obj, obj2;
 	xbps_object_iterator_t iter, iter2;
 	const char *version = NULL, *cffile, *sha256_new = NULL;
 	char buf[PATH_MAX], *sha256_cur = NULL, *sha256_orig = NULL;
 	int rv = 0;
 
-	assert(xbps_object_type(filesd) == XBPS_TYPE_DICTIONARY);
+	assert(xbps_object_type(binpkg_filesd) == XBPS_TYPE_DICTIONARY);
 	assert(entry);
 	assert(entry_pname);
 	assert(pkgver);
 	assert(pkgname);
 
-	iter = xbps_array_iter_from_dict(filesd, "conf_files");
+	iter = xbps_array_iter_from_dict(binpkg_filesd, "conf_files");
 	if (iter == NULL)
 		return -1;
 
@@ -89,8 +89,7 @@ xbps_entry_install_conf_file(struct xbps_handle *xhp,
 	xbps_dbg_printf(xhp, "%s: processing conf_file %s\n",
 	    pkgver, entry_pname);
 
-	forigd = xbps_pkgdb_get_pkg_metadata(xhp, pkgname);
+	if (pkg_filesd == NULL) {
-	if (forigd == NULL) {
 		/*
 		 * File exists on disk but it's not managed by the same package.
 		 * Install it as file.new-<version>.
@@ -107,7 +106,7 @@ xbps_entry_install_conf_file(struct xbps_handle *xhp,
 		goto out;
 	}
 
-	iter2 = xbps_array_iter_from_dict(forigd, "conf_files");
+	iter2 = xbps_array_iter_from_dict(pkg_filesd, "conf_files");
 	if (iter2 != NULL) {
 		while ((obj2 = xbps_object_iterator_next(iter2))) {
 			xbps_dictionary_get_cstring_nocopy(obj2,

lib/package_configure.c

@@ -80,8 +80,8 @@ xbps_configure_pkg(struct xbps_handle *xhp,
 		   bool check_state,
 		   bool update)
 {
-	xbps_dictionary_t pkgd, pkgmetad;
+	xbps_dictionary_t pkgd;
-	char *pkgname, *plist;
+	char *pkgname;
 	int rv = 0;
 	pkg_state_t state = 0;
 
@@ -125,23 +125,7 @@ xbps_configure_pkg(struct xbps_handle *xhp,
 
 	xbps_set_cb_state(xhp, XBPS_STATE_CONFIGURE, 0, pkgver, NULL);
 
-	plist = xbps_xasprintf("%s/.%s.plist", xhp->metadir, pkgname);
+	rv = xbps_pkg_exec_script(xhp, pkgd, "install-script", "post", update);
-	free(pkgname);
-
-	pkgmetad = xbps_dictionary_internalize_from_file(plist);
-	if (pkgmetad == NULL) {
-		rv = errno;
-		xbps_set_cb_state(xhp, XBPS_STATE_CONFIGURE_FAIL,
-		    errno, pkgver,
-		    "%s: [configure] cannot read metadata plist: %s",
-		    pkgver, strerror(rv));
-		return errno;
-	}
-
-	free(plist);
-	assert(pkgmetad);
-
-	rv = xbps_pkg_exec_script(xhp, pkgmetad, "install-script", "post", update);
 	if (rv != 0) {
 		xbps_set_cb_state(xhp, XBPS_STATE_CONFIGURE_FAIL,
 		    errno, pkgver,
@@ -149,11 +133,6 @@ xbps_configure_pkg(struct xbps_handle *xhp,
 		    "the post ACTION: %s", pkgver, strerror(rv));
 		return rv;
 	}
-	if (state == XBPS_PKG_STATE_INSTALLED) {
-		xbps_object_release(pkgmetad);
-		return rv;
-	}
-
 	rv = xbps_set_pkg_state_dictionary(pkgd, XBPS_PKG_STATE_INSTALLED);
 	if (rv != 0) {
 		xbps_set_cb_state(xhp, XBPS_STATE_CONFIGURE_FAIL, rv,
@@ -165,10 +144,5 @@ xbps_configure_pkg(struct xbps_handle *xhp,
 		xbps_set_cb_state(xhp, XBPS_STATE_CONFIGURE_DONE, 0, pkgver, NULL);
 
 	/* show install-msg if exists */
-	rv = xbps_cb_message(xhp, pkgmetad, "install-msg");
+	return xbps_cb_message(xhp, pkgd, "install-msg");
-
-	if (pkgmetad != NULL)
-		xbps_object_release(pkgmetad);
-
-	return rv;
 }

lib/package_register.c

@@ -34,12 +34,10 @@
 int HIDDEN
 xbps_register_pkg(struct xbps_handle *xhp, xbps_dictionary_t pkgrd)
 {
-	xbps_dictionary_t pkgd;
-	xbps_array_t provides, rundeps;
 	char outstr[64];
 	time_t t;
 	struct tm *tmp;
-	const char *desc, *pkgver, *repo;
+	const char *pkgver;
 	char *pkgname = NULL, *buf, *sha256;
 	int rv = 0;
 	bool autoinst = false;
@@ -47,41 +45,17 @@ xbps_register_pkg(struct xbps_handle *xhp, xbps_dictionary_t pkgrd)
 	assert(xbps_object_type(pkgrd) == XBPS_TYPE_DICTIONARY);
 
 	xbps_dictionary_get_cstring_nocopy(pkgrd, "pkgver", &pkgver);
-	xbps_dictionary_get_cstring_nocopy(pkgrd, "short_desc", &desc);
+	pkgname = xbps_pkg_name(pkgver);
-	xbps_dictionary_get_cstring_nocopy(pkgrd, "repository", &repo);
+	assert(pkgname);
-	xbps_dictionary_get_bool(pkgrd, "automatic-install", &autoinst);
-	provides = xbps_dictionary_get(pkgrd, "provides");
-	rundeps = xbps_dictionary_get(pkgrd, "run_depends");
 
-	assert(pkgver != NULL);
-	assert(desc != NULL);
-
-	pkgd = xbps_pkgdb_get_pkg(xhp, pkgver);
-	if (pkgd == NULL) {
-		rv = ENOENT;
-		goto out;
-	}
-	if (!xbps_dictionary_set_cstring_nocopy(pkgd,
-	    "pkgver", pkgver)) {
-		xbps_dbg_printf(xhp, "%s: invalid pkgver for %s\n",
-		    __func__, pkgver);
-		rv = EINVAL;
-		goto out;
-	}
-	if (!xbps_dictionary_set_cstring_nocopy(pkgd,
-	    "short_desc", desc)) {
-		xbps_dbg_printf(xhp, "%s: invalid short_desc for %s\n",
-		    __func__, pkgver);
-		rv = EINVAL;
-		goto out;
-	}
 	if (xhp->flags & XBPS_FLAG_INSTALL_AUTO)
 		autoinst = true;
-
+	/*
-	if (!xbps_dictionary_set_bool(pkgd,
+	 * Set automatic-install to true, iff it was explicitly set; otherwise
-	    "automatic-install", autoinst)) {
+	 * preserve its value.
-		xbps_dbg_printf(xhp, "%s: invalid autoinst for %s\n",
+	 */
-		    __func__, pkgver);
+	if (autoinst && !xbps_dictionary_set_bool(pkgrd, "automatic-install", true)) {
+		xbps_dbg_printf(xhp, "%s: invalid autoinst for %s\n",  __func__, pkgver);
 		rv = EINVAL;
 		goto out;
 	}
@@ -101,50 +75,31 @@ xbps_register_pkg(struct xbps_handle *xhp, xbps_dictionary_t pkgrd)
 		rv = EINVAL;
 		goto out;
 	}
-	if (!xbps_dictionary_set_cstring(pkgd, "install-date", outstr)) {
+	if (!xbps_dictionary_set_cstring(pkgrd, "install-date", outstr)) {
 		xbps_dbg_printf(xhp, "%s: install-date set failed!\n", pkgver);
 		rv = EINVAL;
 		goto out;
 	}
-
-	if (provides && !xbps_dictionary_set(pkgd, "provides", provides)) {
-		xbps_dbg_printf(xhp, "%s: failed to set provides for %s\n",
-		    __func__, pkgver);
-		rv = EINVAL;
-		goto out;
-	}
-	if (rundeps && !xbps_dictionary_set(pkgd, "run_depends", rundeps)) {
-		xbps_dbg_printf(xhp, "%s: failed to set rundeps for %s\n",
-		    __func__, pkgver);
-		rv = EINVAL;
-		goto out;
-	}
-	/* Save the repository origin which was used to install the pkg from */
-	if (!xbps_dictionary_set_cstring(pkgd, "repository-origin", repo)) {
-		xbps_dbg_printf(xhp, "%s: repository-origin set failed!\n", pkgver);
-		rv = EINVAL;
-		goto out;
-	}
-
 	/*
 	 * Create a hash for the pkg's metafile.
 	 */
-	pkgname = xbps_pkg_name(pkgver);
+	buf = xbps_xasprintf("%s/.%s-files.plist", xhp->metadir, pkgname);
-	assert(pkgname);
-	buf = xbps_xasprintf("%s/.%s.plist", xhp->metadir, pkgname);
 	sha256 = xbps_file_hash(buf);
 	assert(sha256);
-	xbps_dictionary_set_cstring(pkgd, "metafile-sha256", sha256);
+	xbps_dictionary_set_cstring(pkgrd, "metafile-sha256", sha256);
 	free(sha256);
 	free(buf);
 	/*
 	 * Remove unneeded objs from pkg dictionary.
 	 */
-	xbps_dictionary_remove(pkgd, "remove-and-update");
+	xbps_dictionary_remove(pkgrd, "download");
-	xbps_dictionary_remove(pkgd, "transaction");
+	xbps_dictionary_remove(pkgrd, "remove-and-update");
-	xbps_dictionary_remove(pkgd, "skip-obsoletes");
+	xbps_dictionary_remove(pkgrd, "transaction");
+	xbps_dictionary_remove(pkgrd, "skip-obsoletes");
+	xbps_dictionary_remove(pkgrd, "pkgname");
+	xbps_dictionary_remove(pkgrd, "version");
 
-	if (!xbps_dictionary_set(xhp->pkgdb, pkgname, pkgd)) {
+	if (!xbps_dictionary_set(xhp->pkgdb, pkgname, pkgrd)) {
 		xbps_dbg_printf(xhp,
 		    "%s: failed to set pkgd for %s\n", __func__, pkgver);
 	}

lib/package_remove.c

@@ -215,7 +215,7 @@ remove_pkg_files(struct xbps_handle *xhp,
 int HIDDEN
 xbps_remove_pkg(struct xbps_handle *xhp, const char *pkgver, bool update)
 {
-	xbps_dictionary_t pkgd = NULL;
+	xbps_dictionary_t pkgd = NULL, pkgfilesd = NULL;
 	char *pkgname, metafile[PATH_MAX];
 	int rv = 0;
 	pkg_state_t state = 0;
@@ -226,7 +226,13 @@ xbps_remove_pkg(struct xbps_handle *xhp, const char *pkgver, bool update)
 	pkgname = xbps_pkg_name(pkgver);
 	assert(pkgname);
 
-	if ((rv = xbps_pkg_state_installed(xhp, pkgname, &state)) != 0) {
+	if ((pkgd = xbps_pkgdb_get_pkg(xhp, pkgname)) == NULL) {
+		rv = errno;
+		xbps_dbg_printf(xhp, "[remove] cannot find %s in pkgdb: %s\n",
+		    pkgver, strerror(rv));
+		goto out;
+	}
+	if ((rv = xbps_pkg_state_dictionary(pkgd, &state)) != 0) {
 		xbps_dbg_printf(xhp, "[remove] cannot find %s in pkgdb: %s\n",
 		    pkgver, strerror(rv));
 		goto out;
@@ -245,10 +251,10 @@ xbps_remove_pkg(struct xbps_handle *xhp, const char *pkgver, bool update)
 		goto out;
 	}
 
-	/* internalize pkg dictionary from metadir */
+	/* internalize pkg files dictionary from metadir */
-	snprintf(metafile, sizeof(metafile), "%s/.%s.plist", xhp->metadir, pkgname);
+	snprintf(metafile, sizeof(metafile), "%s/.%s-files.plist", xhp->metadir, pkgname);
-	pkgd = xbps_dictionary_internalize_from_file(metafile);
+	pkgfilesd = xbps_dictionary_internalize_from_file(metafile);
-	if (pkgd == NULL)
+	if (pkgfilesd == NULL)
 		xbps_dbg_printf(xhp, "WARNING: metaplist for %s "
 		    "doesn't exist!\n", pkgver);
 
@@ -258,9 +264,7 @@ xbps_remove_pkg(struct xbps_handle *xhp, const char *pkgver, bool update)
 	/*
 	 * Run the pre remove action and show pre-remove message if exists.
 	 */
-	if (pkgd) {
+	rv = xbps_pkg_exec_script(xhp, pkgd, "remove-script", "pre", update);
-		rv = xbps_pkg_exec_script(xhp, pkgd, "remove-script",
-		    "pre", update);
 	if (rv != 0) {
 		xbps_set_cb_state(xhp, XBPS_STATE_REMOVE_FAIL,
 		    errno, pkgver,
@@ -270,44 +274,42 @@ xbps_remove_pkg(struct xbps_handle *xhp, const char *pkgver, bool update)
 		goto out;
 	}
 	/* show remove-msg if exists */
-		rv = xbps_cb_message(xhp, pkgd, "remove-msg");
+	if ((rv = xbps_cb_message(xhp, pkgd, "remove-msg")) != 0)
-		if (rv != 0)
 		goto out;
-	}
+
 	/*
 	 * If updating a package, we just need to execute the current
 	 * pre-remove action target and we are done. Its files will be
 	 * overwritten later in unpack phase.
 	 */
 	if (update) {
-		if (pkgd)
-			xbps_object_release(pkgd);
 		free(pkgname);
 		return 0;
 	}
 
-	if (pkgd) {
+	if (pkgfilesd) {
 		/*
 		 * Do the removal in 2 phases:
 		 * 	1- check if user has enough perms to remove all entries
 		 * 	2- perform removal
 		 */
-		if (check_remove_pkg_files(xhp, pkgd, pkgver)) {
+		if (check_remove_pkg_files(xhp, pkgfilesd, pkgver)) {
 			rv = EPERM;
 			goto out;
 		}
 		/* Remove regular files */
-		if ((rv = remove_pkg_files(xhp, pkgd, "files", pkgver)) != 0)
+		if ((rv = remove_pkg_files(xhp, pkgfilesd, "files", pkgver)) != 0)
 			goto out;
 		/* Remove configuration files */
-		if ((rv = remove_pkg_files(xhp, pkgd, "conf_files", pkgver)) != 0)
+		if ((rv = remove_pkg_files(xhp, pkgfilesd, "conf_files", pkgver)) != 0)
 			goto out;
 		/* Remove links */
-		if ((rv = remove_pkg_files(xhp, pkgd, "links", pkgver)) != 0)
+		if ((rv = remove_pkg_files(xhp, pkgfilesd, "links", pkgver)) != 0)
 			goto out;
 		/* Remove dirs */
-		if ((rv = remove_pkg_files(xhp, pkgd, "dirs", pkgver)) != 0)
+		if ((rv = remove_pkg_files(xhp, pkgfilesd, "dirs", pkgver)) != 0)
 			goto out;
+	}
 	/*
 	 * Execute the post REMOVE action if file exists and we aren't
 	 * updating the package.
@@ -320,8 +322,6 @@ xbps_remove_pkg(struct xbps_handle *xhp, const char *pkgver, bool update)
 		    "post ACTION: %s", pkgver, strerror(rv));
 		goto out;
 	}
-	}
-
 	/*
 	 * Set package state to "half-removed".
 	 */
@@ -339,7 +339,6 @@ purge:
 	/*
 	 * Execute the purge REMOVE action if file exists.
 	 */
-	if (pkgd) {
 	rv = xbps_pkg_exec_script(xhp, pkgd, "remove-script", "purge", false);
 	if (rv != 0) {
 		xbps_set_cb_state(xhp, XBPS_STATE_REMOVE_FAIL,
@@ -348,8 +347,6 @@ purge:
 		    "purge ACTION: %s", pkgver, strerror(rv));
 		goto out;
 	}
-		xbps_object_release(pkgd);
-	}
 	/*
 	 * Remove package metadata plist.
 	 */

lib/package_unpack.c

@@ -78,79 +78,6 @@ find_pkg_symlink_target(xbps_dictionary_t d, const char *file)
 	return tgt;
 }
 
-static int
-create_pkg_metaplist(struct xbps_handle *xhp, const char *pkgname, const char *pkgver,
-		     xbps_dictionary_t propsd, xbps_dictionary_t filesd,
-		     const void *instbuf, const size_t instbufsiz,
-		     const void *rembuf, const size_t rembufsiz)
-{
-	xbps_array_t array;
-	xbps_dictionary_t pkg_metad;
-	xbps_data_t data;
-	char *buf;
-	int rv = 0;
-
-	xbps_dictionary_make_immutable(propsd);
-	pkg_metad = xbps_dictionary_copy_mutable(propsd);
-
-	/* Add objects from XBPS_PKGFILES */
-	array = xbps_dictionary_get(filesd, "files");
-	if (xbps_array_count(array))
-		xbps_dictionary_set(pkg_metad, "files", array);
-	array = xbps_dictionary_get(filesd, "conf_files");
-	if (xbps_array_count(array))
-		xbps_dictionary_set(pkg_metad, "conf_files", array);
-	array = xbps_dictionary_get(filesd, "links");
-	if (xbps_array_count(array))
-		xbps_dictionary_set(pkg_metad, "links", array);
-	array = xbps_dictionary_get(filesd, "dirs");
-	if (xbps_array_count(array))
-		xbps_dictionary_set(pkg_metad, "dirs", array);
-
-	/* Add install/remove scripts data objects */
-	if (instbuf != NULL) {
-		data = xbps_data_create_data(instbuf, instbufsiz);
-		assert(data);
-		xbps_dictionary_set(pkg_metad, "install-script", data);
-		xbps_object_release(data);
-	}
-	if (rembuf != NULL) {
-		data = xbps_data_create_data(rembuf, rembufsiz);
-		assert(data);
-		xbps_dictionary_set(pkg_metad, "remove-script", data);
-		xbps_object_release(data);
-	}
-	/* Remove unneeded objs from transaction */
-	xbps_dictionary_remove(pkg_metad, "remove-and-update");
-	xbps_dictionary_remove(pkg_metad, "transaction");
-	xbps_dictionary_remove(pkg_metad, "state");
-	xbps_dictionary_remove(pkg_metad, "pkgname");
-	xbps_dictionary_remove(pkg_metad, "version");
-
-	/*
-	 * Externalize pkg dictionary to metadir.
-	 */
-	if (access(xhp->metadir, R_OK|X_OK) == -1) {
-		if (errno == ENOENT) {
-			xbps_mkpath(xhp->metadir, 0755);
-		} else {
-			return errno;
-		}
-	}
-	buf = xbps_xasprintf("%s/.%s.plist", XBPS_META_PATH, pkgname);
-	if (!xbps_dictionary_externalize_to_file(pkg_metad, buf)) {
-		rv = errno;
-		xbps_set_cb_state(xhp, XBPS_STATE_UNPACK_FAIL,
-		    errno, pkgver,
-		    "%s: [unpack] failed to write metadata file `%s': %s",
-		    pkgver, buf, strerror(errno));
-	}
-	free(buf);
-	xbps_object_release(pkg_metad);
-
-	return rv;
-}
-
 static bool
 match_preserved_file(struct xbps_handle *xhp, const char *entry)
 {
@@ -176,9 +103,10 @@ unpack_archive(struct xbps_handle *xhp,
 	       const char *fname,
 	       struct archive *ar)
 {
-	xbps_dictionary_t propsd, filesd, metapropsd;
+	xbps_dictionary_t binpkg_propsd, binpkg_filesd, pkg_filesd;
 	xbps_array_t array, obsoletes;
 	xbps_object_t obj;
+	xbps_data_t data;
 	const struct stat *entry_statp;
 	void *instbuf = NULL, *rembuf = NULL;
 	struct stat st;
@@ -193,7 +121,7 @@ unpack_archive(struct xbps_handle *xhp,
 	bool skip_extract, force, xucd_stats;
 	uid_t euid;
 
-	propsd = filesd = metapropsd = NULL;
+	binpkg_propsd = binpkg_filesd = pkg_filesd = NULL;
 	force = preserve = update = conf_file = file_exists = false;
 	skip_obsoletes = xucd_stats = false;
 	ar_rv = rv = entry_type = flags = 0;
@@ -258,19 +186,19 @@ unpack_archive(struct xbps_handle *xhp,
 				goto out;
 			}
 		} else if (strcmp("./props.plist", entry_pname) == 0) {
-			propsd = xbps_archive_get_dictionary(ar, entry);
+			binpkg_propsd = xbps_archive_get_dictionary(ar, entry);
-			if (propsd == NULL) {
+			if (binpkg_propsd == NULL) {
 				rv = EINVAL;
 				goto out;
 			}
 		} else if (strcmp("./files.plist", entry_pname) == 0) {
-			filesd = xbps_archive_get_dictionary(ar, entry);
+			binpkg_filesd = xbps_archive_get_dictionary(ar, entry);
-			if (filesd == NULL) {
+			if (binpkg_filesd == NULL) {
 				rv = EINVAL;
 				goto out;
 			}
 		}
-		if (propsd && filesd)
+		if (binpkg_propsd && binpkg_filesd)
 			break;
 	}
 	/*
@@ -286,24 +214,40 @@ unpack_archive(struct xbps_handle *xhp,
 	/*
 	 * Bail out if required metadata files are not in archive.
 	 */
-	if (propsd == NULL || filesd == NULL) {
+	if (binpkg_propsd == NULL || binpkg_filesd == NULL) {
 		xbps_set_cb_state(xhp, XBPS_STATE_UNPACK_FAIL, ENODEV, pkgver,
 		    "%s: [unpack] invalid binary package `%s'.", pkgver, fname);
 		rv = ENODEV;
 		goto out;
 	}
 	/*
-	 * Create new metaplist file before unpacking any real file.
+	 * Internalize current pkg metadata files plist.
 	 */
-	metapropsd = xbps_pkgdb_get_pkg_metadata(xhp, pkgname);
+	pkg_filesd = xbps_pkgdb_get_pkg_files(xhp, pkgname);
-	rv = create_pkg_metaplist(xhp, pkgname, pkgver,
+
-	    propsd, filesd, instbuf, instbufsiz, rembuf, rembufsiz);
+	/* Add pkg install/remove scripts data objects into our dictionary */
-	if (rv != 0) {
+	if (instbuf != NULL) {
+		data = xbps_data_create_data(instbuf, instbufsiz);
+		assert(data);
+		xbps_dictionary_set(pkg_repod, "install-script", data);
+		xbps_object_release(data);
+	}
+	if (rembuf != NULL) {
+		data = xbps_data_create_data(rembuf, rembufsiz);
+		assert(data);
+		xbps_dictionary_set(pkg_repod, "remove-script", data);
+		xbps_object_release(data);
+	}
+	buf = xbps_xasprintf("%s/.%s-files.plist", xhp->metadir, pkgname);
+	if (!xbps_dictionary_externalize_to_file(binpkg_filesd, buf)) {
+		free(buf);
+		rv = errno;
 		xbps_set_cb_state(xhp, XBPS_STATE_UNPACK_FAIL, rv, pkgver,
-		    "%s: [unpack] failed to create metaplist file: %s",
+		    "%s: [unpack] failed to externalize pkg metadata files: %s",
 		    pkgver, strerror(rv));
 		goto out;
 	}
+	free(buf);
 	/*
 	 * Execute INSTALL "pre" ACTION before unpacking files.
 	 */
@@ -350,14 +294,14 @@ unpack_archive(struct xbps_handle *xhp,
 			 * Compute total entries in progress data, if set.
 			 * total_entries = files + conf_files + links.
 			 */
-			if (filesd && !xucd_stats) {
+			if (binpkg_filesd && !xucd_stats) {
-				array = xbps_dictionary_get(filesd, "files");
+				array = xbps_dictionary_get(binpkg_filesd, "files");
 				xucd.entry_total_count +=
 				    (ssize_t)xbps_array_count(array);
-				array = xbps_dictionary_get(filesd, "conf_files");
+				array = xbps_dictionary_get(binpkg_filesd, "conf_files");
 				xucd.entry_total_count +=
 				    (ssize_t)xbps_array_count(array);
-				array = xbps_dictionary_get(filesd, "links");
+				array = xbps_dictionary_get(binpkg_filesd, "links");
 				xucd.entry_total_count +=
 				    (ssize_t)xbps_array_count(array);
 				xucd_stats = true;
@@ -401,14 +345,14 @@ unpack_archive(struct xbps_handle *xhp,
 				 * "conf_files" array on its XBPS_PKGPROPS
 				 * dictionary.
 				 */
-				if (xbps_entry_is_a_conf_file(filesd, buf)) {
+				if (xbps_entry_is_a_conf_file(binpkg_filesd, buf)) {
 					conf_file = true;
 					if (xhp->unpack_cb != NULL)
 						xucd.entry_is_conf = true;
 
 					rv = xbps_entry_install_conf_file(xhp,
-					    filesd, entry, entry_pname, pkgver,
+					    binpkg_filesd, pkg_filesd, entry,
-					    pkgname);
+					    entry_pname, pkgver, pkgname);
 					if (rv == -1) {
 						/* error */
 						goto out;
@@ -421,7 +365,7 @@ unpack_archive(struct xbps_handle *xhp,
 					rv = 0;
 				} else {
 					rv = xbps_file_hash_check_dictionary(
-					    xhp, filesd, "files", buf);
+					    xhp, binpkg_filesd, "files", buf);
 					if (rv == -1) {
 						/* error */
 						xbps_dbg_printf(xhp,
@@ -456,7 +400,7 @@ unpack_archive(struct xbps_handle *xhp,
 					p += strlen(xhp->rootdir);
 				} else
 					p = buf;
-				tgtlnk = find_pkg_symlink_target(filesd,
+				tgtlnk = find_pkg_symlink_target(binpkg_filesd,
 				    entry_pname);
 				assert(tgtlnk);
 				if (strncmp(tgtlnk, "./", 2) == 0) {
@@ -573,10 +517,10 @@ unpack_archive(struct xbps_handle *xhp,
 	 * 	- Package reinstall.
 	 * 	- Package upgrade.
 	 */
-	if (metapropsd == NULL || !xbps_dictionary_count(metapropsd))
+	if (pkg_filesd == NULL || !xbps_dictionary_count(pkg_filesd))
 		goto out;
 
-	obsoletes = xbps_find_pkg_obsoletes(xhp, metapropsd, filesd);
+	obsoletes = xbps_find_pkg_obsoletes(xhp, pkg_filesd, binpkg_filesd);
 	for (unsigned int i = 0; i < xbps_array_count(obsoletes); i++) {
 		obj = xbps_array_get(obsoletes, i);
 		file = xbps_string_cstring_nocopy(obj);
@@ -593,13 +537,13 @@ unpack_archive(struct xbps_handle *xhp,
 		    0, pkgver, "%s: removed obsolete entry: %s", pkgver, file);
 		xbps_object_release(obj);
 	}
-	xbps_object_release(metapropsd);
+	xbps_object_release(pkg_filesd);
 
 out:
-	if (xbps_object_type(filesd) == XBPS_TYPE_DICTIONARY)
+	if (xbps_object_type(binpkg_filesd) == XBPS_TYPE_DICTIONARY)
-		xbps_object_release(filesd);
+		xbps_object_release(binpkg_filesd);
-	if (xbps_object_type(propsd) == XBPS_TYPE_DICTIONARY)
+	if (xbps_object_type(binpkg_propsd) == XBPS_TYPE_DICTIONARY)
-		xbps_object_release(propsd);
+		xbps_object_release(binpkg_propsd);
 	if (pkgname != NULL)
 		free(pkgname);
 	if (instbuf != NULL)
@@ -670,6 +614,19 @@ xbps_unpack_binary_pkg(struct xbps_handle *xhp, xbps_dictionary_t pkg_repod)
 		    pkgver, bpkg, strerror(rv));
 		goto out;
 	}
+	/*
+	 * Externalize pkg files dictionary to metadir.
+	 */
+	if (access(xhp->metadir, R_OK|X_OK) == -1) {
+		rv = errno;
+		if (rv != ENOENT)
+			goto out;
+
+		if (xbps_mkpath(xhp->metadir, 0755) == -1) {
+			rv = errno;
+			goto out;
+		}
+	}
 	/*
 	 * Extract archive files.
 	 */
@@ -682,7 +639,7 @@ xbps_unpack_binary_pkg(struct xbps_handle *xhp, xbps_dictionary_t pkg_repod)
 	/*
 	 * Set package state to unpacked.
 	 */
-	if ((rv = xbps_set_pkg_state_installed(xhp, pkgver,
+	if ((rv = xbps_set_pkg_state_dictionary(pkg_repod,
 	    XBPS_PKG_STATE_UNPACKED)) != 0) {
 		xbps_set_cb_state(xhp, XBPS_STATE_UNPACK_FAIL,
 		    rv, pkgver,

lib/pkgdb.c

@@ -124,11 +124,15 @@ xbps_pkgdb_init(struct xbps_handle *xhp)
 
 	assert(xhp != NULL);
 
-	if (xhp->pkgdb != NULL)
-		return 0;
 	if (xhp->pkgdb_plist == NULL)
 		xhp->pkgdb_plist = xbps_xasprintf("%s/%s", xhp->metadir, XBPS_PKGDB);
 
+	if ((rv = xbps_pkgdb_conversion(xhp)) != 0)
+		return rv;
+
+	if (xhp->pkgdb != NULL)
+		return 0;
+
 	if ((rv = xbps_pkgdb_update(xhp, false)) != 0) {
 		if (rv != ENOENT)
 			xbps_dbg_printf(xhp, "[pkgdb] cannot internalize "
@@ -190,9 +194,6 @@ xbps_pkgdb_release(struct xbps_handle *xhp)
 	/* write pkgdb to storage in case it was modified */
 	(void)xbps_pkgdb_update(xhp, true);
 
-	if (xbps_object_type(xhp->pkg_metad) == XBPS_TYPE_DICTIONARY)
-	       xbps_object_release(xhp->pkg_metad);
-
 	xbps_object_release(xhp->pkgdb);
 	xhp->pkgdb = NULL;
 	xbps_dbg_printf(xhp, "[pkgdb] released ok.\n");
@@ -327,9 +328,9 @@ xbps_pkgdb_get_pkg_revdeps(struct xbps_handle *xhp, const char *pkg)
 }
 
 xbps_dictionary_t
-xbps_pkgdb_get_pkg_metadata(struct xbps_handle *xhp, const char *pkg)
+xbps_pkgdb_get_pkg_files(struct xbps_handle *xhp, const char *pkg)
 {
-	xbps_dictionary_t pkgd, pkg_metad;
+	xbps_dictionary_t pkgd, pkgfilesd;
 	const char *pkgver;
 	char *pkgname, *plist;
 
@@ -344,27 +345,16 @@ xbps_pkgdb_get_pkg_metadata(struct xbps_handle *xhp, const char *pkg)
 	pkgname = xbps_pkg_name(pkgver);
 	assert(pkgname);
 
-	if ((pkg_metad = xbps_dictionary_get(xhp->pkg_metad, pkgname)) != NULL) {
+	plist = xbps_xasprintf("%s/.%s-files.plist", xhp->metadir, pkgname);
 	free(pkgname);
-		return pkg_metad;
+	pkgfilesd = xbps_dictionary_internalize_from_file(plist);
-	}
-	plist = xbps_xasprintf("%s/.%s.plist", xhp->metadir, pkgname);
-	pkg_metad = xbps_dictionary_internalize_from_file(plist);
 	free(plist);
 
-	if (pkg_metad == NULL) {
+	if (pkgfilesd == NULL) {
 		xbps_dbg_printf(xhp, "[pkgdb] cannot read %s metadata: %s\n",
 		    pkgver, strerror(errno));
-		free(pkgname);
 		return NULL;
 	}
 
-	if (xhp->pkg_metad == NULL)
+	return pkgfilesd;
-		xhp->pkg_metad = xbps_dictionary_create();
-
-	xbps_dictionary_set(xhp->pkg_metad, pkgname, pkg_metad);
-	xbps_object_release(pkg_metad);
-	free(pkgname);
-
-	return pkg_metad;
 }

lib/pkgdb_conversion.c

@@ -0,0 +1,211 @@
+/*-
+ * Copyright (c) 2014 Juan Romero Pardines.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <stdio.h>
+#include <stdbool.h>
+#include <stdlib.h>
+#include <string.h>
+#include <errno.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+
+#include "xbps_api_impl.h"
+
+static int
+pkgdb038(struct xbps_handle *xhp, const char *opkgdb_plist)
+{
+	xbps_dictionary_t pkgdb, opkgdb;
+	xbps_object_iterator_t iter;
+	xbps_object_t obj;
+	int rv = 0;
+
+	/*
+	 * The pkgdb-0.38.plist format contains all pkg metadata objects,
+	 * except its files list. To avoid a broken conversion, the old
+	 * pkg metadata plists are kept, and the converted ones are written
+	 * to another path.
+	 *
+	 * 	- <metadir>/pkgdb-0.38.plist
+	 * 	- <metadir>/.<pkgname>-files.plist
+	 */
+	opkgdb = xbps_dictionary_internalize_from_file(opkgdb_plist);
+	if (opkgdb == NULL)
+		return EINVAL;
+
+	pkgdb = xbps_dictionary_create();
+	assert(pkgdb);
+	/*
+	 * Iterate over the old pkgdb dictionary and copy all pkg objects
+	 * into the new pkgdb dictionary.
+	 */
+	iter = xbps_dictionary_iterator(opkgdb);
+	assert(iter);
+
+	while ((obj = xbps_object_iterator_next(iter))) {
+		xbps_dictionary_t pkgd, pkgfilesd, pkgmetad;
+		xbps_object_iterator_t iter2;
+		xbps_object_t obj2;
+		const char *pkgname;
+		char *pkgmeta;
+
+		pkgname = xbps_dictionary_keysym_cstring_nocopy(obj);
+		pkgd = xbps_dictionary_get_keysym(opkgdb, obj);
+		/*
+		 * Copy old pkgdb objects to the new pkgdb.
+		 */
+		if (!xbps_dictionary_set(pkgdb, pkgname, pkgd)) {
+			xbps_dbg_printf(xhp, "%s: failed to copy %s pkgd "
+			   "for pkgdb conversion\n", __func__, pkgname);
+			rv = EINVAL;
+			goto out;
+		}
+		/*
+		 * Copy pkg metadata objs to the new pkgdb.
+		 */
+		pkgmeta = xbps_xasprintf("%s/.%s.plist", xhp->metadir, pkgname);
+		pkgmetad = xbps_dictionary_internalize_from_file(pkgmeta);
+		if (pkgmetad == NULL) {
+			rv = EINVAL;
+			xbps_dbg_printf(xhp, "%s: cannot open %s: %s\n",
+			    __func__, pkgmeta, strerror(errno));
+			goto out;
+		}
+		pkgfilesd = xbps_dictionary_create();
+		assert(pkgfilesd);
+
+		iter2 = xbps_dictionary_iterator(pkgmetad);
+		assert(iter2);
+
+		while ((obj2 = xbps_object_iterator_next(iter2))) {
+			xbps_object_t curobj;
+			const char *key, *excluded[] = {
+				"conf_files", "dirs", "files", "links"
+			};
+			bool skip = false;
+
+			key = xbps_dictionary_keysym_cstring_nocopy(obj2);
+			curobj = xbps_dictionary_get_keysym(pkgmetad, obj2);
+			for (uint8_t i = 0; i < __arraycount(excluded); i++) {
+				if (strcmp(excluded[i], key) == 0) {
+					skip = true;
+					break;
+				}
+			}
+			if (skip) {
+				assert(xbps_object_type(curobj) == XBPS_TYPE_ARRAY);
+				if (xbps_array_count(curobj))
+					xbps_dictionary_set(pkgfilesd, key, curobj);
+
+				continue;
+			}
+			if (!xbps_dictionary_set(pkgd, key, curobj)) {
+				xbps_dbg_printf(xhp, "%s: failed to copy %s "
+				    "pkgd for pkgdb conversion\n", pkgname, key);
+				xbps_object_iterator_release(iter2);
+				xbps_object_release(pkgmetad);
+				free(pkgmeta);
+				rv = EINVAL;
+				goto out;
+			}
+		}
+		xbps_object_iterator_release(iter2);
+		/*
+		 * Externalize <pkgname>-files.plist if pkg contains any file.
+		 */
+		if (xbps_dictionary_count(pkgfilesd)) {
+			char *pkgfiles, *sha256;
+
+			pkgfiles = xbps_xasprintf("%s/.%s-files.plist", xhp->metadir, pkgname);
+			if (!xbps_dictionary_externalize_to_file(pkgfilesd, pkgfiles)) {
+				xbps_dbg_printf(xhp, "%s: failed to "
+				    "externalize %s: %s\n", __func__, pkgfiles, strerror(errno));
+				rv = EINVAL;
+				goto out;
+			}
+			xbps_dbg_printf(xhp, "%s: externalized %s successfully\n", __func__, pkgfiles);
+			/*
+			 * Update SHA56 hash for the pkg files plist.
+			 */
+			sha256 = xbps_file_hash(pkgfiles);
+			assert(sha256);
+			xbps_dictionary_set_cstring(pkgd, "metafile-sha256", sha256);
+			free(sha256);
+			free(pkgfiles);
+		}
+		xbps_object_release(pkgfilesd);
+		xbps_object_release(pkgmetad);
+		free(pkgmeta);
+	}
+	/*
+	 * Externalize the new pkgdb plist.
+	 */
+	if (!xbps_dictionary_externalize_to_file(pkgdb, xhp->pkgdb_plist)) {
+		xbps_dbg_printf(xhp, "%s: failed to externalize %s: "
+		    "%s!\n", __func__, xhp->pkgdb_plist, strerror(errno));
+		rv = EINVAL;
+		goto out;
+	}
+out:
+	if (iter)
+		xbps_object_iterator_release(iter);
+	if (pkgdb)
+		xbps_object_release(pkgdb);
+	if (opkgdb)
+		xbps_object_release(opkgdb);
+
+	return rv;
+}
+
+int HIDDEN
+xbps_pkgdb_conversion(struct xbps_handle *xhp)
+{
+	char *opkgdb = NULL;
+	int rv = 0;
+
+	/*
+	 * If pkgdb-0.38.plist exists there's nothing to do.
+	 */
+	if (xhp && xhp->pkgdb_plist && (access(xhp->pkgdb_plist, R_OK) == 0))
+		return 0;
+	/*
+	 * If pkgdb-0.21.plist does not exist there's nothing to do.
+	 */
+	opkgdb = xbps_xasprintf("%s/pkgdb-0.21.plist", xhp->metadir);
+	if (access(opkgdb, R_OK) == 0) {
+		/*
+		 * Make the conversion and exit on success. It's just
+		 * better to make the upgrade in two steps.
+		 */
+		xbps_set_cb_state(xhp, XBPS_STATE_PKGDB, 0, NULL, NULL);
+		if ((rv = pkgdb038(xhp, opkgdb)) == 0) {
+			xbps_set_cb_state(xhp, XBPS_STATE_PKGDB_DONE, 0, NULL, NULL);
+			xbps_end(xhp);
+			exit(EXIT_SUCCESS);
+		}
+	}
+	free(opkgdb);
+
+	return rv;
+}

lib/repo_pkgdeps.c

@@ -49,7 +49,8 @@ store_dependency(struct xbps_handle *xhp,
 	/*
 	 * Add required objects into package dep's dictionary.
 	 */
-	if (!xbps_dictionary_set_bool(repo_pkgd, "automatic-install", true))
+	if (!xbps_dictionary_get(repo_pkgd, "automatic-install") &&
+	    !xbps_dictionary_set_bool(repo_pkgd, "automatic-install", true))
 		return EINVAL;
 
 	xbps_dictionary_get_cstring_nocopy(repo_pkgd, "pkgver", &pkgver);

lib/transaction_dictionary.c

@@ -121,7 +121,7 @@ compute_transaction_stats(struct xbps_handle *xhp)
 
 			pkgname = xbps_pkg_name(pkgver);
 			assert(pkgname);
-			pkg_metad = xbps_pkgdb_get_pkg_metadata(xhp, pkgname);
+			pkg_metad = xbps_pkgdb_get_pkg(xhp, pkgname);
 			free(pkgname);
 			if (pkg_metad == NULL)
 				continue;
@@ -220,6 +220,18 @@ xbps_transaction_init(struct xbps_handle *xhp)
 	}
 	xbps_object_release(array);
 
+	if ((array = xbps_array_create()) == NULL) {
+		xbps_object_release(xhp->transd);
+		xhp->transd = NULL;
+		return ENOMEM;
+	}
+	if (!xbps_dictionary_set(xhp->transd, "missing_shlibs", array)) {
+		xbps_object_release(xhp->transd);
+		xhp->transd = NULL;
+		return EINVAL;
+	}
+	xbps_object_release(array);
+
 	if ((array = xbps_array_create()) == NULL) {
 		xbps_object_release(xhp->transd);
 		xhp->transd = NULL;
@@ -270,7 +282,6 @@ xbps_transaction_prepare(struct xbps_handle *xhp)
 	array = xbps_dictionary_get(xhp->transd, "conflicts");
 	if (xbps_array_count(array))
 		return EAGAIN;
-
 	/*
 	 * Check for packages to be replaced.
 	 */
@@ -301,9 +312,10 @@ xbps_transaction_prepare(struct xbps_handle *xhp)
 		return rv;
 	}
 	/*
-	 * The missing deps and conflicts arrays are not necessary anymore.
+	 * Remove now unneeded objects.
 	 */
 	xbps_dictionary_remove(xhp->transd, "unsorted");
+	xbps_dictionary_remove(xhp->transd, "missing_shlibs");
 	xbps_dictionary_remove(xhp->transd, "missing_deps");
 	xbps_dictionary_remove(xhp->transd, "conflicts");
 	xbps_dictionary_make_immutable(xhp->transd);

lib/transaction_ops.c

@@ -136,7 +136,7 @@ trans_find_pkg(struct xbps_handle *xhp, const char *pkg, bool reinstall)
 		 * If pkg is already installed, respect its automatic-install
 		 * property.
 		 */
-		xbps_dictionary_get_bool(pkg_pkgdb, "automatic-install", &autoinst);
+		if (xbps_dictionary_get_bool(pkg_pkgdb, "automatic-install", &autoinst))
 			xbps_dictionary_set_bool(pkg_repod, "automatic-install", autoinst);
 	}
 	/*

lib/transaction_shlibs.c

@@ -57,18 +57,16 @@ shlib_in_pkgdb(struct xbps_handle *xhp, const char *pkgver, const char *shlib)
 
 	while ((obj = xbps_object_iterator_next(iter))) {
 		xbps_array_t shprovides;
-		xbps_dictionary_t pkgd, pkgmetad;
+		xbps_dictionary_t pkgd;
 		const char *curpkgver;
 
 		pkgd = xbps_dictionary_get_keysym(xhp->pkgdb, obj);
-		xbps_dictionary_get_cstring_nocopy(pkgd, "pkgver", &curpkgver);
+		shprovides = xbps_dictionary_get(pkgd, "shlib-provides");
-		pkgmetad = xbps_pkgdb_get_pkg_metadata(xhp, curpkgver);
-		assert(pkgmetad);
-		shprovides = xbps_dictionary_get(pkgmetad, "shlib-provides");
 		if (!shprovides)
 			continue;
 		if (xbps_match_string_in_array(shprovides, shlib)) {
 			/* shlib matched */
+			xbps_dictionary_get_cstring_nocopy(pkgd, "pkgver", &curpkgver);
 			xbps_dbg_printf(xhp, "[trans] %s requires `%s': "
 			    "matched by `%s' (pkgdb)\n", pkgver, shlib, curpkgver);
 			found = true;
@@ -109,9 +107,10 @@ shlib_in_transaction(struct xbps_handle *xhp, const char *pkgver, const char *sh
 bool HIDDEN
 xbps_transaction_shlibs(struct xbps_handle *xhp)
 {
-	xbps_array_t unsorted;
+	xbps_array_t unsorted, mshlibs;
 	bool unmatched = false;
 
+	mshlibs = xbps_dictionary_get(xhp->transd, "missing_shlibs");
 	unsorted = xbps_dictionary_get(xhp->transd, "unsorted_deps");
 	for (unsigned int i = 0; i < xbps_array_count(unsorted); i++) {
 		xbps_array_t shrequires;
@@ -142,8 +141,13 @@ xbps_transaction_shlibs(struct xbps_handle *xhp)
 			xbps_array_get_cstring_nocopy(shrequires, x, &shlib);
 			if ((!shlib_in_pkgdb(xhp, pkgver, shlib)) &&
 			    (!shlib_in_transaction(xhp, pkgver, shlib))) {
-				xbps_dbg_printf(xhp, "[trans] %s: needs `%s' "
+				char *buf;
-				    "not provided by any pkg!\n", pkgver, shlib);
+
+				buf = xbps_xasprintf("%s: needs `%s' shlib, not"
+				    " provided by any pkg!", pkgver, shlib);
+				xbps_dbg_printf(xhp, "%s\n", buf);
+				xbps_array_add_cstring(mshlibs, buf);
+				free(buf);
 				unmatched = true;
 			}
 		}

tests/xbps/libxbps/find_pkg_orphans/Makefile

@@ -3,6 +3,6 @@ TOPDIR = ../../../..
 
 TESTSSUBDIR = xbps/libxbps/find_pkg_orphans
 TEST = find_pkg_orphans_test
-EXTRA_FILES = Kyuafile pkgdb-0.21.plist
+EXTRA_FILES = Kyuafile pkgdb-0.38.plist
 
 include $(TOPDIR)/mk/test.mk

tests/xbps/libxbps/pkgdb/Makefile

@@ -3,6 +3,6 @@ TOPDIR = ../../../..
 
 TESTSSUBDIR = xbps/libxbps/pkgdb
 TEST = pkgdb_test
-EXTRA_FILES = Kyuafile pkgdb-0.21.plist
+EXTRA_FILES = Kyuafile pkgdb-0.38.plist
 
 include $(TOPDIR)/mk/test.mk

tests/xbps/libxbps/shell/conf_files_test.sh

@@ -18,7 +18,7 @@ tc1_body() {
 	rm -rf pkg_a
 	xbps-rindex -a *.xbps
 	atf_check_equal $? 0
-	xbps-install -C null.conf -r rootdir --repository=$PWD -yv a
+	xbps-install -C null.conf -r rootdir --repository=$PWD -yvd a
 	atf_check_equal $? 0
 
 	sed -e 's,fooblah,blahfoo,' -i rootdir/cf1.conf
@@ -29,7 +29,7 @@ tc1_body() {
 	xbps-rindex -a *.xbps
 	rm -rf pkg_a
 	atf_check_equal $? 0
-	xbps-install -C null.conf -r rootdir --repository=$PWD -yuv
+	xbps-install -C null.conf -r rootdir --repository=$PWD -yuvd
 	atf_check_equal $? 0
 	result="$(cat rootdir/cf1.conf)"
 	rval=1
@@ -52,32 +52,34 @@ tc2_body() {
 	cd repo
 	mkdir pkg_a
 	echo "fooblah" > pkg_a/cf1.conf
-	chmod 755 pkg_a/cf1.conf
+	chmod 644 pkg_a/cf1.conf
 	xbps-create -A noarch -n a-0.1_1 -s "pkg a" --config-files "/cf1.conf" pkg_a
 	atf_check_equal $? 0
 	rm -rf pkg_a
 	xbps-rindex -a *.xbps
 	atf_check_equal $? 0
-	xbps-install -C null.conf -r rootdir --repository=$PWD -yv a
+	xbps-install -C null.conf -r rootdir --repository=$PWD -yvd a
 	atf_check_equal $? 0
 
 	sed -e 's,fooblah,blahfoo,' -i rootdir/cf1.conf
 	chmod 644 rootdir/cf1.conf
 	mkdir pkg_a
 	echo "bazbar" > pkg_a/cf1.conf
-	chmod 755 pkg_a/cf1.conf
+	chmod 644 pkg_a/cf1.conf
 	xbps-create -A noarch -n a-0.2_1 -s "pkg a" --config-files "/cf1.conf" pkg_a
 	atf_check_equal $? 0
 	xbps-rindex -a *.xbps
 	rm -rf pkg_a
 	atf_check_equal $? 0
-	xbps-install -C null.conf -r rootdir --repository=$PWD -yuv
+	xbps-install -C null.conf -r rootdir --repository=$PWD -yuvd
 	atf_check_equal $? 0
 	result="$(cat rootdir/cf1.conf)"
 	rval=1
 	if [ "${result}" = "blahfoo" ]; then
 		rval=0
 	fi
+	echo "result: ${result}"
+	echo "expected: blahfoo"
 	atf_check_equal $rval 0
 	rval=1
 	if [ -s rootdir/cf1.conf.new-0.2_1 ]; then
@@ -99,26 +101,26 @@ tc3_body() {
 	cd repo
 	mkdir pkg_a
 	echo "fooblah" > pkg_a/cf1.conf
-	chmod 755 pkg_a/cf1.conf
+	chmod 644 pkg_a/cf1.conf
 	xbps-create -A noarch -n a-0.1_1 -s "pkg a" --config-files "/cf1.conf" pkg_a
 	atf_check_equal $? 0
 	rm -rf pkg_a
 	xbps-rindex -a *.xbps
 	atf_check_equal $? 0
-	xbps-install -C null.conf -r rootdir --repository=$PWD -yv a
+	xbps-install -C null.conf -r rootdir --repository=$PWD -yvd a
 	atf_check_equal $? 0
 
 	sed -e 's,fooblah,blahfoo,' -i rootdir/cf1.conf
 	chmod 644 rootdir/cf1.conf
 	mkdir pkg_a
 	echo "fooblah" > pkg_a/cf1.conf
-	chmod 755 pkg_a/cf1.conf
+	chmod 644 pkg_a/cf1.conf
 	xbps-create -A noarch -n a-0.2_1 -s "pkg a" --config-files "/cf1.conf" pkg_a
 	atf_check_equal $? 0
 	xbps-rindex -a *.xbps
 	rm -rf pkg_a
 	atf_check_equal $? 0
-	xbps-install -C null.conf -r rootdir --repository=$PWD -yuv
+	xbps-install -C null.conf -r rootdir --repository=$PWD -yuvd
 	atf_check_equal $? 0
 	result="$(cat rootdir/cf1.conf)"
 	rval=1
@@ -141,7 +143,7 @@ tc4_body() {
 	cd repo
 	mkdir -p pkg_a/etc
 	echo "fooblah" > pkg_a/etc/cf1.conf
-	chmod 755 pkg_a/etc/cf1.conf
+	chmod 644 pkg_a/etc/cf1.conf
 	xbps-create -A noarch -n a-0.1_1 -s "pkg a" --config-files "/etc/cf1.conf" pkg_a
 	atf_check_equal $? 0
 	rm -rf pkg_a

tests/xbps/libxbps/shell/installmode_test.sh

@@ -19,7 +19,7 @@ instmode_update_body() {
 	atf_check_equal $? 0
 
 	cd ..
-	xbps-install -r root -C null.conf --repository=$PWD/some_repo -A -y A-1.0_1
+	xbps-install -r root -C null.conf --repository=$PWD/some_repo -Ayd A-1.0_1
 	atf_check_equal $? 0
 
 	cd some_repo
@@ -29,7 +29,7 @@ instmode_update_body() {
 	atf_check_equal $? 0
 
 	cd ..
-	xbps-install -r root -C null.conf --repository=$PWD/some_repo -yu
+	xbps-install -r root -C null.conf --repository=$PWD/some_repo -yud
 	atf_check_equal $? 0
 	out=$(xbps-query -r root --property=automatic-install A)
 	atf_check_equal $out yes