lib: add binary package internalization step

- internalize scripts so we can use them before unpacking packages.
- moves some required metadata checks out of the package unpack stage
  so errors do not leave the system in a half unpacked state.

xbps_transaction_files will be changed later to use the loaded
files.plist instead of opening each binary package on its own again.

include/xbps_api_impl.h

@@ -138,6 +138,7 @@ int HIDDEN xbps_transaction_files(struct xbps_handle *,
 int HIDDEN xbps_transaction_fetch(struct xbps_handle *,
 		xbps_object_iterator_t);
 int HIDDEN xbps_transaction_pkg_deps(struct xbps_handle *, xbps_array_t, xbps_dictionary_t);
+int HIDDEN xbps_transaction_internalize(struct xbps_handle *, xbps_object_iterator_t);
 
 char HIDDEN *xbps_get_remote_repo_string(const char *);
 int HIDDEN xbps_repo_sync(struct xbps_handle *, const char *);

lib/Makefile

@@ -43,6 +43,7 @@ OBJS += transaction_ops.o transaction_store.o transaction_check_replaces.o
 OBJS += transaction_check_revdeps.o transaction_check_conflicts.o
 OBJS += transaction_check_shlibs.o
 OBJS += transaction_files.o transaction_fetch.o transaction_pkg_deps.o
+OBJS += transaction_internalize.o
 OBJS += pubkey2fp.o package_fulldeptree.o
 OBJS += download.o initend.o pkgdb.o
 OBJS += plist.o plist_find.o plist_match.o archive.o

lib/package_unpack.c

@@ -73,25 +73,22 @@ unpack_archive(struct xbps_handle *xhp,
 	       const char *fname,
 	       struct archive *ar)
 {
-	xbps_dictionary_t binpkg_propsd, binpkg_filesd, pkg_filesd, obsd;
+	xbps_dictionary_t binpkg_filesd, pkg_filesd, obsd;
 	xbps_array_t array, obsoletes;
-	xbps_data_t data;
 	xbps_trans_type_t ttype;
 	const struct stat *entry_statp;
-	void *instbuf = NULL, *rembuf = NULL;
 	struct stat st;
 	struct xbps_unpack_cb_data xucd;
 	struct archive_entry *entry;
-	size_t  instbufsiz = 0, rembufsiz = 0;
 	ssize_t entry_size;
-	const char *entry_pname, *binpkg_pkgver, *pkgname;
+	const char *entry_pname, *pkgname;
 	char *buf = NULL;
 	int ar_rv, rv, error, entry_type, flags;
 	bool preserve, update, file_exists, keep_conf_file;
 	bool skip_extract, force, xucd_stats;
 	uid_t euid;
 
-	binpkg_propsd = binpkg_filesd = pkg_filesd = NULL;
+	binpkg_filesd = pkg_filesd = NULL;
 	force = preserve = update = file_exists = false;
 	xucd_stats = false;
 	ar_rv = rv = error = entry_type = flags = 0;
@@ -161,45 +158,20 @@ unpack_archive(struct xbps_handle *xhp,
 		entry_pname = archive_entry_pathname(entry);
 		entry_size = archive_entry_size(entry);
 
-		if (strcmp("./INSTALL", entry_pname) == 0) {
+		if (strcmp("./INSTALL", entry_pname) == 0 ||
-			/*
+		    strcmp("./REMOVE", entry_pname) == 0 ||
-			 * Store file in a buffer to execute it later.
+		    strcmp("./props.plist", entry_pname) == 0) {
-			 */
+			archive_read_data_skip(ar);
-			instbufsiz = entry_size;
-			instbuf = malloc(entry_size);
-			assert(instbuf);
-			if (archive_read_data(ar, instbuf, entry_size) != entry_size) {
-				rv = EINVAL;
-				goto out;
-			}
-		} else if (strcmp("./REMOVE", entry_pname) == 0) {
-			/*
-			 * Store file in a buffer to execute it later.
-			 */
-			rembufsiz = entry_size;
-			rembuf = malloc(entry_size);
-			assert(rembuf);
-			if (archive_read_data(ar, rembuf, entry_size) != entry_size) {
-				rv = EINVAL;
-				goto out;
-			}
-		} else if (strcmp("./props.plist", entry_pname) == 0) {
-			binpkg_propsd = xbps_archive_get_dictionary(ar, entry);
-			if (binpkg_propsd == NULL) {
-				rv = EINVAL;
-				goto out;
-			}
 		} else if (strcmp("./files.plist", entry_pname) == 0) {
 			binpkg_filesd = xbps_archive_get_dictionary(ar, entry);
 			if (binpkg_filesd == NULL) {
 				rv = EINVAL;
 				goto out;
 			}
-		} else {
-			archive_read_data_skip(ar);
-		}
-		if (binpkg_filesd)
 			break;
+		} else {
+			break;
+		}
 	}
 	/*
 	 * If there was any error extracting files from archive, error out.
@@ -214,53 +186,27 @@ unpack_archive(struct xbps_handle *xhp,
 	/*
 	 * Bail out if required metadata files are not in archive.
 	 */
-	if (binpkg_propsd == NULL || binpkg_filesd == NULL) {
+	if (binpkg_filesd == NULL) {
 		xbps_set_cb_state(xhp, XBPS_STATE_UNPACK_FAIL, ENODEV, pkgver,
 		    "%s: [unpack] invalid binary package `%s'.", pkgver, fname);
 		rv = ENODEV;
 		goto out;
 	}
 
-	/*
-	 * Check that the pkgver in the binpkg matches the one we're looking for.
-	 */
-	xbps_dictionary_get_cstring_nocopy(binpkg_propsd, "pkgver", &binpkg_pkgver);
-	if (strcmp(pkgver, binpkg_pkgver) != 0) {
-		xbps_set_cb_state(xhp, XBPS_STATE_UNPACK_FAIL, EINVAL, pkgver,
-		    "%s: [unpack] pkgver mismatch repodata: `%s' binpkg: `%s'.", fname, pkgver, binpkg_pkgver);
-		rv = EINVAL;
-		goto out;
-	}
-
 	/*
 	 * Internalize current pkg metadata files plist.
 	 */
 	pkg_filesd = xbps_pkgdb_get_pkg_files(xhp, pkgname);
 
-	/* Add pkg install/remove scripts data objects into our dictionary */
-	if (instbuf != NULL) {
-		data = xbps_data_create_data(instbuf, instbufsiz);
-		assert(data);
-		xbps_dictionary_set(pkg_repod, "install-script", data);
-		xbps_object_release(data);
-	}
-	if (rembuf != NULL) {
-		data = xbps_data_create_data(rembuf, rembufsiz);
-		assert(data);
-		xbps_dictionary_set(pkg_repod, "remove-script", data);
-		xbps_object_release(data);
-	}
 	/*
 	 * Execute INSTALL "pre" ACTION before unpacking files.
 	 */
-	if (instbuf != NULL) {
+	rv = xbps_pkg_exec_script(xhp, pkg_repod, "install-script", "pre", update);
-		rv = xbps_pkg_exec_buffer(xhp, instbuf, instbufsiz, pkgver, "pre", update);
+	if (rv != 0) {
-		if (rv != 0) {
+		xbps_set_cb_state(xhp, XBPS_STATE_UNPACK_FAIL, rv, pkgver,
-			xbps_set_cb_state(xhp, XBPS_STATE_UNPACK_FAIL, rv, pkgver,
+		    "%s: [unpack] INSTALL script failed to execute pre ACTION: %s",
-			    "%s: [unpack] INSTALL script failed to execute pre ACTION: %s",
+		    pkgver, strerror(rv));
-			    pkgver, strerror(rv));
+		goto out;
-			goto out;
-		}
 	}
 	/*
 	 * Unpack all files on archive now.
@@ -523,14 +469,7 @@ out:
 		unlink(buf);
 		free(buf);
 	}
-	if (xbps_object_type(binpkg_propsd) == XBPS_TYPE_DICTIONARY)
+	xbps_object_release(binpkg_filesd);
-		xbps_object_release(binpkg_propsd);
-	if (xbps_object_type(binpkg_filesd) == XBPS_TYPE_DICTIONARY)
-		xbps_object_release(binpkg_filesd);
-	if (instbuf != NULL)
-		free(instbuf);
-	if (rembuf != NULL)
-		free(rembuf);
 
 	return rv;
 }

lib/transaction_commit.c

@@ -110,6 +110,15 @@ xbps_transaction_commit(struct xbps_handle *xhp)
 	 */
 	xbps_fetch_unset_cache_connection();
 
+	/*
+	 * Internalize metadata of downloaded binary packages.
+	 */
+	if ((rv = xbps_transaction_internalize(xhp, iter)) < 0) {
+		xbps_dbg_printf(xhp, "[trans] failed to internalize transaction binpkgs: "
+		    "%s\n", strerror(-rv));
+		goto out;
+	}
+
 	/*
 	 * Collect files in the transaction and find some issues
 	 * like multiple packages installing the same file.

lib/transaction_internalize.c

@@ -0,0 +1,223 @@
+/*-
+ * Copyright (c) 2021 Duncan Overbruck <mail@duncano.de>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+#include <errno.h>
+#include <fcntl.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "xbps_api_impl.h"
+
+static int
+internalize_script(xbps_dictionary_t pkg_repod, const char *script,
+		struct archive *ar, struct archive_entry *entry)
+{
+	char buffer[BUFSIZ];
+	xbps_data_t data = NULL;
+	char *buf = NULL;
+	int64_t entry_size = archive_entry_size(entry);
+
+	if (entry_size == 0)
+		return 0;
+	if (entry_size < 0)
+		return -EINVAL;
+
+	if ((size_t)entry_size > sizeof buffer) {
+		buf = malloc(entry_size);
+		if (buf == NULL)
+			return -errno;
+	}
+
+	if (archive_read_data(ar, buf != NULL ? buf : buffer, entry_size) != entry_size) {
+		free(buf);
+		return -errno;
+	}
+
+	data = xbps_data_create_data(buf != NULL ? buf : buffer, entry_size);
+	if (data == NULL) {
+		free(buf);
+		return -errno;
+	}
+
+	free(buf);
+	xbps_dictionary_set(pkg_repod, script, data);
+	xbps_object_release(data);
+	return 0;
+}
+
+static int
+internalize_binpkg(struct xbps_handle *xhp, xbps_dictionary_t pkg_repod)
+{
+	xbps_dictionary_t filesd = NULL, propsd = NULL;
+	struct stat st;
+	struct archive *ar = NULL;
+	struct archive_entry *entry;
+	const char *pkgver, *pkgname, *binpkg_pkgver;
+	int pkg_fd = -1;
+	char *pkgfile;
+	int rv = 0;
+
+	xbps_dictionary_get_cstring_nocopy(pkg_repod, "pkgver", &pkgver);
+	assert(pkgver);
+	xbps_dictionary_get_cstring_nocopy(pkg_repod, "pkgname", &pkgname);
+	assert(pkgname);
+
+	pkgfile = xbps_repository_pkg_path(xhp, pkg_repod);
+	if (pkgfile == NULL)
+		return -errno;
+
+	if ((ar = archive_read_new()) == NULL) {
+		free(pkgfile);
+		return -errno;
+	}
+
+	/*
+	 * Enable support for tar format and gzip/bzip2/lzma compression methods.
+	 */
+	archive_read_support_filter_gzip(ar);
+	archive_read_support_filter_bzip2(ar);
+	archive_read_support_filter_xz(ar);
+	archive_read_support_filter_lz4(ar);
+	archive_read_support_filter_zstd(ar);
+	archive_read_support_format_tar(ar);
+
+	pkg_fd = open(pkgfile, O_RDONLY|O_CLOEXEC);
+	if (pkg_fd == -1) {
+		rv = -errno;
+		xbps_set_cb_state(xhp, XBPS_STATE_FILES_FAIL,
+		    -rv, pkgver,
+		    "%s: failed to open binary package `%s': %s",
+		    pkgver, pkgfile, strerror(rv));
+		goto out;
+	}
+	if (fstat(pkg_fd, &st) == -1) {
+		rv = -errno;
+		xbps_set_cb_state(xhp, XBPS_STATE_FILES_FAIL,
+		    -rv, pkgver,
+		    "%s: failed to fstat binary package `%s': %s",
+		    pkgver, pkgfile, strerror(rv));
+		goto out;
+	}
+	if (archive_read_open_fd(ar, pkg_fd, st.st_blksize) == ARCHIVE_FATAL) {
+		rv = archive_errno(ar);
+		xbps_set_cb_state(xhp, XBPS_STATE_FILES_FAIL,
+		    -rv, pkgver,
+		    "%s: failed to read binary package `%s': %s",
+		    pkgver, pkgfile, strerror(rv));
+		goto out;
+	}
+
+	for (uint8_t i = 0; i < 4; i++) {
+		const char *entry_pname;
+		int ar_rv = archive_read_next_header(ar, &entry);
+		if (ar_rv == ARCHIVE_EOF || ar_rv == ARCHIVE_FATAL)
+			break;
+		else if (ar_rv == ARCHIVE_RETRY)
+			continue;
+
+		entry_pname = archive_entry_pathname(entry);
+
+		if (strcmp("./INSTALL", entry_pname) == 0) {
+			rv = internalize_script(pkg_repod, "install-script", ar, entry);
+			if (rv < 0)
+				goto out;
+		} else if (strcmp("./REMOVE", entry_pname) == 0) {
+			rv = internalize_script(pkg_repod, "remove-script", ar, entry);
+			if (rv < 0)
+				goto out;
+		} else if ((strcmp("./files.plist", entry_pname)) == 0) {
+			filesd = xbps_archive_get_dictionary(ar, entry);
+			if (filesd == NULL) {
+				rv = -EINVAL;
+				goto out;
+			}
+		} else if (strcmp("./props.plist", entry_pname) == 0) {
+			propsd = xbps_archive_get_dictionary(ar, entry);
+			if (propsd == NULL) {
+				rv = -EINVAL;
+				goto out;
+			}
+		} else {
+			break;
+		}
+	}
+
+	/*
+	 * Bail out if required metadata files are not in archive.
+	 */
+	if (propsd == NULL || filesd == NULL) {
+		rv = -ENODEV;
+		xbps_set_cb_state(xhp, XBPS_STATE_FILES_FAIL, -rv, pkgver,
+		    "%s: [files] invalid binary package `%s'.", pkgver, pkgfile);
+		goto out;
+	}
+
+	/*
+	 * Bail out if repo pkgver does not match binpkg pkgver, i.e. downgrade attack
+	 * by advertising a old signed package with a new version.
+	 */
+	xbps_dictionary_get_cstring_nocopy(propsd, "pkgver", &binpkg_pkgver);
+	if (strcmp(pkgver, binpkg_pkgver) != 0) {
+		rv = -EINVAL;
+		xbps_set_cb_state(xhp, XBPS_STATE_FILES_FAIL, -rv, pkgver,
+		    "%s: [files] pkgver mismatch repodata: `%s' binpkg: `%s'.",
+		    pkgfile, pkgver, binpkg_pkgver);
+		goto out;
+	}
+
+out:
+	xbps_object_release(propsd);
+	xbps_object_release(filesd);
+	if (pkg_fd != -1)
+		close(pkg_fd);
+	if (ar != NULL)
+		archive_read_finish(ar);
+	free(pkgfile);
+	return rv;
+}
+
+int
+xbps_transaction_internalize(struct xbps_handle *xhp, xbps_object_iterator_t iter)
+{
+	xbps_object_t obj;
+
+	assert(xhp);
+	assert(iter);
+
+	while ((obj = xbps_object_iterator_next(iter)) != NULL) {
+		xbps_trans_type_t ttype;
+		int rv;
+
+		ttype = xbps_transaction_pkg_type(obj);
+		if (ttype != XBPS_TRANS_INSTALL && ttype != XBPS_TRANS_UPDATE)
+			continue;
+
+		rv = internalize_binpkg(xhp, obj);
+		if (rv < 0)
+			return rv;
+	}
+	xbps_object_iterator_reset(iter);
+
+	return 0;
+}