Merge pull request #52 from Gottox/xdelta

support for fetching file deltas for repo_sync.

NEWS

@@ -66,6 +66,9 @@ xbps-0.38 (???):
    reproduced easily by installing any awk package (gawk, mawk, or nawk),
    which are providing/replacing the "awk" virtual package.
 
+ * libxbps: add support for handling vcdiff files generated by xdelta. This is
+   currently used with repo_sync.
+
  * libfetch: synchronized with NetBSD pkgsrc/libfetch.
 
  * libfetch: add support for TLS SNI (Server Name Identification) from NetBSD, with

bin/xbps-uhelper/main.c

@@ -59,6 +59,7 @@ usage(void)
 	"    pkgmatch\t\t<pkg-version> <pkg-pattern>\n"
 	"    version\t\t<pkgname>\n"
 	"    real-version\t<pkgname>\n"
+	"    xfetch\t\t<oldfile> <URL[>filename]>\n"
 	"\n"
 	"  Options shared by all actions:\n"
 	"    -C\t\tPath to xbps.conf file.\n"
@@ -81,6 +82,20 @@ usage(void)
 	exit(EXIT_FAILURE);
 }
 
+static char*
+fname(char *url) {
+	char *filename;
+
+	if( (filename = strrchr(url, '>')) ) {
+		*filename = '\0';
+	} else {
+		filename = strrchr(url, '/');
+	}
+	if(filename == NULL)
+		return NULL;
+	return filename + 1;
+}
+
 int
 main(int argc, char **argv)
 {
@@ -88,7 +103,7 @@ main(int argc, char **argv)
 	struct xbps_handle xh;
 	struct xferstat xfer;
 	const char *version, *rootdir = NULL, *conffile = NULL;
-	char *pkgname, *hash, *sep;
+	char *pkgname, *hash, *filename;
 	int flags = 0, c, rv = 0;
 
 	while ((c = getopt(argc, argv, "C:dr:V")) != -1) {
@@ -122,6 +137,7 @@ main(int argc, char **argv)
 
 	if ((strcmp(argv[0], "version") == 0) ||
 	    (strcmp(argv[0], "real-version") == 0) ||
+	    (strcmp(argv[0], "xfetch") == 0) ||
 	    (strcmp(argv[0], "fetch") == 0)) {
 		/*
 		* Initialize libxbps.
@@ -244,18 +260,30 @@ main(int argc, char **argv)
 			}
 			printf("%s\n", hash);
 		}
+	} else if (strcmp(argv[0], "xfetch") == 0) {
+		/* apply a delta from specified URL */
+		if (argc != 3)
+			usage();
+
+		filename = fname(argv[2]);
+		rv = xbps_fetch_delta(&xh, argv[1], argv[2], filename, "v");
+
+		if (rv == -1) {
+			printf("%s: %s\n", argv[2],
+				xbps_fetch_error_string());
+		} else if (rv == 0) {
+			printf("%s: file is identical than remote.\n",
+				argv[3]);
+		} else
+			rv = 0;
 	} else if (strcmp(argv[0], "fetch") == 0) {
 		/* Fetch a file from specified URL */
 		if (argc < 2)
 			usage();
 
 		for (int i = 1; i < argc; i++) {
-			if( (sep = strrchr(argv[i], '>')) ) {
+			filename = fname(argv[i]);
-				*sep = '\0';
+			rv = xbps_fetch_file_dest(&xh, argv[i], filename, "v");
-				rv = xbps_fetch_file_dest(&xh, argv[i], sep+1, "v");
-			} else {
-				rv = xbps_fetch_file(&xh, argv[i], "v");
-			}
 
 			if (rv == -1) {
 				printf("%s: %s\n", argv[i],

include/xbps.h.in

@@ -713,6 +713,22 @@ int xbps_fetch_file(struct xbps_handle *xhp, const char *uri,
 int xbps_fetch_file_dest(struct xbps_handle *xhp, const char *uri,
 		    const char *filename, const char *flags);
 
+/**
+ * Download a vcdiff from a remote URL to current working directory. And apply
+ * it to a given file
+ * 
+ * @param[in] xhp Pointer to an xbps_handle struct.
+ * @param[in] uri Remote URI string.
+ * @param[in] basefile Local basefile to apply the delta to.
+ * @param[in] filename Local filename to safe the file.
+ * @param[in] flags Flags passed to libfetch's fetchXget().
+ * 
+ * @return -1 on error, 0 if not downloaded (because local/remote size/mtime
+ * do not match) and 1 if downloaded successfully.
+ **/
+int xbps_fetch_delta(struct xbps_handle *xhp, const char *basefile,
+		const char *uri, const char *filename, const char *flags);
+
 /**
  * Returns last error string reported by xbps_fetch_file().
  *

lib/download.c

@@ -41,6 +41,8 @@
 #undef _BSD_SOURCE
 #include <errno.h>
 #include <fcntl.h>
+#include <sys/wait.h>
+#include <libgen.h>
 
 #include "xbps_api_impl.h"
 #include "fetch.h"
@@ -129,7 +131,7 @@ xbps_fetch_file_dest(struct xbps_handle *xhp, const char *uri, const char *filen
 	} else {
 		if (errno != ENOENT) {
 			rv = -1;
-			goto out;
+			goto fetch_file_out;
 		}
 	}
 	/*
@@ -143,7 +145,7 @@ xbps_fetch_file_dest(struct xbps_handle *xhp, const char *uri, const char *filen
 	} else {
 		if (errno != ENOENT) {
 			rv = -1;
-			goto out;
+			goto fetch_file_out;
 		}
 	}
 	if (refetch && !restart) {
@@ -178,10 +180,10 @@ xbps_fetch_file_dest(struct xbps_handle *xhp, const char *uri, const char *filen
 	if (fio == NULL) {
 		if (fetchLastErrCode == FETCH_UNCHANGED) {
 			/* Last-Modified matched */
-			goto out;
+			goto fetch_file_out;
 		}
 		rv = -1;
-		goto out;
+		goto fetch_file_out;
 	}
 	if (url_st.size == -1) {
 		xbps_dbg_printf(xhp, "Remote file size is unknown, resume "
@@ -207,7 +209,7 @@ xbps_fetch_file_dest(struct xbps_handle *xhp, const char *uri, const char *filen
 
 	if (fd == -1) {
 		rv = -1;
-		goto out;
+		goto fetch_file_out;
 	}
 	/*
 	 * Initialize data for the fetch progress function callback
@@ -225,7 +227,7 @@ xbps_fetch_file_dest(struct xbps_handle *xhp, const char *uri, const char *filen
 			xbps_dbg_printf(xhp,
 			    "Couldn't write to %s!\n", tempfile);
 			rv = -1;
-			goto out;
+			goto fetch_file_out;
 		}
 		bytes_dload += bytes_read;
 		/*
@@ -241,12 +243,12 @@ xbps_fetch_file_dest(struct xbps_handle *xhp, const char *uri, const char *filen
 		    filename, fetchLastErrString);
 		errno = EIO;
 		rv = -1;
-		goto out;
+		goto fetch_file_out;
 	} else if (url_st.size > 0 && ((bytes_dload + url->offset) != url_st.size)) {
 		xbps_dbg_printf(xhp, "file %s is truncated\n", filename);
 		errno = EIO;
 		rv = -1;
-		goto out;
+		goto fetch_file_out;
 	}
 
 	/*
@@ -264,7 +266,7 @@ xbps_fetch_file_dest(struct xbps_handle *xhp, const char *uri, const char *filen
 	ts[0].tv_nsec = ts[1].tv_nsec = 0;
 	if (futimens(fd, ts) == -1) {
 		rv = -1;
-		goto out;
+		goto fetch_file_out;
 	}
 	(void)close(fd);
 	fd = -1;
@@ -274,11 +276,11 @@ xbps_fetch_file_dest(struct xbps_handle *xhp, const char *uri, const char *filen
 		xbps_dbg_printf(xhp, "failed to rename %s to %s: %s",
 		    tempfile, filename, strerror(errno));
 		rv = -1;
-		goto out;
+		goto fetch_file_out;
 	}
 	rv = 1;
 
-out:
+fetch_file_out:
 	if (fio != NULL)
 		fetchIO_close(fio);
 	if (fd != -1)
@@ -290,6 +292,7 @@ out:
 
 	return rv;
 }
+
 int
 xbps_fetch_file(struct xbps_handle *xhp, const char *uri, const char *flags)
 {
@@ -305,3 +308,83 @@ xbps_fetch_file(struct xbps_handle *xhp, const char *uri, const char *flags)
 
 	return xbps_fetch_file_dest(xhp, uri, filename, flags);
 }
+
+
+int
+xbps_fetch_delta(struct xbps_handle *xhp, const char *basefile, const char *uri, const char *filename, const char *flags)
+{
+	const char xdelta[] = "/usr/bin/xdelta3";
+	char *basehash = NULL, *dname = NULL, *durl = NULL, *tempfile = NULL;
+	int status, exitcode;
+	pid_t pid;
+	int rv = 0;
+	struct stat dummystat;
+
+	if (basefile == NULL ||
+	    stat(basefile, &dummystat) ||
+	    stat(xdelta, &dummystat)) {
+		goto fetch_delta_fallback;
+	}
+	xbps_dbg_printf(xhp, "%s: found. Trying binary diff.\n", xdelta);
+
+	basehash = xbps_file_hash(basefile);
+	assert(basehash);
+
+	dname = xbps_xasprintf("%s.%s.vcdiff", basename(uri), basehash);
+	durl = xbps_xasprintf("%s.%s.vcdiff", uri, basehash);
+	tempfile = xbps_xasprintf("%s.tmp", filename);
+
+	if (xbps_fetch_file_dest(xhp, durl, dname, flags) < 0) {
+		xbps_dbg_printf(xhp, "error while downloading %s, fallback to full "
+				"download\n", durl);
+		goto fetch_delta_fallback;
+	}
+
+	if ((pid = fork()) == 0) {
+		execl(xdelta, xdelta, "-d", "-f", "-s", basefile, dname, tempfile,
+				NULL);
+		exit(127);
+	} else if (pid < 0) {
+		xbps_dbg_printf(xhp, "error while forking, fallback to full "
+				"download\n");
+		goto fetch_delta_fallback;
+	}
+
+	// wait for termination of background process
+	waitpid(pid, &status, 0);
+
+	exitcode = WEXITSTATUS(status);
+	unlink(dname);
+	switch(exitcode) {
+	case 0:    // success
+		rv = 1;
+		if (rename(tempfile, filename) == -1) {
+			xbps_dbg_printf(xhp, "failed to rename %s to %s: %s",
+				tempfile, filename, strerror(errno));
+			rv = -1;
+		}
+		goto fetch_delta_out;
+	case 127:  // cannot execute binary
+		xbps_dbg_printf(xhp, "failed to `%s`, fallback to full download\n",
+				xdelta);
+		goto fetch_delta_fallback;
+	default:   // other error
+		xbps_dbg_printf(xhp, "`%s` exited with code %d, fallback to full "
+				"download\n", xdelta, exitcode);
+		goto fetch_delta_fallback;
+	}
+
+fetch_delta_fallback:
+	rv = xbps_fetch_file_dest(xhp, uri, filename, flags);
+fetch_delta_out:
+	if(tempfile != NULL)
+		free(tempfile);
+	if(dname != NULL)
+		free(dname);
+	if(durl != NULL)
+		free(durl);
+	if(basehash != NULL)
+		free(basehash);
+
+	return rv;
+}

lib/repo_sync.c

@@ -75,7 +75,7 @@ int HIDDEN
 xbps_repo_sync(struct xbps_handle *xhp, const char *uri)
 {
 	const char *arch, *fetchstr = NULL;
-	char *repodata, *lrepodir, *uri_fixedp;
+	char *repodata, *repofile, *lrepodir, *uri_fixedp;
 	int rv = 0;
 
 	assert(uri != NULL);
@@ -124,14 +124,15 @@ xbps_repo_sync(struct xbps_handle *xhp, const char *uri)
 	/*
 	 * Remote repository plist index full URL.
 	 */
-	repodata = xbps_xasprintf("%s/%s-repodata", uri, arch);
+	repofile = xbps_xasprintf("%s-repodata", arch);
+	repodata = xbps_xasprintf("%s/%s", uri, repofile);
 
 	/* reposync start cb */
 	xbps_set_cb_state(xhp, XBPS_STATE_REPOSYNC, 0, repodata, NULL);
 	/*
 	 * Download plist index file from repository.
 	 */
-	if ((rv = xbps_fetch_file(xhp, repodata, NULL)) == -1) {
+	if ((rv = xbps_fetch_delta(xhp, repofile, repodata, repofile, NULL)) == -1) {
 		/* reposync error cb */
 		fetchstr = xbps_fetch_error_string();
 		xbps_set_cb_state(xhp, XBPS_STATE_REPOSYNC_FAIL,
@@ -142,6 +143,7 @@ xbps_repo_sync(struct xbps_handle *xhp, const char *uri)
 		rv = 0;
 
 	free(repodata);
+	free(repofile);
 
 	return rv;
 }