diff --git a/NEWS b/NEWS index d40c6b85..a90ced87 100644 --- a/NEWS +++ b/NEWS @@ -66,6 +66,9 @@ xbps-0.38 (???): reproduced easily by installing any awk package (gawk, mawk, or nawk), which are providing/replacing the "awk" virtual package. + * libxbps: add support for handling vcdiff files generated by xdelta. This is + currently used with repo_sync. + * libfetch: synchronized with NetBSD pkgsrc/libfetch. * libfetch: add support for TLS SNI (Server Name Identification) from NetBSD, with diff --git a/bin/xbps-uhelper/main.c b/bin/xbps-uhelper/main.c index be4f957b..711907a7 100644 --- a/bin/xbps-uhelper/main.c +++ b/bin/xbps-uhelper/main.c @@ -59,6 +59,7 @@ usage(void) " pkgmatch\t\t \n" " version\t\t\n" " real-version\t\n" + " xfetch\t\t filename]>\n" "\n" " Options shared by all actions:\n" " -C\t\tPath to xbps.conf file.\n" @@ -81,6 +82,20 @@ usage(void) exit(EXIT_FAILURE); } +static char* +fname(char *url) { + char *filename; + + if( (filename = strrchr(url, '>')) ) { + *filename = '\0'; + } else { + filename = strrchr(url, '/'); + } + if(filename == NULL) + return NULL; + return filename + 1; +} + int main(int argc, char **argv) { @@ -88,7 +103,7 @@ main(int argc, char **argv) struct xbps_handle xh; struct xferstat xfer; const char *version, *rootdir = NULL, *conffile = NULL; - char *pkgname, *hash, *sep; + char *pkgname, *hash, *filename; int flags = 0, c, rv = 0; while ((c = getopt(argc, argv, "C:dr:V")) != -1) { @@ -122,6 +137,7 @@ main(int argc, char **argv) if ((strcmp(argv[0], "version") == 0) || (strcmp(argv[0], "real-version") == 0) || + (strcmp(argv[0], "xfetch") == 0) || (strcmp(argv[0], "fetch") == 0)) { /* * Initialize libxbps. @@ -244,18 +260,30 @@ main(int argc, char **argv) } printf("%s\n", hash); } + } else if (strcmp(argv[0], "xfetch") == 0) { + /* apply a delta from specified URL */ + if (argc != 3) + usage(); + + filename = fname(argv[2]); + rv = xbps_fetch_delta(&xh, argv[1], argv[2], filename, "v"); + + if (rv == -1) { + printf("%s: %s\n", argv[2], + xbps_fetch_error_string()); + } else if (rv == 0) { + printf("%s: file is identical than remote.\n", + argv[3]); + } else + rv = 0; } else if (strcmp(argv[0], "fetch") == 0) { /* Fetch a file from specified URL */ if (argc < 2) usage(); for (int i = 1; i < argc; i++) { - if( (sep = strrchr(argv[i], '>')) ) { - *sep = '\0'; - rv = xbps_fetch_file_dest(&xh, argv[i], sep+1, "v"); - } else { - rv = xbps_fetch_file(&xh, argv[i], "v"); - } + filename = fname(argv[i]); + rv = xbps_fetch_file_dest(&xh, argv[i], filename, "v"); if (rv == -1) { printf("%s: %s\n", argv[i], diff --git a/include/xbps.h.in b/include/xbps.h.in index 21912c88..57d0533f 100644 --- a/include/xbps.h.in +++ b/include/xbps.h.in @@ -713,6 +713,22 @@ int xbps_fetch_file(struct xbps_handle *xhp, const char *uri, int xbps_fetch_file_dest(struct xbps_handle *xhp, const char *uri, const char *filename, const char *flags); +/** + * Download a vcdiff from a remote URL to current working directory. And apply + * it to a given file + * + * @param[in] xhp Pointer to an xbps_handle struct. + * @param[in] uri Remote URI string. + * @param[in] basefile Local basefile to apply the delta to. + * @param[in] filename Local filename to safe the file. + * @param[in] flags Flags passed to libfetch's fetchXget(). + * + * @return -1 on error, 0 if not downloaded (because local/remote size/mtime + * do not match) and 1 if downloaded successfully. + **/ +int xbps_fetch_delta(struct xbps_handle *xhp, const char *basefile, + const char *uri, const char *filename, const char *flags); + /** * Returns last error string reported by xbps_fetch_file(). * diff --git a/lib/download.c b/lib/download.c index 0ebfb70d..aec3438e 100644 --- a/lib/download.c +++ b/lib/download.c @@ -41,6 +41,8 @@ #undef _BSD_SOURCE #include #include +#include +#include #include "xbps_api_impl.h" #include "fetch.h" @@ -129,7 +131,7 @@ xbps_fetch_file_dest(struct xbps_handle *xhp, const char *uri, const char *filen } else { if (errno != ENOENT) { rv = -1; - goto out; + goto fetch_file_out; } } /* @@ -143,7 +145,7 @@ xbps_fetch_file_dest(struct xbps_handle *xhp, const char *uri, const char *filen } else { if (errno != ENOENT) { rv = -1; - goto out; + goto fetch_file_out; } } if (refetch && !restart) { @@ -178,10 +180,10 @@ xbps_fetch_file_dest(struct xbps_handle *xhp, const char *uri, const char *filen if (fio == NULL) { if (fetchLastErrCode == FETCH_UNCHANGED) { /* Last-Modified matched */ - goto out; + goto fetch_file_out; } rv = -1; - goto out; + goto fetch_file_out; } if (url_st.size == -1) { xbps_dbg_printf(xhp, "Remote file size is unknown, resume " @@ -207,7 +209,7 @@ xbps_fetch_file_dest(struct xbps_handle *xhp, const char *uri, const char *filen if (fd == -1) { rv = -1; - goto out; + goto fetch_file_out; } /* * Initialize data for the fetch progress function callback @@ -225,7 +227,7 @@ xbps_fetch_file_dest(struct xbps_handle *xhp, const char *uri, const char *filen xbps_dbg_printf(xhp, "Couldn't write to %s!\n", tempfile); rv = -1; - goto out; + goto fetch_file_out; } bytes_dload += bytes_read; /* @@ -241,12 +243,12 @@ xbps_fetch_file_dest(struct xbps_handle *xhp, const char *uri, const char *filen filename, fetchLastErrString); errno = EIO; rv = -1; - goto out; + goto fetch_file_out; } else if (url_st.size > 0 && ((bytes_dload + url->offset) != url_st.size)) { xbps_dbg_printf(xhp, "file %s is truncated\n", filename); errno = EIO; rv = -1; - goto out; + goto fetch_file_out; } /* @@ -264,7 +266,7 @@ xbps_fetch_file_dest(struct xbps_handle *xhp, const char *uri, const char *filen ts[0].tv_nsec = ts[1].tv_nsec = 0; if (futimens(fd, ts) == -1) { rv = -1; - goto out; + goto fetch_file_out; } (void)close(fd); fd = -1; @@ -274,11 +276,11 @@ xbps_fetch_file_dest(struct xbps_handle *xhp, const char *uri, const char *filen xbps_dbg_printf(xhp, "failed to rename %s to %s: %s", tempfile, filename, strerror(errno)); rv = -1; - goto out; + goto fetch_file_out; } rv = 1; -out: +fetch_file_out: if (fio != NULL) fetchIO_close(fio); if (fd != -1) @@ -290,6 +292,7 @@ out: return rv; } + int xbps_fetch_file(struct xbps_handle *xhp, const char *uri, const char *flags) { @@ -305,3 +308,83 @@ xbps_fetch_file(struct xbps_handle *xhp, const char *uri, const char *flags) return xbps_fetch_file_dest(xhp, uri, filename, flags); } + + +int +xbps_fetch_delta(struct xbps_handle *xhp, const char *basefile, const char *uri, const char *filename, const char *flags) +{ + const char xdelta[] = "/usr/bin/xdelta3"; + char *basehash = NULL, *dname = NULL, *durl = NULL, *tempfile = NULL; + int status, exitcode; + pid_t pid; + int rv = 0; + struct stat dummystat; + + if (basefile == NULL || + stat(basefile, &dummystat) || + stat(xdelta, &dummystat)) { + goto fetch_delta_fallback; + } + xbps_dbg_printf(xhp, "%s: found. Trying binary diff.\n", xdelta); + + basehash = xbps_file_hash(basefile); + assert(basehash); + + dname = xbps_xasprintf("%s.%s.vcdiff", basename(uri), basehash); + durl = xbps_xasprintf("%s.%s.vcdiff", uri, basehash); + tempfile = xbps_xasprintf("%s.tmp", filename); + + if (xbps_fetch_file_dest(xhp, durl, dname, flags) < 0) { + xbps_dbg_printf(xhp, "error while downloading %s, fallback to full " + "download\n", durl); + goto fetch_delta_fallback; + } + + if ((pid = fork()) == 0) { + execl(xdelta, xdelta, "-d", "-f", "-s", basefile, dname, tempfile, + NULL); + exit(127); + } else if (pid < 0) { + xbps_dbg_printf(xhp, "error while forking, fallback to full " + "download\n"); + goto fetch_delta_fallback; + } + + // wait for termination of background process + waitpid(pid, &status, 0); + + exitcode = WEXITSTATUS(status); + unlink(dname); + switch(exitcode) { + case 0: // success + rv = 1; + if (rename(tempfile, filename) == -1) { + xbps_dbg_printf(xhp, "failed to rename %s to %s: %s", + tempfile, filename, strerror(errno)); + rv = -1; + } + goto fetch_delta_out; + case 127: // cannot execute binary + xbps_dbg_printf(xhp, "failed to `%s`, fallback to full download\n", + xdelta); + goto fetch_delta_fallback; + default: // other error + xbps_dbg_printf(xhp, "`%s` exited with code %d, fallback to full " + "download\n", xdelta, exitcode); + goto fetch_delta_fallback; + } + +fetch_delta_fallback: + rv = xbps_fetch_file_dest(xhp, uri, filename, flags); +fetch_delta_out: + if(tempfile != NULL) + free(tempfile); + if(dname != NULL) + free(dname); + if(durl != NULL) + free(durl); + if(basehash != NULL) + free(basehash); + + return rv; +} diff --git a/lib/repo_sync.c b/lib/repo_sync.c index 171d1aa7..e334818e 100644 --- a/lib/repo_sync.c +++ b/lib/repo_sync.c @@ -75,7 +75,7 @@ int HIDDEN xbps_repo_sync(struct xbps_handle *xhp, const char *uri) { const char *arch, *fetchstr = NULL; - char *repodata, *lrepodir, *uri_fixedp; + char *repodata, *repofile, *lrepodir, *uri_fixedp; int rv = 0; assert(uri != NULL); @@ -124,14 +124,15 @@ xbps_repo_sync(struct xbps_handle *xhp, const char *uri) /* * Remote repository plist index full URL. */ - repodata = xbps_xasprintf("%s/%s-repodata", uri, arch); + repofile = xbps_xasprintf("%s-repodata", arch); + repodata = xbps_xasprintf("%s/%s", uri, repofile); /* reposync start cb */ xbps_set_cb_state(xhp, XBPS_STATE_REPOSYNC, 0, repodata, NULL); /* * Download plist index file from repository. */ - if ((rv = xbps_fetch_file(xhp, repodata, NULL)) == -1) { + if ((rv = xbps_fetch_delta(xhp, repofile, repodata, repofile, NULL)) == -1) { /* reposync error cb */ fetchstr = xbps_fetch_error_string(); xbps_set_cb_state(xhp, XBPS_STATE_REPOSYNC_FAIL, @@ -142,6 +143,7 @@ xbps_repo_sync(struct xbps_handle *xhp, const char *uri) rv = 0; free(repodata); + free(repofile); return rv; }