From a77727887dcf52d3d71e1a988a2dd019af3c9681 Mon Sep 17 00:00:00 2001 From: Juan RP Date: Sun, 11 Nov 2012 11:29:49 +0100 Subject: [PATCH] Improvements for xbps_fetch_file and xbps_repository_sync_index. xbps_fetch_file: - A temp file is created with .part extension to improve resuming. - Files are downloaded in cwd. - Switch to futimens(2) and fsync(2). xbps_repository_sync_index: - Do not create local repodir in metadir if it already exists. - Simplify the code thanks to new xbps_fetch_file(). --- bin/xbps-uhelper/main.c | 2 +- configure | 2 +- include/xbps_api.h.in | 12 +--- lib/download.c | 110 +++++++++++++++++++++--------------- lib/external/fexec.c | 1 + lib/repository_sync_index.c | 105 ++++++++++++---------------------- lib/transaction_commit.c | 39 +++++++------ 7 files changed, 129 insertions(+), 142 deletions(-) diff --git a/bin/xbps-uhelper/main.c b/bin/xbps-uhelper/main.c index b484aa66..b4b9d565 100644 --- a/bin/xbps-uhelper/main.c +++ b/bin/xbps-uhelper/main.c @@ -265,7 +265,7 @@ main(int argc, char **argv) usage(); for (i = 1; i < argc; i++) { - rv = xbps_fetch_file(&xh, argv[i], ".", false, "v"); + rv = xbps_fetch_file(&xh, argv[i], "v"); if (rv == -1) { printf("%s: %s\n", argv[1], xbps_fetch_error_string()); diff --git a/configure b/configure index c2871aa7..7ed01c91 100755 --- a/configure +++ b/configure @@ -205,7 +205,7 @@ fi case "$OS" in linux) - echo "CPPFLAGS += -D_XOPEN_SOURCE=500" >>$CONFIG_MK + echo "CPPFLAGS += -D_XOPEN_SOURCE=700" >>$CONFIG_MK echo "CPPFLAGS += -D_FILE_OFFSET_BITS=64" >> $CONFIG_MK ;; *) diff --git a/include/xbps_api.h.in b/include/xbps_api.h.in index d220d017..53fddb81 100644 --- a/include/xbps_api.h.in +++ b/include/xbps_api.h.in @@ -56,7 +56,7 @@ */ #define XBPS_PKGINDEX_VERSION "1.5" -#define XBPS_API_VERSION "20121111" +#define XBPS_API_VERSION "20121111-1" #ifndef XBPS_VERSION #define XBPS_VERSION "UNSET" @@ -641,22 +641,16 @@ int xbps_configure_packages(struct xbps_handle *xhp, bool flush); /** - * Download a file from a remote URL. + * Download a file from a remote URL to current working directory. * * @param[in] xhp Pointer to an xbps_handle struct. * @param[in] uri Remote URI string. - * @param[in] outputdir Directory string to store downloaded file. - * @param[in] refetch If true and local/remote size/mtime do not match, - * fetch the file from scratch. * @param[in] flags Flags passed to libfetch's fetchXget(). * * @return -1 on error, 0 if not downloaded (because local/remote size/mtime * do not match) and 1 if downloaded successfully. **/ -int xbps_fetch_file(struct xbps_handle *xhp, - const char *uri, - const char *outputdir, - bool refetch, +int xbps_fetch_file(struct xbps_handle *xhp, const char *uri, const char *flags); /** diff --git a/lib/download.c b/lib/download.c index e51f8f1f..539ce54c 100644 --- a/lib/download.c +++ b/lib/download.c @@ -88,48 +88,40 @@ xbps_fetch_error_string(void) } int -xbps_fetch_file(struct xbps_handle *xhp, - const char *uri, - const char *outputdir, - bool refetch, - const char *flags) +xbps_fetch_file(struct xbps_handle *xhp, const char *uri, const char *flags) { - struct stat st; + struct stat st, st_tmpfile, *stp; struct url *url = NULL; struct url_stat url_st; struct fetchIO *fio = NULL; - struct timeval tv[2]; + struct timespec ts[2]; off_t bytes_dload = -1; - ssize_t bytes_read = -1, bytes_written; - char buf[4096], *filename, *destfile = NULL; + ssize_t bytes_read = -1, bytes_written = -1; + char buf[4096], *filename, *tempfile; int fd = -1, rv = 0; - bool restart = false; + bool refetch = false, restart = false; - assert(uri != NULL); - assert(outputdir != NULL); + assert(xhp); + assert(uri); + /* Extern vars declared in libfetch */ fetchLastErrCode = 0; - fetchTimeout = xhp->fetch_timeout; fetchRestartCalls = 1; /* * Get the filename specified in URI argument. */ - if ((filename = strrchr(uri, '/')) == NULL) + filename = strrchr(uri, '/') + 1; + if (filename == NULL) return -1; - /* Skip first '/' */ - filename++; - /* - * Compute destination file path. - */ - destfile = xbps_xasprintf("%s/%s", outputdir, filename); + tempfile = xbps_xasprintf("%s.part", filename); /* * Check if we have to resume a transfer. */ - memset(&st, 0, sizeof(st)); - if (stat(destfile, &st) == 0) { - if (st.st_size > 0) + memset(&st_tmpfile, 0, sizeof(st_tmpfile)); + if (stat(tempfile, &st_tmpfile) == 0) { + if (st_tmpfile.st_size > 0) restart = true; } else { if (errno != ENOENT) { @@ -137,6 +129,19 @@ xbps_fetch_file(struct xbps_handle *xhp, goto out; } } + /* + * Check if we have to refetch a transfer. + */ + memset(&st, 0, sizeof(st)); + if (stat(filename, &st) == 0) { + refetch = true; + restart = true; + } else { + if (errno != ENOENT) { + rv = -1; + goto out; + } + } /* * Prepare stuff for libfetch. */ @@ -149,6 +154,7 @@ xbps_fetch_file(struct xbps_handle *xhp, * Check if we want to refetch from scratch a file. */ if (refetch) { + stp = &st; /* * Issue a HEAD request to know size and mtime. */ @@ -159,8 +165,8 @@ xbps_fetch_file(struct xbps_handle *xhp, * If mtime and size match do nothing. */ if (restart && url_st.size && url_st.mtime && - url_st.size == st.st_size && - url_st.mtime == st.st_mtime) + url_st.size == stp->st_size && + url_st.mtime == stp->st_mtime) goto out; /* @@ -172,7 +178,7 @@ xbps_fetch_file(struct xbps_handle *xhp, /* * Remove current file (if exists). */ - if (restart && remove(destfile) == -1) { + if (restart && remove(tempfile) == -1) { rv = -1; goto out; } @@ -187,14 +193,15 @@ xbps_fetch_file(struct xbps_handle *xhp, * Issue a GET and skip the HEAD request, some servers * (googlecode.com) return a 404 in HEAD requests! */ - url->offset = st.st_size; + stp = &st_tmpfile; + url->offset = stp->st_size; fio = fetchXGet(url, &url_st, flags); } /* debug stuff */ - xbps_dbg_printf(xhp, "st.st_size: %zd\n", (ssize_t)st.st_size); - xbps_dbg_printf(xhp, "st.st_atime: %s\n", print_time(&st.st_atime)); - xbps_dbg_printf(xhp, "st.st_mtime: %s\n", print_time(&st.st_mtime)); + xbps_dbg_printf(xhp, "st.st_size: %zd\n", (ssize_t)stp->st_size); + xbps_dbg_printf(xhp, "st.st_atime: %s\n", print_time(&stp->st_atime)); + xbps_dbg_printf(xhp, "st.st_mtime: %s\n", print_time(&stp->st_mtime)); xbps_dbg_printf(xhp, "url->scheme: %s\n", url->scheme); xbps_dbg_printf(xhp, "url->host: %s\n", url->host); xbps_dbg_printf(xhp, "url->port: %d\n", url->port); @@ -225,16 +232,18 @@ xbps_fetch_file(struct xbps_handle *xhp, xbps_dbg_printf(xhp, "Remote file size is unknown, resume " "not possible...\n"); restart = false; - } else if (st.st_size > url_st.size) { + } else if (stp->st_size > url_st.size) { /* * Remove local file if bigger than remote, and refetch the * whole shit again. */ xbps_dbg_printf(xhp, "Local file %s is greater than remote, " "removing local file and refetching...\n", filename); - (void)remove(destfile); + (void)remove(tempfile); + restart = false; } else if (restart && url_st.mtime && url_st.size && - url_st.size == st.st_size && url_st.mtime == st.st_mtime) { + url_st.size == stp->st_size && + url_st.mtime == stp->st_mtime) { /* Local and remote size/mtime match, do nothing. */ goto out; } @@ -242,9 +251,9 @@ xbps_fetch_file(struct xbps_handle *xhp, * If restarting, open the file for appending otherwise create it. */ if (restart) - fd = open(destfile, O_WRONLY|O_APPEND); + fd = open(tempfile, O_WRONLY|O_APPEND); else - fd = open(destfile, O_WRONLY|O_CREAT|O_TRUNC, 0644); + fd = open(tempfile, O_WRONLY|O_CREAT|O_TRUNC, 0644); if (fd == -1) { rv = -1; @@ -263,7 +272,8 @@ xbps_fetch_file(struct xbps_handle *xhp, while ((bytes_read = fetchIO_read(fio, buf, sizeof(buf))) > 0) { bytes_written = write(fd, buf, (size_t)bytes_read); if (bytes_written != bytes_read) { - xbps_dbg_printf(xhp, "Couldn't write to %s!\n", destfile); + xbps_dbg_printf(xhp, + "Couldn't write to %s!\n", tempfile); rv = -1; goto out; } @@ -277,8 +287,8 @@ xbps_fetch_file(struct xbps_handle *xhp, filename, false, true, false); } if (bytes_read == -1) { - xbps_dbg_printf(xhp, "IO error while fetching %s: %s\n", filename, - fetchLastErrString); + xbps_dbg_printf(xhp, "IO error while fetching %s: %s\n", + filename, fetchLastErrString); errno = EIO; rv = -1; goto out; @@ -297,14 +307,24 @@ xbps_fetch_file(struct xbps_handle *xhp, * Update mtime in local file to match remote file if transfer * was successful. */ - tv[0].tv_sec = url_st.atime ? url_st.atime : url_st.mtime; - tv[1].tv_sec = url_st.mtime; - tv[0].tv_usec = tv[1].tv_usec = 0; - if (utimes(destfile, tv) == -1) { + ts[0].tv_sec = url_st.atime ? url_st.atime : url_st.mtime; + ts[1].tv_sec = url_st.mtime; + ts[0].tv_nsec = ts[1].tv_nsec = 0; + if (futimens(fd, ts) == -1) { + rv = -1; + goto out; + } + /* sync and close fd */ + (void)fsync(fd); + (void)close(fd); + + /* File downloaded successfully, rename to destfile */ + if (rename(tempfile, filename) == -1) { + xbps_dbg_printf(xhp, "failed to rename %s to %s: %s", + tempfile, filename, strerror(errno)); rv = -1; goto out; } - /* File downloaded successfully */ rv = 1; out: @@ -314,8 +334,8 @@ out: fetchIO_close(fio); if (url != NULL) fetchFreeURL(url); - if (destfile != NULL) - free(destfile); + if (tempfile != NULL) + free(tempfile); return rv; } diff --git a/lib/external/fexec.c b/lib/external/fexec.c index f721edfc..663f0428 100644 --- a/lib/external/fexec.c +++ b/lib/external/fexec.c @@ -27,6 +27,7 @@ * POSSIBILITY OF SUCH DAMAGE. */ +#define _BSD_SOURCE /* for vfork and chroot */ #include #include #include diff --git a/lib/repository_sync_index.c b/lib/repository_sync_index.c index c041f616..c030747a 100644 --- a/lib/repository_sync_index.c +++ b/lib/repository_sync_index.c @@ -23,7 +23,6 @@ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -#include #include #include #include @@ -88,70 +87,60 @@ xbps_repository_sync_pkg_index(struct xbps_handle *xhp, const char *plistf) { prop_array_t array; - struct url *url = NULL; - struct stat st; - const char *fetch_outputdir, *fetchstr = NULL; - char *rpidx, *lrepodir, *uri_fixedp; - char *tmp_metafile, *lrepofile; + const char *fetchstr = NULL; + char *rpidx, *lrepodir, *uri_fixedp, *lrepofile; int rv = 0; - bool only_sync = false; assert(uri != NULL); - tmp_metafile = rpidx = lrepodir = lrepofile = NULL; + rpidx = uri_fixedp = lrepodir = lrepofile = NULL; /* ignore non remote repositories */ if (!xbps_check_is_repository_uri_remote(uri)) return 0; - if ((url = fetchParseURL(uri)) == NULL) - return -1; - uri_fixedp = xbps_get_remote_repo_string(uri); - if (uri_fixedp == NULL) { - fetchFreeURL(url); + if (uri_fixedp == NULL) return -1; - } - /* - * Create metadir if necessary. - */ - if ((rv = xbps_mkpath(xhp->metadir, 0755)) == -1) { - xbps_set_cb_state(xhp, XBPS_STATE_REPOSYNC_FAIL, - errno, NULL, NULL, - "[reposync] failed to create metadir `%s': %s", - xhp->metadir, strerror(errno)); - goto out; - } /* * Remote repository plist index full URL. */ rpidx = xbps_xasprintf("%s/%s", uri, plistf); - /* - * Save temporary file in metadir, and rename if it - * was downloaded successfully. - */ - tmp_metafile = xbps_xasprintf("%s/%s", xhp->metadir, plistf); /* * Full path to repository directory to store the plist * index file. */ lrepodir = xbps_xasprintf("%s/%s", xhp->metadir, uri_fixedp); /* - * If directory exists probably the plist index file - * was downloaded previously... + * Full path to the local repository index file. */ - rv = stat(lrepodir, &st); - if (rv == 0 && S_ISDIR(st.st_mode)) { - only_sync = true; - fetch_outputdir = lrepodir; - } else - fetch_outputdir = xhp->metadir; + lrepofile = xbps_xasprintf("%s/%s", lrepodir, plistf); + /* + * Create repodir in metadir. + */ + if (access(lrepodir, R_OK|X_OK|W_OK) == -1) { + if ((rv = xbps_mkpath(lrepodir, 0755)) == -1) { + xbps_set_cb_state(xhp, XBPS_STATE_REPOSYNC_FAIL, + errno, NULL, NULL, + "[reposync] failed to create repodir `%s': %s", + lrepodir, strerror(errno)); + goto out; + } + } + if (chdir(lrepodir) == -1) { + xbps_set_cb_state(xhp, XBPS_STATE_REPOSYNC_FAIL, + errno, NULL, NULL, + "[reposync] failed to change dir to repodir `%s': %s", + lrepodir, strerror(errno)); + rv = -1; + goto out; + } /* reposync start cb */ xbps_set_cb_state(xhp, XBPS_STATE_REPOSYNC, 0, uri, plistf, NULL); /* * Download plist index file from repository. */ - if (xbps_fetch_file(xhp, rpidx, fetch_outputdir, true, NULL) == -1) { + if ((rv = xbps_fetch_file(xhp, rpidx, NULL)) == -1) { /* reposync error cb */ fetchstr = xbps_fetch_error_string(); xbps_set_cb_state(xhp, XBPS_STATE_REPOSYNC_FAIL, @@ -159,59 +148,35 @@ xbps_repository_sync_pkg_index(struct xbps_handle *xhp, NULL, NULL, "[reposync] failed to fetch file `%s': %s", rpidx, fetchstr ? fetchstr : strerror(errno)); - rv = -1; goto out; + } else if (rv == 0) { + goto out; + } else { + rv = 0; } - if (only_sync) - goto out; /* * Make sure that downloaded plist file can be internalized, i.e * some HTTP servers don't return proper errors and sometimes - you get an HTML ASCII file :-) + * you get an HTML ASCII file :-) */ - array = prop_array_internalize_from_zfile(tmp_metafile); + array = prop_array_internalize_from_zfile(lrepofile); if (array == NULL) { xbps_set_cb_state(xhp, XBPS_STATE_REPOSYNC_FAIL, 0, NULL, NULL, "[reposync] downloaded file `%s' is not valid.", rpidx); - (void)unlink(tmp_metafile); + (void)unlink(lrepofile); + (void)remove(lrepodir); rv = -1; goto out; } prop_object_release(array); - lrepofile = xbps_xasprintf("%s/%s", lrepodir, plistf); - /* - * Create local repodir to store plist index file. - */ - if ((rv = xbps_mkpath(lrepodir, 0755)) == -1) { - xbps_set_cb_state(xhp, XBPS_STATE_REPOSYNC_FAIL, errno, NULL, NULL, - "[reposync] failed to create repodir for `%s': %s", - lrepodir, strerror(rv)); - goto out; - } - - /* - * Rename to destination file now it has been fetched successfully. - */ - if ((rv = rename(tmp_metafile, lrepofile)) == -1) { - xbps_set_cb_state(xhp, XBPS_STATE_REPOSYNC_FAIL, errno, NULL, NULL, - "[reposync] failed to rename index file `%s' to `%s': %s", - tmp_metafile, lrepofile, strerror(errno)); - } else { - rv = 1; /* success */ - } - out: if (rpidx) free(rpidx); if (lrepodir) free(lrepodir); - if (tmp_metafile) - free(tmp_metafile); if (lrepofile) free(lrepofile); - if (url) - fetchFreeURL(url); if (uri_fixedp) free(uri_fixedp); diff --git a/lib/transaction_commit.c b/lib/transaction_commit.c index 05815786..3159ed0e 100644 --- a/lib/transaction_commit.c +++ b/lib/transaction_commit.c @@ -74,14 +74,10 @@ check_binpkgs_hash(struct xbps_handle *xhp, prop_object_iterator_t iter) prop_dictionary_get_cstring_nocopy(obj, "pkgname", &pkgname); prop_dictionary_get_cstring_nocopy(obj, "version", &version); prop_dictionary_get_cstring_nocopy(obj, "repository", &repoloc); - assert(repoloc != NULL); prop_dictionary_get_cstring_nocopy(obj, "pkgver", &pkgver); - assert(pkgver != NULL); prop_dictionary_get_cstring_nocopy(obj, "filename", &filen); - assert(filen != NULL); prop_dictionary_get_cstring_nocopy(obj, "filename-sha256", &sha256); - assert(sha256 != NULL); binfile = xbps_path_from_repository_uri(xhp, obj, repoloc); if (binfile == NULL) { @@ -124,11 +120,8 @@ download_binpkgs(struct xbps_handle *xhp, prop_object_iterator_t iter) prop_dictionary_get_cstring_nocopy(obj, "pkgname", &pkgname); prop_dictionary_get_cstring_nocopy(obj, "version", &version); prop_dictionary_get_cstring_nocopy(obj, "repository", &repoloc); - assert(repoloc != NULL); prop_dictionary_get_cstring_nocopy(obj, "pkgver", &pkgver); - assert(pkgver != NULL); prop_dictionary_get_cstring_nocopy(obj, "filename", &filen); - assert(filen != NULL); binfile = xbps_path_from_repository_uri(xhp, obj, repoloc); if (binfile == NULL) { @@ -145,14 +138,17 @@ download_binpkgs(struct xbps_handle *xhp, prop_object_iterator_t iter) /* * Create cachedir. */ - if (xbps_mkpath(xhp->cachedir, 0755) == -1) { - xbps_set_cb_state(xhp, XBPS_STATE_DOWNLOAD_FAIL, - errno, pkgname, version, - "%s: [trans] cannot create cachedir `%s': %s", - pkgver, xhp->cachedir, strerror(errno)); - free(binfile); - rv = errno; - break; + if (access(xhp->cachedir, R_OK|X_OK|W_OK) == -1) { + if (xbps_mkpath(xhp->cachedir, 0755) == -1) { + xbps_set_cb_state(xhp, XBPS_STATE_DOWNLOAD_FAIL, + errno, pkgname, version, + "%s: [trans] cannot create cachedir `%s':" + "%s", pkgver, xhp->cachedir, + strerror(errno)); + free(binfile); + rv = errno; + break; + } } xbps_set_cb_state(xhp, XBPS_STATE_DOWNLOAD, 0, pkgname, version, @@ -161,7 +157,18 @@ download_binpkgs(struct xbps_handle *xhp, prop_object_iterator_t iter) /* * Fetch binary package. */ - rv = xbps_fetch_file(xhp, binfile, xhp->cachedir, false, NULL); + if (chdir(xhp->cachedir) == -1) { + xbps_set_cb_state(xhp, XBPS_STATE_DOWNLOAD_FAIL, + errno, pkgname, version, + "%s: [trans] failed to change dir to cachedir" + "`%s': %s", pkgver, xhp->cachedir, + strerror(errno)); + rv = errno; + free(binfile); + break; + } + + rv = xbps_fetch_file(xhp, binfile, NULL); if (rv == -1) { fetchstr = xbps_fetch_error_string(); xbps_set_cb_state(xhp, XBPS_STATE_DOWNLOAD_FAIL,