emo/xbps
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Improve download code, as well as fixing refetching when syncing to a

Browse Source 
remote package index. Bump XBPS_RELVER to 20091118.

--HG--
extra : convert_revision : xtraeme%40gmail.com-20091118062814-3pqbs9n1797einhm
This commit is contained in:
Juan RP 2009-11-18 07:28:14 +01:00
parent d5f9366bfc
commit bc14a026a5
5 changed files with 84 additions and 58 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
bin/xbps-bin/install.c
View File

@ -164,7 +164,7 @@ download_package_list(prop_object_iterator_t iter)
} }
printf("Downloading %s-%s binary package ...\n", printf("Downloading %s-%s binary package ...\n",
pkgname, version); pkgname, version);
rv = xbps_fetch_file(binfile, savedir, NULL); rv = xbps_fetch_file(binfile, savedir, false, NULL);
free(savedir); free(savedir);
free(binfile); free(binfile);
if (rv != 0) { if (rv != 0) {

2
bin/xbps-fetch/main.c
View File

@ -32,7 +32,7 @@ main(int argc, char **argv)
if (argc != 1) if (argc != 1)
usage(); usage();
rv = xbps_fetch_file(argv[0], ".", flags); rv = xbps_fetch_file(argv[0], ".", false, flags);
if (rv != 0) { if (rv != 0) {
printf("%s: %s\n", argv[0], xbps_fetch_error_string()); printf("%s: %s\n", argv[0], xbps_fetch_error_string());
exit(EXIT_FAILURE); exit(EXIT_FAILURE);

7
include/xbps_api.h
View File

@ -37,7 +37,7 @@
#include <archive_entry.h> #include <archive_entry.h>
/* Current release version */ /* Current release version */
#define XBPS_RELVER "20091109" #define XBPS_RELVER "20091118"
/* Default root PATH for xbps to store metadata info. */ /* Default root PATH for xbps to store metadata info. */
#define XBPS_META_PATH "/var/db/xbps" #define XBPS_META_PATH "/var/db/xbps"
@ -82,10 +82,7 @@ int SYMEXPORT xbps_configure_all_pkgs(void);
int SYMEXPORT xbps_cmpver(const char *, const char *); int SYMEXPORT xbps_cmpver(const char *, const char *);
/* From lib/download.c */ /* From lib/download.c */
int SYMEXPORT xbps_fetch_file(const char *, const char *, const char *); int SYMEXPORT xbps_fetch_file(const char *, const char *, bool, const char *);
void SYMEXPORT (*xbps_fetch_start_cb)(const char *, off_t *, off_t *);
void SYMEXPORT (*xbps_fetch_update_cb)(off_t *);
void SYMEXPORT (*xbps_fetch_end_cb)(void);
const char SYMEXPORT *xbps_fetch_error_string(void); const char SYMEXPORT *xbps_fetch_error_string(void);
/* From lib/fexec.c */ /* From lib/fexec.c */

115
lib/download.c
View File

@ -115,15 +115,14 @@ stat_display(struct xferstat *xsp)
HN_AUTOSCALE, HN_NOSPACE|HN_DECIMAL); HN_AUTOSCALE, HN_NOSPACE|HN_DECIMAL);
(void)xbps_humanize_number(recvsize, 7, (int64_t)xsp->rcvd, "", (void)xbps_humanize_number(recvsize, 7, (int64_t)xsp->rcvd, "",
HN_AUTOSCALE, HN_NOSPACE|HN_DECIMAL); HN_AUTOSCALE, HN_NOSPACE|HN_DECIMAL);
printf("Downloading %s ...", xsp->name); fprintf(stderr, "\r%s: %sB [%d%% of %sB]",
printf("\n\t%sB [%d%% of %sB]", recvsize, xsp->name, recvsize,
(int)((double)(100.0 * (double)xsp->rcvd) / (double)xsp->size), (int)((double)(100.0 * (double)xsp->rcvd) / (double)xsp->size),
totsize); totsize);
printf(" %s", stat_bps(xsp)); fprintf(stderr, " %s", stat_bps(xsp));
if (xsp->size > 0 && xsp->rcvd > 0 && if (xsp->size > 0 && xsp->rcvd > 0 &&
xsp->last.tv_sec >= xsp->start.tv_sec + 10) xsp->last.tv_sec >= xsp->start.tv_sec + 10)
printf(" ETA: %s", stat_eta(xsp)); fprintf(stderr, " ETA: %s", stat_eta(xsp));
printf("\n\033[2A\033[K");
} }
/* /*
@ -160,10 +159,9 @@ stat_end(struct xferstat *xsp)
(void)xbps_humanize_number(size, 6, (int64_t)xsp->size, "", (void)xbps_humanize_number(size, 6, (int64_t)xsp->size, "",
HN_AUTOSCALE, HN_NOSPACE|HN_DECIMAL); HN_AUTOSCALE, HN_NOSPACE|HN_DECIMAL);
printf("\033[1A\033[K"); fprintf(stderr, "\rDownloaded %sB for %s [avg rate: %s]",
printf("Downloaded %s successfully (%sB at %s)\n", size, xsp->name, stat_bps(xsp));
xsp->name, size, stat_bps(xsp)); fprintf(stderr, "\033[K\n");
printf("\033[J");
} }
const char SYMEXPORT * const char SYMEXPORT *
@ -186,7 +184,8 @@ print_time(time_t *t)
#endif #endif
int SYMEXPORT int SYMEXPORT
xbps_fetch_file(const char *uri, const char *outputdir, const char *flags) xbps_fetch_file(const char *uri, const char *outputdir, bool refetch,
const char *flags)
{ {
struct stat st; struct stat st;
struct xferstat xs; struct xferstat xs;
@ -196,10 +195,11 @@ xbps_fetch_file(const char *uri, const char *outputdir, const char *flags)
struct timeval tv[2]; struct timeval tv[2];
ssize_t bytes_read, bytes_written; ssize_t bytes_read, bytes_written;
off_t bytes_dld = -1; off_t bytes_dld = -1;
char buf[4096], *filename, *destfile = NULL; char buf[4096], *filename, *destfile;
int fd = -1, rv = 0; int fd = -1, rv = 0;
bool restart = false; bool restart = false;
filename = destfile = NULL;
bytes_read = bytes_written = -1; bytes_read = bytes_written = -1;
fetchLastErrCode = 0; fetchLastErrCode = 0;
@ -210,7 +210,6 @@ xbps_fetch_file(const char *uri, const char *outputdir, const char *flags)
if (filename == NULL) if (filename == NULL)
return EINVAL; return EINVAL;
filename++; filename++;
/* /*
* Compute destination file path. * Compute destination file path.
*/ */
@ -219,20 +218,19 @@ xbps_fetch_file(const char *uri, const char *outputdir, const char *flags)
rv = errno; rv = errno;
goto out; goto out;
} }
/* /*
* Check if we have to resume a transfer. * Check if we have to resume a transfer.
*/ */
memset(&st, 0, sizeof(st)); memset(&st, 0, sizeof(st));
if (stat(destfile, &st) == 0) if (stat(destfile, &st) == 0) {
if (st.st_size > 0)
restart = true; restart = true;
else { } else {
if (errno != ENOENT) { if (errno != ENOENT) {
rv = errno; rv = errno;
goto out; goto out;
} }
} }
/* /*
* Prepare stuff for libfetch. * Prepare stuff for libfetch.
*/ */
@ -242,10 +240,45 @@ xbps_fetch_file(const char *uri, const char *outputdir, const char *flags)
} }
/* /*
* Establish connection to remote host. * Check if we want to refetch from scratch a file.
*/
if (refetch) {
/*
* Issue a HEAD request to know size and mtime.
*/
if (fetchStat(url, &url_st, NULL) == -1) {
rv = fetchLastErrCode;
goto out;
}
/*
* If mtime and size match do nothing.
*/
if (restart && url_st.size && url_st.mtime &&
url_st.size == st.st_size &&
url_st.mtime == st.st_mtime)
goto out;
/*
* Remove current file (if exists).
*/
if (restart && remove(destfile) == -1) {
rv = errno;
goto out;
}
restart = false;
url->offset = 0;
/*
* Issue the GET request to refetch.
*/
fio = fetchGet(url, flags);
} else {
/*
* Issue a GET and skip the HEAD request, some servers
* (googlecode.com) return a 404 in HEAD requests!
*/ */
url->offset = st.st_size; url->offset = st.st_size;
fio = fetchXGet(url, &url_st, flags); fio = fetchXGet(url, &url_st, flags);
}
#ifdef DEBUG #ifdef DEBUG
printf("st.st_size: %zd\n", (ssize_t)st.st_size); printf("st.st_size: %zd\n", (ssize_t)st.st_size);
printf("st.st_atime: %s\n", print_time(&st.st_atime)); printf("st.st_atime: %s\n", print_time(&st.st_atime));
@ -263,20 +296,20 @@ xbps_fetch_file(const char *uri, const char *outputdir, const char *flags)
printf("url_stat.atime: %s\n", print_time(&url_st.atime)); printf("url_stat.atime: %s\n", print_time(&url_st.atime));
printf("url_stat.mtime: %s\n", print_time(&url_st.mtime)); printf("url_stat.mtime: %s\n", print_time(&url_st.mtime));
#endif #endif
if (fio == NULL) { if (fio == NULL && fetchLastErrCode != FETCH_OK) {
/* Local and remote size match, do nothing */ if (restart && fetchLastErrCode == FETCH_UNAVAIL) {
if (fetchLastErrCode != FETCH_OK) { /*
rv = fetchLastErrCode; * In HTTP when 416 is returned and length==0
goto out; * means that local and remote file size match.
} * Because we are requesting offset==st_size! grr,
* stupid http servers...
*/
if (url->length == 0) if (url->length == 0)
goto out; goto out;
}
rv = fetchLastErrCode; rv = fetchLastErrCode;
goto out; goto out;
} }
if (url_st.size == -1) { if (url_st.size == -1) {
printf("Remote file size is unknown!\n"); printf("Remote file size is unknown!\n");
rv = EINVAL; rv = EINVAL;
@ -286,12 +319,13 @@ xbps_fetch_file(const char *uri, const char *outputdir, const char *flags)
filename); filename);
rv = EFBIG; rv = EFBIG;
goto out; goto out;
} else if (st.st_size == url_st.size && st.st_mtime == url_st.mtime) { } else if (restart && url_st.mtime && url_st.size &&
url_st.size == st.st_size && url_st.mtime == st.st_mtime) {
/* Local and remote size/mtime match, do nothing. */ /* Local and remote size/mtime match, do nothing. */
rv = 0; rv = 0;
goto out; goto out;
} }
printf("Connected to %s.\n", url->host); fprintf(stderr, "Connected to %s.\n", url->host);
/* /*
* If restarting, open the file for appending otherwise create it. * If restarting, open the file for appending otherwise create it.
@ -299,7 +333,7 @@ xbps_fetch_file(const char *uri, const char *outputdir, const char *flags)
if (restart) if (restart)
fd = open(destfile, O_WRONLY|O_APPEND); fd = open(destfile, O_WRONLY|O_APPEND);
else else
fd = open(destfile, O_WRONLY|O_CREAT, 0644); fd = open(destfile, O_WRONLY|O_CREAT|O_TRUNC, 0644);
if (fd == -1) { if (fd == -1) {
rv = errno; rv = errno;
@ -309,42 +343,37 @@ xbps_fetch_file(const char *uri, const char *outputdir, const char *flags)
/* /*
* Start fetching requested file. * Start fetching requested file.
*/ */
if (xbps_fetch_start_cb != NULL)
(*xbps_fetch_start_cb)(filename, &url_st.size, &url->offset);
else
stat_start(&xs, filename, &url_st.size, &url->offset); stat_start(&xs, filename, &url_st.size, &url->offset);
while ((bytes_read = fetchIO_read(fio, buf, sizeof(buf))) > 0) { while ((bytes_read = fetchIO_read(fio, buf, sizeof(buf))) > 0) {
bytes_written = write(fd, buf, (size_t)bytes_read); bytes_written = write(fd, buf, (size_t)bytes_read);
if (bytes_written != bytes_read) { if (bytes_written != bytes_read) {
printf("Couldn't write to %s!\n", destfile); fprintf(stderr, "Couldn't write to %s!\n", destfile);
rv = errno; rv = errno;
goto out; goto out;
} }
bytes_dld += bytes_read; bytes_dld += bytes_read;
if (xbps_fetch_update_cb != NULL)
(*xbps_fetch_update_cb)(&bytes_dld);
else
stat_update(&xs, bytes_dld); stat_update(&xs, bytes_dld);
} }
if (bytes_read == -1) { if (bytes_read == -1) {
printf("IO error while fetching %s: %s\n", filename, fprintf(stderr, "IO error while fetching %s: %s\n", filename,
fetchLastErrString); fetchLastErrString);
rv = EINVAL; rv = EINVAL;
goto out; goto out;
} }
if (xbps_fetch_end_cb != NULL)
(*xbps_fetch_end_cb)();
else
stat_end(&xs); stat_end(&xs);
if (fd == -1)
goto out;
/* /*
* Update mtime to match remote file if download was successful. * Update mtime in local file to match remote file if transfer
* was successful.
*/ */
tv[0].tv_sec = url_st.atime ? url_st.atime : url_st.mtime; tv[0].tv_sec = url_st.atime ? url_st.atime : url_st.mtime;
tv[1].tv_sec = url_st.mtime; tv[1].tv_sec = url_st.mtime;
tv[0].tv_usec = tv[1].tv_usec = 0; tv[0].tv_usec = tv[1].tv_usec = 0;
rv = utimes(destfile, tv); if (utimes(destfile, tv) == -1)
rv = errno;
out: out:
if (fd != -1) if (fd != -1)

2
lib/sync_remote_pkgidx.c
View File

@ -133,7 +133,7 @@ xbps_sync_repository_pkg_index(const char *uri)
free(lrepodir); free(lrepodir);
return errno; return errno;
} }
rv = xbps_fetch_file(rpidx, lrepodir, NULL); rv = xbps_fetch_file(rpidx, lrepodir, true, NULL);
free(rpidx); free(rpidx);
free(lrepodir); free(lrepodir);