emo/xbps
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Fixed issue 21 "xbps-repo find-files is awfully slow" reported by ojab.

Browse Source
This commit is contained in:
Juan RP 2012-01-15 14:24:44 +01:00
parent 8780e15635
commit 2186e16a5e
10 changed files with 322 additions and 90 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
NEWS
View File

@ -1,5 +1,13 @@
xbps-0.12.0 (???):
* xbps-repo(8): the 'genindex' target now creates a plist file to cache
all files provided by binary packages in a repository. This makes
the 'find-files' target marginally faster, because this avoids having
to fetch data from network in remote repositories. The new file
'index-files.plist' will be downloaded every time the 'sync' target
is issued but only iff the file has been modified.
This fixes Issue 21 "`xbps-repo find-files` is awfully slow".
* It's now possible to fetch files thru xbps_fetch_file() from HTTP
servers that don't set the Content-Length header. This fixes issue 19:
"xbps-src cannot fetch source with unknown Content-Length" reported

1
bin/xbps-repo/Makefile
View File

@ -3,6 +3,7 @@ TOPDIR = ../..
BIN = xbps-repo
OBJS = main.o index.o show.o find-files.o list.o
OBJS += index-files.o
OBJS += ../xbps-bin/fetch_cb.o ../xbps-bin/util.o
OBJS += ../xbps-bin/state_cb.o ../xbps-bin/list.o
MAN = $(BIN).8

5
bin/xbps-repo/defs.h
View File

@ -1,5 +1,5 @@
/*-
* Copyright (c) 2009-2011 Juan Romero Pardines.
* Copyright (c) 2009-2012 Juan Romero Pardines.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
@ -41,6 +41,9 @@ struct repo_search_data {
/* From index.c */
int repo_genindex(const char *);
/* From index-files.c */
int repo_genindex_files(const char *);
/* From find-files.c */
int repo_find_files_in_packages(int, char **);

131
bin/xbps-repo/find-files.c
View File

@ -1,5 +1,5 @@
/*-
* Copyright (c) 2010 Juan Romero Pardines.
* Copyright (c) 2010-2012 Juan Romero Pardines.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
@ -37,100 +37,81 @@ struct ffdata {
char **patterns;
};
static int
match_files_by_pattern(prop_dictionary_t pkg_filesd,
prop_dictionary_keysym_t key,
struct ffdata *ffd,
const char *pkgver)
static void
match_files_by_pattern(prop_dictionary_t pkg_filesd, struct ffdata *ffd)
{
prop_object_iterator_t iter;
prop_array_t array;
prop_array_t array, allkeys;
prop_object_t obj;
const char *keyname, *filestr, *typestr;
int i;
prop_dictionary_keysym_t key;
const char *keyname, *filestr, *typestr, *pkgver;
size_t i;
int x;
keyname = prop_dictionary_keysym_cstring_nocopy(key);
array = prop_dictionary_get_keysym(pkg_filesd, key);
if (prop_object_type(array) != PROP_TYPE_ARRAY)
return 0;
allkeys = prop_dictionary_all_keys(pkg_filesd);
for (i = 0; i < prop_array_count(allkeys); i++) {
key = prop_array_get(allkeys, i);
keyname = prop_dictionary_keysym_cstring_nocopy(key);
array = prop_dictionary_get_keysym(pkg_filesd, key);
if (prop_object_type(array) != PROP_TYPE_ARRAY)
break;
if (strcmp(keyname, "files") == 0)
typestr = "regular file";
else if (strcmp(keyname, "dirs") == 0)
typestr = "directory";
else if (strcmp(keyname, "links") == 0)
typestr = "link";
else
typestr = "configuration file";
if (strcmp(keyname, "files") == 0)
typestr = "regular file";
else if (strcmp(keyname, "links") == 0)
typestr = "link";
else
typestr = "configuration file";
iter = prop_array_iterator(array);
while ((obj = prop_object_iterator_next(iter))) {
prop_dictionary_get_cstring_nocopy(obj, "file", &filestr);
for (i = 1; i < ffd->npatterns; i++) {
if ((strcmp(filestr, ffd->patterns[i]) == 0) ||
(strstr(filestr, ffd->patterns[i])) ||
(xbps_pkgpattern_match(filestr,
ffd->patterns[i]) == 1))
printf(" %s: %s (%s)\n", pkgver, filestr, typestr);
iter = prop_array_iterator(array);
while ((obj = prop_object_iterator_next(iter))) {
prop_dictionary_get_cstring_nocopy(obj, "file", &filestr);
for (x = 1; x < ffd->npatterns; x++) {
if ((strcmp(filestr, ffd->patterns[x]) == 0) ||
(strstr(filestr, ffd->patterns[x])) ||
(xbps_pkgpattern_match(filestr,
ffd->patterns[x]) == 1)) {
prop_dictionary_get_cstring_nocopy(
pkg_filesd, "pkgver", &pkgver);
printf(" %s: %s (%s)\n",
pkgver, filestr, typestr);
}
}
}
prop_object_iterator_release(iter);
}
prop_object_iterator_release(iter);
return 0;
prop_object_release(allkeys);
}
static int
find_files_in_package(struct repository_pool_index *rpi, void *arg, bool *done)
{
prop_dictionary_t pkg_filesd;
prop_dictionary_t idxfilesd;
prop_array_t files_keys;
prop_object_t obj;
prop_object_iterator_t iter;
struct ffdata *ffd = arg;
const char *pkgname, *pkgver;
char *url;
int rv = 0;
unsigned int i, count;
char *plist;
unsigned int i;
(void)done;
iter = xbps_array_iter_from_dict(rpi->rpi_repod, "packages");
if (iter == NULL)
return -1;
printf("Looking in repository '%s', please wait...\n", rpi->rpi_uri);
while ((obj = prop_object_iterator_next(iter))) {
url = xbps_path_from_repository_uri(obj, rpi->rpi_uri);
if (url == NULL) {
rv = -1;
break;
}
prop_dictionary_get_cstring_nocopy(obj, "pkgname", &pkgname);
prop_dictionary_get_cstring_nocopy(obj, "pkgver", &pkgver);
pkg_filesd = xbps_dictionary_metadata_plist_by_url(url,
XBPS_PKGFILES);
free(url);
if (pkg_filesd == NULL) {
xbps_error_printf("xbps-repo: couldn't read '%s' "
"from '%s (%s)': %s\n", XBPS_PKGFILES, pkgname,
rpi->rpi_uri, strerror(errno));
rv = -1;
break;
}
files_keys = prop_dictionary_all_keys(pkg_filesd);
count = prop_array_count(files_keys);
for (i = 0; i < count; i++) {
rv = match_files_by_pattern(pkg_filesd,
prop_array_get(files_keys, i), ffd, pkgver);
if (rv == -1)
break;
}
prop_object_release(files_keys);
prop_object_release(pkg_filesd);
if (rv == -1)
break;
plist = xbps_pkg_index_files_plist(rpi->rpi_uri);
if (plist == NULL)
return ENOMEM;
idxfilesd = prop_dictionary_internalize_from_zfile(plist);
if (idxfilesd == NULL) {
free(plist);
return errno;
}
prop_object_iterator_release(iter);
return rv;
free(plist);
files_keys = prop_dictionary_get(idxfilesd, "packages");
for (i = 0; i < prop_array_count(files_keys); i++)
match_files_by_pattern(prop_array_get(files_keys, i), ffd);
prop_object_release(idxfilesd);
return 0;
}
int

196
bin/xbps-repo/index-files.c Normal file
View File

@ -0,0 +1,196 @@
/*-
* Copyright (c) 2012 Juan Romero Pardines.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
* IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <stdbool.h>
#include <errno.h>
#include <xbps_api.h>
#include "defs.h"
struct index_files_data {
prop_array_t idxfiles;
const char *pkgdir;
};
static int
genindex_files_cb(prop_object_t obj, void *arg, bool *done)
{
prop_dictionary_t pkg_filesd, pkgd;
prop_array_t array;
struct index_files_data *ifd = arg;
const char *binpkg, *pkgver;
char *file;
bool found = false;
(void)done;
prop_dictionary_get_cstring_nocopy(obj, "filename", &binpkg);
prop_dictionary_get_cstring_nocopy(obj, "pkgver", &pkgver);
file = xbps_xasprintf("%s/%s", ifd->pkgdir, binpkg);
if (file == NULL)
return ENOMEM;
/* internalize files.plist from binary package archive */
pkg_filesd = xbps_dictionary_metadata_plist_by_url(file, XBPS_PKGFILES);
if (pkg_filesd == NULL) {
free(file);
return EINVAL;
}
free(file);
/* create pkg dictionary */
pkgd = prop_dictionary_create();
if (pkgd == NULL) {
prop_object_release(pkg_filesd);
return ENOMEM;
}
/* add conf_files array in pkgd */
array = prop_dictionary_get(pkg_filesd, "conf_files");
if (array != NULL && prop_array_count(array)) {
found = true;
if (!prop_dictionary_set(pkgd, "conf_files", array)) {
prop_object_release(pkgd);
prop_object_release(pkg_filesd);
return EINVAL;
}
}
/* add files array in pkgd */
array = prop_dictionary_get(pkg_filesd, "files");
if (array != NULL && prop_array_count(array)) {
found = true;
if (!prop_dictionary_set(pkgd, "files", array)) {
prop_object_release(pkgd);
prop_object_release(pkg_filesd);
return EINVAL;
}
}
/* add links array in pkgd */
array = prop_dictionary_get(pkg_filesd, "links");
if (array != NULL && prop_array_count(array)) {
found = true;
if (!prop_dictionary_set(pkgd, "links", array)) {
prop_object_release(pkgd);
prop_object_release(pkg_filesd);
return EINVAL;
}
}
prop_object_release(pkg_filesd);
if (!found) {
prop_object_release(pkgd);
return 0;
}
/* pkgver obj in pkgd */
if (!prop_dictionary_set_cstring(pkgd, "pkgver", pkgver)) {
prop_object_release(pkgd);
return EINVAL;
}
/* add pkgd into provided array */
if (!prop_array_add(ifd->idxfiles, pkgd)) {
prop_object_release(pkgd);
return EINVAL;
}
prop_object_release(pkgd);
return 0;
}
/*
* Create the index files cache for all packages in repository.
*/
int
repo_genindex_files(const char *pkgdir)
{
prop_dictionary_t idxdict, idxfilesd;
struct index_files_data *ifd;
char *plist, *files_plist;
int rv;
plist = xbps_pkg_index_plist(pkgdir);
if (plist == NULL)
return ENOMEM;
/* internalize repository index plist */
idxdict = prop_dictionary_internalize_from_zfile(plist);
if (idxdict == NULL) {
free(plist);
return errno;
}
ifd = malloc(sizeof(*ifd));
if (ifd == NULL) {
prop_object_release(idxdict);
free(plist);
return ENOMEM;
}
ifd->pkgdir = pkgdir;
ifd->idxfiles = prop_array_create();
printf("Creating repository's index files cache...\n");
/* iterate over index.plist packages array */
rv = xbps_callback_array_iter_in_dict(idxdict,
"packages", genindex_files_cb, ifd);
prop_object_release(idxdict);
free(plist);
if (rv != 0) {
prop_object_release(ifd->idxfiles);
free(ifd);
return rv;
}
idxfilesd = prop_dictionary_create();
/* add array into the index-files dictionary */
if (!prop_dictionary_set(idxfilesd, "packages", ifd->idxfiles)) {
prop_object_release(ifd->idxfiles);
prop_object_release(idxfilesd);
free(ifd);
return EINVAL;
}
files_plist = xbps_pkg_index_files_plist(pkgdir);
if (files_plist == NULL) {
prop_object_release(ifd->idxfiles);
prop_object_release(idxfilesd);
free(ifd);
return ENOMEM;
}
/* externalize index-files dictionary to the plist file */
if (!prop_dictionary_externalize_to_zfile(idxfilesd, files_plist)) {
free(files_plist);
prop_object_release(ifd->idxfiles);
prop_object_release(idxfilesd);
free(ifd);
return errno;
}
free(files_plist);
prop_object_release(idxfilesd);
prop_object_release(ifd->idxfiles);
free(ifd);
return 0;
}

5
bin/xbps-repo/main.c
View File

@ -1,5 +1,5 @@
/*-
* Copyright (c) 2008-2011 Juan Romero Pardines.
* Copyright (c) 2008-2012 Juan Romero Pardines.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
@ -239,6 +239,9 @@ main(int argc, char **argv)
usage(xhp);
rv = repo_genindex(argv[1]);
if (rv == 0)
rv = repo_genindex_files(argv[1]);
} else if (strcasecmp(argv[0], "sync") == 0) {
/* Syncs the pkg index for all registered remote repos */
if (argc != 1)

23
include/xbps_api.h
View File

@ -56,7 +56,7 @@
*/
#define XBPS_PKGINDEX_VERSION "1.3"
#define XBPS_API_VERSION "20120105"
#define XBPS_API_VERSION "20120115"
#define XBPS_VERSION "0.12"
/**
@ -103,6 +103,12 @@
*/
#define XBPS_PKGINDEX "index.plist"
/**
* @def XBPS_PKGINDEX_FILES
* Filename for the repository package index files property list.
*/
#define XBPS_PKGINDEX_FILES "index-files.plist"
/**
* @def XBPS_SYSCONF_PATH
* Default configuration PATH to find XBPS_CONF_PLIST.
@ -1450,12 +1456,13 @@ prop_dictionary_t
* by the \a uri argument (if necessary).
*
* @param[in] uri URI to a remote repository.
* @param[in] plistf Plist file to sync.
*
* @return -1 on error (errno is set appropiately), 0 if transfer was
* not necessary (local/remote size/mtime matched) or 1 if
* downloaded successfully.
*/
int xbps_repository_sync_pkg_index(const char *uri);
int xbps_repository_sync_pkg_index(const char *uri, const char *plistf);
/*@}*/
@ -1692,6 +1699,18 @@ char *xbps_path_from_repository_uri(prop_dictionary_t pkgd, const char *repoloc)
*/
char *xbps_pkg_index_plist(const char *uri);
/**
* Returns the full path to a repository package index files plist file,
* as specified by \a uri.
*
* @param[in] uri Repository URI.
*
* @return A pointer to a malloc(3)ed string, NULL otherwise and
* errno is set appropiately. The pointer should be free(3)d when it's
* no longer needded.
*/
char *xbps_pkg_index_files_plist(const char *uri);
/**
* Gets the name of a package string. Package strings are composed
* by a @<pkgname@>/@<version@> pair and separated by the <em>minus</em>

15
lib/repository_pool.c
View File

@ -196,9 +196,20 @@ xbps_repository_pool_sync(const struct xbps_handle *xhp)
continue;
}
/*
* Fetch repository index file.
* Fetch repository index.plist.
*/
rv = xbps_repository_sync_pkg_index(repouri);
rv = xbps_repository_sync_pkg_index(repouri, XBPS_PKGINDEX);
if (rv == -1) {
xbps_dbg_printf("[rpool] `%s' failed to fetch: %s\n",
repouri, fetchLastErrCode == 0 ?
strerror(errno) : xbps_fetch_error_string());
continue;
}
/*
* Fetch repository index-files.plist.
*/
rv = xbps_repository_sync_pkg_index(repouri,
XBPS_PKGINDEX_FILES);
if (rv == -1) {
xbps_dbg_printf("[rpool] `%s' failed to fetch: %s\n",
repouri, fetchLastErrCode == 0 ?

8
lib/repository_sync_index.c
View File

@ -86,7 +86,7 @@ xbps_get_remote_repo_string(const char *uri)
* size and/or mtime match) and 1 if downloaded successfully.
*/
int
xbps_repository_sync_pkg_index(const char *uri)
xbps_repository_sync_pkg_index(const char *uri, const char *plistf)
{
prop_dictionary_t tmpd;
struct xbps_handle *xhp;
@ -132,7 +132,7 @@ xbps_repository_sync_pkg_index(const char *uri)
/*
* Remote repository index.plist full URL.
*/
rpidx = xbps_xasprintf("%s/%s", uri, XBPS_PKGINDEX);
rpidx = xbps_xasprintf("%s/%s", uri, plistf);
if (rpidx == NULL) {
rv = -1;
goto out;
@ -141,7 +141,7 @@ xbps_repository_sync_pkg_index(const char *uri)
* Save temporary file in XBPS_META_PATH, and rename if it
* was downloaded successfully.
*/
tmp_metafile = xbps_xasprintf("%s/%s", metadir, XBPS_PKGINDEX);
tmp_metafile = xbps_xasprintf("%s/%s", metadir, plistf);
if (tmp_metafile == NULL) {
rv = -1;
goto out;
@ -200,7 +200,7 @@ xbps_repository_sync_pkg_index(const char *uri)
}
prop_object_release(tmpd);
lrepofile = xbps_xasprintf("%s/%s", lrepodir, XBPS_PKGINDEX);
lrepofile = xbps_xasprintf("%s/%s", lrepodir, plistf);
if (lrepofile == NULL) {
rv = -1;
goto out;

20
lib/util.c
View File

@ -1,5 +1,5 @@
/*-
* Copyright (c) 2008-2011 Juan Romero Pardines.
* Copyright (c) 2008-2012 Juan Romero Pardines.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
@ -201,7 +201,7 @@ xbps_pkgpattern_version(const char *pkg)
}
static char *
get_pkg_index_remote_plist(const char *uri)
get_pkg_index_remote_plist(const char *uri, const char *plistf)
{
struct xbps_handle *xhp;
char *uri_fixed, *repodir;
@ -215,11 +215,11 @@ get_pkg_index_remote_plist(const char *uri)
if (strcmp(xhp->rootdir, "/") == 0) {
repodir = xbps_xasprintf("/%s/%s/%s",
XBPS_META_PATH, uri_fixed, XBPS_PKGINDEX);
XBPS_META_PATH, uri_fixed, plistf);
} else {
repodir = xbps_xasprintf("%s/%s/%s/%s",
xhp->rootdir,
XBPS_META_PATH, uri_fixed, XBPS_PKGINDEX);
XBPS_META_PATH, uri_fixed, plistf);
}
free(uri_fixed);
return repodir;
@ -231,11 +231,21 @@ xbps_pkg_index_plist(const char *uri)
assert(uri != NULL);
if (xbps_check_is_repository_uri_remote(uri))
return get_pkg_index_remote_plist(uri);
return get_pkg_index_remote_plist(uri, XBPS_PKGINDEX);
return xbps_xasprintf("%s/%s", uri, XBPS_PKGINDEX);
}
char *
xbps_pkg_index_files_plist(const char *uri)
{
assert(uri != NULL);
if (xbps_check_is_repository_uri_remote(uri))
return get_pkg_index_remote_plist(uri, XBPS_PKGINDEX_FILES);
return xbps_xasprintf("%s/%s", uri, XBPS_PKGINDEX_FILES);
}
char *
xbps_path_from_repository_uri(prop_dictionary_t pkg_repod, const char *repoloc)
{