From a0443fc4ce6529de3f427df6f5fafe66113fffb9 Mon Sep 17 00:00:00 2001 From: Juan RP Date: Fri, 14 Dec 2012 04:59:47 +0100 Subject: [PATCH] xbps-rindex(8): clean mode (-c) is now multithreaded. --- NEWS | 3 + bin/xbps-rindex/Makefile | 2 +- bin/xbps-rindex/defs.h | 4 +- bin/xbps-rindex/{index.c => index-add.c} | 130 -------------- bin/xbps-rindex/index-clean.c | 205 +++++++++++++++++++++++ 5 files changed, 212 insertions(+), 132 deletions(-) rename bin/xbps-rindex/{index.c => index-add.c} (74%) create mode 100644 bin/xbps-rindex/index-clean.c diff --git a/NEWS b/NEWS index e68afac9..98e16015 100644 --- a/NEWS +++ b/NEWS @@ -1,5 +1,8 @@ xbps-0.19 (???): + * xbps-rindex(8): in clean mode (-c) is now multithreaded and + will spawn a thread per core to process a fraction of rindex. + * xbps-pkgdb(8): -a, --all mode is now multithreaded and will spawn a thread per core to process a fraction of pkgdb. diff --git a/bin/xbps-rindex/Makefile b/bin/xbps-rindex/Makefile index 3500ffb7..f41a4692 100644 --- a/bin/xbps-rindex/Makefile +++ b/bin/xbps-rindex/Makefile @@ -2,7 +2,7 @@ TOPDIR = ../.. -include $(TOPDIR)/config.mk BIN = xbps-rindex -OBJS = main.o index.o remove-obsoletes.o common.o +OBJS = main.o index-add.o index-clean.o remove-obsoletes.o common.o MAN = $(BIN).8 include $(TOPDIR)/mk/prog.mk diff --git a/bin/xbps-rindex/defs.h b/bin/xbps-rindex/defs.h index 9b863eb6..ad09e8a9 100644 --- a/bin/xbps-rindex/defs.h +++ b/bin/xbps-rindex/defs.h @@ -31,8 +31,10 @@ /* From common.c */ int remove_pkg(const char *, const char *, const char *); -/* From index.c */ +/* From index-add.c */ int index_add(struct xbps_handle *, int, char **); + +/* From index-clean.c */ int index_clean(struct xbps_handle *, const char *); /* From index-files.c */ diff --git a/bin/xbps-rindex/index.c b/bin/xbps-rindex/index-add.c similarity index 74% rename from bin/xbps-rindex/index.c rename to bin/xbps-rindex/index-add.c index cb7a7b67..0710d5e7 100644 --- a/bin/xbps-rindex/index.c +++ b/bin/xbps-rindex/index-add.c @@ -36,136 +36,6 @@ #include #include "defs.h" -/* - * Removes stalled pkg entries in repository's index.plist file, if any - * binary package cannot be read (unavailable, not enough perms, etc). - */ -int -index_clean(struct xbps_handle *xhp, const char *repodir) -{ - prop_array_t result = NULL; - prop_object_t obj; - prop_object_iterator_t iter; - prop_dictionary_t idx, idxfiles, pkgd; - const char *filen, *pkgver, *arch, *keyname, *sha256; - char *plist, *plistf; - size_t i; - int rv = 0; - bool flush = false; - - plist = xbps_pkg_index_plist(xhp, repodir); - assert(plist); - plistf = xbps_pkg_index_files_plist(xhp, repodir); - assert(plistf); - - idx = prop_dictionary_internalize_from_zfile(plist); - if (idx == NULL) { - if (errno != ENOENT) { - fprintf(stderr, "index: cannot read `%s': %s\n", - plist, strerror(errno)); - free(plist); - return -1; - } else { - free(plist); - return 0; - } - } - idxfiles = prop_dictionary_internalize_from_zfile(plistf); - if (idxfiles == NULL) { - if (errno != ENOENT) { - fprintf(stderr, "index: cannot read `%s': %s\n", - plistf, strerror(errno)); - rv = -1; - goto out; - } else { - goto out; - } - } - if (chdir(repodir) == -1) { - fprintf(stderr, "index: cannot chdir to %s: %s\n", - repodir, strerror(errno)); - rv = -1; - goto out; - } - printf("Cleaning `%s' index, please wait...\n", repodir); - - iter = prop_dictionary_iterator(idx); - while ((obj = prop_object_iterator_next(iter))) { - pkgd = prop_dictionary_get_keysym(idx, obj); - prop_dictionary_get_cstring_nocopy(pkgd, "filename", &filen); - if (access(filen, R_OK) == -1) { - /* - * File cannot be read, might be permissions, - * broken or simply unexistent; either way, remove it. - */ - prop_dictionary_get_cstring_nocopy(pkgd, - "pkgver", &pkgver); - prop_dictionary_get_cstring_nocopy(pkgd, - "architecture", &arch); - printf("index: removed obsolete entry `%s' (%s)\n", - pkgver, arch); - if (result == NULL) - result = prop_array_create(); - - keyname = prop_dictionary_keysym_cstring_nocopy(obj); - prop_array_add_cstring_nocopy(result, keyname); - flush = true; - continue; - } - /* - * File can be read; check its hash. - */ - prop_dictionary_get_cstring_nocopy(pkgd, - "filename-sha256", &sha256); - rv = xbps_file_hash_check(filen, sha256); - if (rv != 0) { - prop_dictionary_get_cstring_nocopy(pkgd, - "pkgver", &pkgver); - prop_dictionary_get_cstring_nocopy(pkgd, - "architecture", &arch); - printf("index: removed entry `%s' due to " - "unmatched hash (%s)\n", pkgver, arch); - if (result == NULL) - result = prop_array_create(); - - keyname = prop_dictionary_keysym_cstring_nocopy(obj); - prop_array_add_cstring_nocopy(result, keyname); - flush = true; - } - } - prop_object_iterator_release(iter); - - if (flush) { - for (i = 0; i < prop_array_count(result); i++) { - prop_array_get_cstring_nocopy(result, i, &keyname); - prop_dictionary_remove(idx, keyname); - prop_dictionary_remove(idxfiles, keyname); - } - prop_object_release(result); - if (!prop_dictionary_externalize_to_zfile(idx, plist) && - !prop_dictionary_externalize_to_zfile(idxfiles, plistf)) { - fprintf(stderr, "index: failed to externalize %s: %s\n", - plist, strerror(errno)); - goto out; - } - } - printf("index: %u packages registered.\n", - prop_dictionary_count(idx)); - printf("index-files: %u packages registered.\n", - prop_dictionary_count(idxfiles)); -out: - if (plist) - free(plist); - if (plistf) - free(plistf); - if (idx) - prop_object_release(idx); - if (idxfiles) - prop_object_release(idxfiles); - - return rv; -} - /* * Adds a binary package into the index and removes old binary package * and entry when it's necessary. diff --git a/bin/xbps-rindex/index-clean.c b/bin/xbps-rindex/index-clean.c new file mode 100644 index 00000000..b061688f --- /dev/null +++ b/bin/xbps-rindex/index-clean.c @@ -0,0 +1,205 @@ +/*- + * Copyright (c) 2012 Juan Romero Pardines. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include "defs.h" + +struct thread_data { + pthread_t thread; + prop_dictionary_t idx; + prop_array_t result; + struct xbps_handle *xhp; + unsigned int start; + unsigned int end; + int thread_num; +}; + +static void * +cleaner_thread(void *arg) +{ + prop_object_t obj; + prop_dictionary_t pkgd; + prop_array_t array; + struct thread_data *thd = arg; + const char *pkgver, *filen, *sha256; + unsigned int i; + + /* process pkgs from start until end */ + array = prop_dictionary_all_keys(thd->idx); + + for (i = thd->start; i < thd->end; i++) { + obj = prop_array_get(array, i); + pkgd = prop_dictionary_get_keysym(thd->idx, obj); + prop_dictionary_get_cstring_nocopy(pkgd, "filename", &filen); + prop_dictionary_get_cstring_nocopy(pkgd, "pkgver", &pkgver); + xbps_dbg_printf(thd->xhp, "thread[%d] checking %s\n", + thd->thread_num, pkgver); + if (access(filen, R_OK) == -1) { + /* + * File cannot be read, might be permissions, + * broken or simply unexistent; either way, remove it. + */ + prop_array_add_cstring_nocopy(thd->result, pkgver); + continue; + } + /* + * File can be read; check its hash. + */ + prop_dictionary_get_cstring_nocopy(pkgd, + "filename-sha256", &sha256); + if (xbps_file_hash_check(filen, sha256) != 0) + prop_array_add_cstring_nocopy(thd->result, pkgver); + } + prop_object_release(array); + + return NULL; +} + +/* + * Removes stalled pkg entries in repository's index.plist file, if any + * binary package cannot be read (unavailable, not enough perms, etc). + */ +int +index_clean(struct xbps_handle *xhp, const char *repodir) +{ + struct thread_data *thd; + prop_dictionary_t idx, idxfiles; + const char *keyname; + char *plist, *plistf, *pkgname; + size_t x, pkgcount, slicecount; + int i, maxthreads, rv = 0; + bool flush = false; + + plist = xbps_pkg_index_plist(xhp, repodir); + assert(plist); + plistf = xbps_pkg_index_files_plist(xhp, repodir); + assert(plistf); + + idx = prop_dictionary_internalize_from_zfile(plist); + if (idx == NULL) { + if (errno != ENOENT) { + fprintf(stderr, "index: cannot read `%s': %s\n", + plist, strerror(errno)); + free(plist); + return -1; + } else { + free(plist); + return 0; + } + } + idxfiles = prop_dictionary_internalize_from_zfile(plistf); + if (idxfiles == NULL) { + if (errno != ENOENT) { + fprintf(stderr, "index: cannot read `%s': %s\n", + plistf, strerror(errno)); + rv = -1; + goto out; + } else { + goto out; + } + } + if (chdir(repodir) == -1) { + fprintf(stderr, "index: cannot chdir to %s: %s\n", + repodir, strerror(errno)); + rv = -1; + goto out; + } + printf("Cleaning `%s' index, please wait...\n", repodir); + + maxthreads = (int)sysconf(_SC_NPROCESSORS_ONLN); + thd = calloc(maxthreads, sizeof(*thd)); + + slicecount = prop_dictionary_count(idx) / maxthreads; + pkgcount = 0; + + for (i = 0; i < maxthreads; i++) { + thd[i].thread_num = i; + thd[i].idx = idx; + thd[i].result = prop_array_create(); + thd[i].xhp = xhp; + thd[i].start = pkgcount; + if (i + 1 >= maxthreads) + thd[i].end = prop_dictionary_count(idx); + else + thd[i].end = pkgcount + slicecount; + pthread_create(&thd[i].thread, NULL, cleaner_thread, &thd[i]); + pkgcount += slicecount; + } + /* wait for all threads */ + for (i = 0; i < maxthreads; i++) + pthread_join(thd[i].thread, NULL); + + for (i = 0; i < maxthreads; i++) { + if (!prop_array_count(thd[i].result)) + continue; + for (x = 0; x < prop_array_count(thd[i].result); x++) { + prop_array_get_cstring_nocopy(thd[i].result, + x, &keyname); + printf("index: removed entry %s\n", keyname); + pkgname = xbps_pkg_name(keyname); + prop_dictionary_remove(idx, pkgname); + prop_dictionary_remove(idxfiles, pkgname); + free(pkgname); + flush = true; + } + prop_object_release(thd[i].result); + } + free(thd); + if (!flush) + goto out; + + if (!prop_dictionary_externalize_to_zfile(idx, plist) && + !prop_dictionary_externalize_to_zfile(idxfiles, plistf)) + fprintf(stderr, "index: failed to externalize %s: %s\n", + plist, strerror(errno)); + +out: + printf("index: %u packages registered.\n", + prop_dictionary_count(idx)); + printf("index-files: %u packages registered.\n", + prop_dictionary_count(idxfiles)); + + if (plist) + free(plist); + if (plistf) + free(plistf); + if (idx) + prop_object_release(idx); + if (idxfiles) + prop_object_release(idxfiles); + + return rv; +}