xbps-rindex(8): clean mode (-c) is now multithreaded.
This commit is contained in:
Juan RP
parent
33a8150a98
commit
a0443fc4ce
3
NEWS
3
NEWS
@ -1,5 +1,8 @@
|
||||
xbps-0.19 (???):
|
||||
|
||||
* xbps-rindex(8): in clean mode (-c) is now multithreaded and
|
||||
will spawn a thread per core to process a fraction of rindex.
|
||||
|
||||
* xbps-pkgdb(8): -a, --all mode is now multithreaded and will
|
||||
spawn a thread per core to process a fraction of pkgdb.
|
||||
|
||||
|
||||
@ -2,7 +2,7 @@ TOPDIR = ../..
|
||||
-include $(TOPDIR)/config.mk
|
||||
|
||||
BIN = xbps-rindex
|
||||
OBJS = main.o index.o remove-obsoletes.o common.o
|
||||
OBJS = main.o index-add.o index-clean.o remove-obsoletes.o common.o
|
||||
MAN = $(BIN).8
|
||||
|
||||
include $(TOPDIR)/mk/prog.mk
|
||||
|
||||
@ -31,8 +31,10 @@
|
||||
/* From common.c */
|
||||
int remove_pkg(const char *, const char *, const char *);
|
||||
|
||||
/* From index.c */
|
||||
/* From index-add.c */
|
||||
int index_add(struct xbps_handle *, int, char **);
|
||||
|
||||
/* From index-clean.c */
|
||||
int index_clean(struct xbps_handle *, const char *);
|
||||
|
||||
/* From index-files.c */
|
||||
|
||||
@ -36,136 +36,6 @@
|
||||
#include <xbps_api.h>
|
||||
#include "defs.h"
|
||||
|
||||
/*
|
||||
* Removes stalled pkg entries in repository's index.plist file, if any
|
||||
* binary package cannot be read (unavailable, not enough perms, etc).
|
||||
*/
|
||||
int
|
||||
index_clean(struct xbps_handle *xhp, const char *repodir)
|
||||
{
|
||||
prop_array_t result = NULL;
|
||||
prop_object_t obj;
|
||||
prop_object_iterator_t iter;
|
||||
prop_dictionary_t idx, idxfiles, pkgd;
|
||||
const char *filen, *pkgver, *arch, *keyname, *sha256;
|
||||
char *plist, *plistf;
|
||||
size_t i;
|
||||
int rv = 0;
|
||||
bool flush = false;
|
||||
|
||||
plist = xbps_pkg_index_plist(xhp, repodir);
|
||||
assert(plist);
|
||||
plistf = xbps_pkg_index_files_plist(xhp, repodir);
|
||||
assert(plistf);
|
||||
|
||||
idx = prop_dictionary_internalize_from_zfile(plist);
|
||||
if (idx == NULL) {
|
||||
if (errno != ENOENT) {
|
||||
fprintf(stderr, "index: cannot read `%s': %s\n",
|
||||
plist, strerror(errno));
|
||||
free(plist);
|
||||
return -1;
|
||||
} else {
|
||||
free(plist);
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
idxfiles = prop_dictionary_internalize_from_zfile(plistf);
|
||||
if (idxfiles == NULL) {
|
||||
if (errno != ENOENT) {
|
||||
fprintf(stderr, "index: cannot read `%s': %s\n",
|
||||
plistf, strerror(errno));
|
||||
rv = -1;
|
||||
goto out;
|
||||
} else {
|
||||
goto out;
|
||||
}
|
||||
}
|
||||
if (chdir(repodir) == -1) {
|
||||
fprintf(stderr, "index: cannot chdir to %s: %s\n",
|
||||
repodir, strerror(errno));
|
||||
rv = -1;
|
||||
goto out;
|
||||
}
|
||||
printf("Cleaning `%s' index, please wait...\n", repodir);
|
||||
|
||||
iter = prop_dictionary_iterator(idx);
|
||||
while ((obj = prop_object_iterator_next(iter))) {
|
||||
pkgd = prop_dictionary_get_keysym(idx, obj);
|
||||
prop_dictionary_get_cstring_nocopy(pkgd, "filename", &filen);
|
||||
if (access(filen, R_OK) == -1) {
|
||||
/*
|
||||
* File cannot be read, might be permissions,
|
||||
* broken or simply unexistent; either way, remove it.
|
||||
*/
|
||||
prop_dictionary_get_cstring_nocopy(pkgd,
|
||||
"pkgver", &pkgver);
|
||||
prop_dictionary_get_cstring_nocopy(pkgd,
|
||||
"architecture", &arch);
|
||||
printf("index: removed obsolete entry `%s' (%s)\n",
|
||||
pkgver, arch);
|
||||
if (result == NULL)
|
||||
result = prop_array_create();
|
||||
|
||||
keyname = prop_dictionary_keysym_cstring_nocopy(obj);
|
||||
prop_array_add_cstring_nocopy(result, keyname);
|
||||
flush = true;
|
||||
continue;
|
||||
}
|
||||
/*
|
||||
* File can be read; check its hash.
|
||||
*/
|
||||
prop_dictionary_get_cstring_nocopy(pkgd,
|
||||
"filename-sha256", &sha256);
|
||||
rv = xbps_file_hash_check(filen, sha256);
|
||||
if (rv != 0) {
|
||||
prop_dictionary_get_cstring_nocopy(pkgd,
|
||||
"pkgver", &pkgver);
|
||||
prop_dictionary_get_cstring_nocopy(pkgd,
|
||||
"architecture", &arch);
|
||||
printf("index: removed entry `%s' due to "
|
||||
"unmatched hash (%s)\n", pkgver, arch);
|
||||
if (result == NULL)
|
||||
result = prop_array_create();
|
||||
|
||||
keyname = prop_dictionary_keysym_cstring_nocopy(obj);
|
||||
prop_array_add_cstring_nocopy(result, keyname);
|
||||
flush = true;
|
||||
}
|
||||
}
|
||||
prop_object_iterator_release(iter);
|
||||
|
||||
if (flush) {
|
||||
for (i = 0; i < prop_array_count(result); i++) {
|
||||
prop_array_get_cstring_nocopy(result, i, &keyname);
|
||||
prop_dictionary_remove(idx, keyname);
|
||||
prop_dictionary_remove(idxfiles, keyname);
|
||||
}
|
||||
prop_object_release(result);
|
||||
if (!prop_dictionary_externalize_to_zfile(idx, plist) &&
|
||||
!prop_dictionary_externalize_to_zfile(idxfiles, plistf)) {
|
||||
fprintf(stderr, "index: failed to externalize %s: %s\n",
|
||||
plist, strerror(errno));
|
||||
goto out;
|
||||
}
|
||||
}
|
||||
printf("index: %u packages registered.\n",
|
||||
prop_dictionary_count(idx));
|
||||
printf("index-files: %u packages registered.\n",
|
||||
prop_dictionary_count(idxfiles));
|
||||
out:
|
||||
if (plist)
|
||||
free(plist);
|
||||
if (plistf)
|
||||
free(plistf);
|
||||
if (idx)
|
||||
prop_object_release(idx);
|
||||
if (idxfiles)
|
||||
prop_object_release(idxfiles);
|
||||
|
||||
return rv;
|
||||
}
|
||||
|
||||
/*
|
||||
* Adds a binary package into the index and removes old binary package
|
||||
* and entry when it's necessary.
|
||||
205
bin/xbps-rindex/index-clean.c
Normal file
205
bin/xbps-rindex/index-clean.c
Normal file
@ -0,0 +1,205 @@
|
||||
/*-
|
||||
* Copyright (c) 2012 Juan Romero Pardines.
|
||||
* All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions
|
||||
* are met:
|
||||
* 1. Redistributions of source code must retain the above copyright
|
||||
* notice, this list of conditions and the following disclaimer.
|
||||
* 2. Redistributions in binary form must reproduce the above copyright
|
||||
* notice, this list of conditions and the following disclaimer in the
|
||||
* documentation and/or other materials provided with the distribution.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
|
||||
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
|
||||
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
|
||||
* IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
|
||||
* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
|
||||
* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
|
||||
* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*/
|
||||
|
||||
#include <sys/stat.h>
|
||||
#include <stdio.h>
|
||||
#include <stdbool.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <errno.h>
|
||||
#include <dirent.h>
|
||||
#include <libgen.h>
|
||||
#include <assert.h>
|
||||
#include <pthread.h>
|
||||
|
||||
#include <xbps_api.h>
|
||||
#include "defs.h"
|
||||
|
||||
struct thread_data {
|
||||
pthread_t thread;
|
||||
prop_dictionary_t idx;
|
||||
prop_array_t result;
|
||||
struct xbps_handle *xhp;
|
||||
unsigned int start;
|
||||
unsigned int end;
|
||||
int thread_num;
|
||||
};
|
||||
|
||||
static void *
|
||||
cleaner_thread(void *arg)
|
||||
{
|
||||
prop_object_t obj;
|
||||
prop_dictionary_t pkgd;
|
||||
prop_array_t array;
|
||||
struct thread_data *thd = arg;
|
||||
const char *pkgver, *filen, *sha256;
|
||||
unsigned int i;
|
||||
|
||||
/* process pkgs from start until end */
|
||||
array = prop_dictionary_all_keys(thd->idx);
|
||||
|
||||
for (i = thd->start; i < thd->end; i++) {
|
||||
obj = prop_array_get(array, i);
|
||||
pkgd = prop_dictionary_get_keysym(thd->idx, obj);
|
||||
prop_dictionary_get_cstring_nocopy(pkgd, "filename", &filen);
|
||||
prop_dictionary_get_cstring_nocopy(pkgd, "pkgver", &pkgver);
|
||||
xbps_dbg_printf(thd->xhp, "thread[%d] checking %s\n",
|
||||
thd->thread_num, pkgver);
|
||||
if (access(filen, R_OK) == -1) {
|
||||
/*
|
||||
* File cannot be read, might be permissions,
|
||||
* broken or simply unexistent; either way, remove it.
|
||||
*/
|
||||
prop_array_add_cstring_nocopy(thd->result, pkgver);
|
||||
continue;
|
||||
}
|
||||
/*
|
||||
* File can be read; check its hash.
|
||||
*/
|
||||
prop_dictionary_get_cstring_nocopy(pkgd,
|
||||
"filename-sha256", &sha256);
|
||||
if (xbps_file_hash_check(filen, sha256) != 0)
|
||||
prop_array_add_cstring_nocopy(thd->result, pkgver);
|
||||
}
|
||||
prop_object_release(array);
|
||||
|
||||
return NULL;
|
||||
}
|
||||
|
||||
/*
|
||||
* Removes stalled pkg entries in repository's index.plist file, if any
|
||||
* binary package cannot be read (unavailable, not enough perms, etc).
|
||||
*/
|
||||
int
|
||||
index_clean(struct xbps_handle *xhp, const char *repodir)
|
||||
{
|
||||
struct thread_data *thd;
|
||||
prop_dictionary_t idx, idxfiles;
|
||||
const char *keyname;
|
||||
char *plist, *plistf, *pkgname;
|
||||
size_t x, pkgcount, slicecount;
|
||||
int i, maxthreads, rv = 0;
|
||||
bool flush = false;
|
||||
|
||||
plist = xbps_pkg_index_plist(xhp, repodir);
|
||||
assert(plist);
|
||||
plistf = xbps_pkg_index_files_plist(xhp, repodir);
|
||||
assert(plistf);
|
||||
|
||||
idx = prop_dictionary_internalize_from_zfile(plist);
|
||||
if (idx == NULL) {
|
||||
if (errno != ENOENT) {
|
||||
fprintf(stderr, "index: cannot read `%s': %s\n",
|
||||
plist, strerror(errno));
|
||||
free(plist);
|
||||
return -1;
|
||||
} else {
|
||||
free(plist);
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
idxfiles = prop_dictionary_internalize_from_zfile(plistf);
|
||||
if (idxfiles == NULL) {
|
||||
if (errno != ENOENT) {
|
||||
fprintf(stderr, "index: cannot read `%s': %s\n",
|
||||
plistf, strerror(errno));
|
||||
rv = -1;
|
||||
goto out;
|
||||
} else {
|
||||
goto out;
|
||||
}
|
||||
}
|
||||
if (chdir(repodir) == -1) {
|
||||
fprintf(stderr, "index: cannot chdir to %s: %s\n",
|
||||
repodir, strerror(errno));
|
||||
rv = -1;
|
||||
goto out;
|
||||
}
|
||||
printf("Cleaning `%s' index, please wait...\n", repodir);
|
||||
|
||||
maxthreads = (int)sysconf(_SC_NPROCESSORS_ONLN);
|
||||
thd = calloc(maxthreads, sizeof(*thd));
|
||||
|
||||
slicecount = prop_dictionary_count(idx) / maxthreads;
|
||||
pkgcount = 0;
|
||||
|
||||
for (i = 0; i < maxthreads; i++) {
|
||||
thd[i].thread_num = i;
|
||||
thd[i].idx = idx;
|
||||
thd[i].result = prop_array_create();
|
||||
thd[i].xhp = xhp;
|
||||
thd[i].start = pkgcount;
|
||||
if (i + 1 >= maxthreads)
|
||||
thd[i].end = prop_dictionary_count(idx);
|
||||
else
|
||||
thd[i].end = pkgcount + slicecount;
|
||||
pthread_create(&thd[i].thread, NULL, cleaner_thread, &thd[i]);
|
||||
pkgcount += slicecount;
|
||||
}
|
||||
/* wait for all threads */
|
||||
for (i = 0; i < maxthreads; i++)
|
||||
pthread_join(thd[i].thread, NULL);
|
||||
|
||||
for (i = 0; i < maxthreads; i++) {
|
||||
if (!prop_array_count(thd[i].result))
|
||||
continue;
|
||||
for (x = 0; x < prop_array_count(thd[i].result); x++) {
|
||||
prop_array_get_cstring_nocopy(thd[i].result,
|
||||
x, &keyname);
|
||||
printf("index: removed entry %s\n", keyname);
|
||||
pkgname = xbps_pkg_name(keyname);
|
||||
prop_dictionary_remove(idx, pkgname);
|
||||
prop_dictionary_remove(idxfiles, pkgname);
|
||||
free(pkgname);
|
||||
flush = true;
|
||||
}
|
||||
prop_object_release(thd[i].result);
|
||||
}
|
||||
free(thd);
|
||||
if (!flush)
|
||||
goto out;
|
||||
|
||||
if (!prop_dictionary_externalize_to_zfile(idx, plist) &&
|
||||
!prop_dictionary_externalize_to_zfile(idxfiles, plistf))
|
||||
fprintf(stderr, "index: failed to externalize %s: %s\n",
|
||||
plist, strerror(errno));
|
||||
|
||||
out:
|
||||
printf("index: %u packages registered.\n",
|
||||
prop_dictionary_count(idx));
|
||||
printf("index-files: %u packages registered.\n",
|
||||
prop_dictionary_count(idxfiles));
|
||||
|
||||
if (plist)
|
||||
free(plist);
|
||||
if (plistf)
|
||||
free(plistf);
|
||||
if (idx)
|
||||
prop_object_release(idx);
|
||||
if (idxfiles)
|
||||
prop_object_release(idxfiles);
|
||||
|
||||
return rv;
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user