emo/xbps
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

xbps-rindex: added --compression option and set zstd by default.

Browse Source 
Multiple compression formats are now supported for repodata,
none, gzip, bzip2, xz, lz4 and zstd.
This commit is contained in:
Juan RP 2019-06-17 07:24:05 +02:00
parent 7f75910054
commit 6c1759862e
No known key found for this signature in database
GPG Key ID: AF19F6CB482F9368
7 changed files with 63 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
bin/xbps-rindex/defs.h
View File

@ -67,21 +67,21 @@
#define _XBPS_RINDEX "xbps-rindex" #define _XBPS_RINDEX "xbps-rindex"
/* From index-add.c */ /* From index-add.c */
int index_add(struct xbps_handle *, int, int, char **, bool); int index_add(struct xbps_handle *, int, int, char **, bool, const char *);
/* From index-clean.c */ /* From index-clean.c */
int index_clean(struct xbps_handle *, const char *, bool); int index_clean(struct xbps_handle *, const char *, bool, const char *);
/* From remove-obsoletes.c */ /* From remove-obsoletes.c */
int remove_obsoletes(struct xbps_handle *, const char *); int remove_obsoletes(struct xbps_handle *, const char *);
/* From sign.c */ /* From sign.c */
int sign_repo(struct xbps_handle *, const char *, const char *, int sign_repo(struct xbps_handle *, const char *, const char *,
const char *); const char *, const char *);
int sign_pkgs(struct xbps_handle *, int, int, char **, const char *, bool); int sign_pkgs(struct xbps_handle *, int, int, char **, const char *, bool);
/* From repoflush.c */ /* From repoflush.c */
bool repodata_flush(struct xbps_handle *, const char *, const char *, bool repodata_flush(struct xbps_handle *, const char *, const char *,
xbps_dictionary_t, xbps_dictionary_t); xbps_dictionary_t, xbps_dictionary_t, const char *);
#endif /* !_XBPS_RINDEX_DEFS_H_ */ #endif /* !_XBPS_RINDEX_DEFS_H_ */

12
bin/xbps-rindex/index-add.c
View File

@ -56,7 +56,9 @@ set_build_date(const xbps_dictionary_t pkgd, time_t timestamp)
static bool static bool
repodata_commit(struct xbps_handle *xhp, const char *repodir, repodata_commit(struct xbps_handle *xhp, const char *repodir,
xbps_dictionary_t idx, xbps_dictionary_t meta, xbps_dictionary_t stage) { xbps_dictionary_t idx, xbps_dictionary_t meta, xbps_dictionary_t stage,
const char *compression)
{
xbps_object_iterator_t iter; xbps_object_iterator_t iter;
xbps_object_t keysym; xbps_object_t keysym;
int rv; int rv;
@ -187,7 +189,7 @@ repodata_commit(struct xbps_handle *xhp, const char *repodir,
printf("stage: added `%s' (%s)\n", pkgver, arch); printf("stage: added `%s' (%s)\n", pkgver, arch);
} }
xbps_object_iterator_release(iter); xbps_object_iterator_release(iter);
rv = repodata_flush(xhp, repodir, "stagedata", stage, NULL); rv = repodata_flush(xhp, repodir, "stagedata", stage, NULL, compression);
} }
else { else {
char *stagefile; char *stagefile;
@ -205,7 +207,7 @@ repodata_commit(struct xbps_handle *xhp, const char *repodir,
stagefile = xbps_repo_path_with_name(xhp, repodir, "stagedata"); stagefile = xbps_repo_path_with_name(xhp, repodir, "stagedata");
unlink(stagefile); unlink(stagefile);
free(stagefile); free(stagefile);
rv = repodata_flush(xhp, repodir, "repodata", idx, meta); rv = repodata_flush(xhp, repodir, "repodata", idx, meta, compression);
} }
xbps_object_release(usedshlibs); xbps_object_release(usedshlibs);
xbps_object_release(oldshlibs); xbps_object_release(oldshlibs);
@ -213,7 +215,7 @@ repodata_commit(struct xbps_handle *xhp, const char *repodir,
} }
int int
index_add(struct xbps_handle *xhp, int args, int argmax, char **argv, bool force) index_add(struct xbps_handle *xhp, int args, int argmax, char **argv, bool force, const char *compression)
{ {
xbps_dictionary_t idx, idxmeta, idxstage, binpkgd, curpkgd; xbps_dictionary_t idx, idxmeta, idxstage, binpkgd, curpkgd;
struct xbps_repo *repo = NULL, *stage = NULL; struct xbps_repo *repo = NULL, *stage = NULL;
@ -405,7 +407,7 @@ index_add(struct xbps_handle *xhp, int args, int argmax, char **argv, bool force
/* /*
* Generate repository data files. * Generate repository data files.
*/ */
if (!repodata_commit(xhp, repodir, idx, idxmeta, idxstage)) { if (!repodata_commit(xhp, repodir, idx, idxmeta, idxstage, compression)) {
fprintf(stderr, "%s: failed to write repodata: %s\n", fprintf(stderr, "%s: failed to write repodata: %s\n",
_XBPS_RINDEX, strerror(errno)); _XBPS_RINDEX, strerror(errno));
goto out; goto out;

21
bin/xbps-rindex/index-clean.c
View File

@ -1,5 +1,5 @@
/*- /*-
* Copyright (c) 2012-2015 Juan Romero Pardines. * Copyright (c) 2012-2019 Juan Romero Pardines.
* All rights reserved. * All rights reserved.
* *
* Redistribution and use in source and binary forms, with or without * Redistribution and use in source and binary forms, with or without
@ -44,6 +44,7 @@ struct CleanerCbInfo {
const char *repourl; const char *repourl;
bool hashcheck; bool hashcheck;
}; };
static int static int
idx_cleaner_cb(struct xbps_handle *xhp, idx_cleaner_cb(struct xbps_handle *xhp,
xbps_object_t obj, xbps_object_t obj,
@ -94,7 +95,8 @@ out:
static int static int
cleanup_repo(struct xbps_handle *xhp, const char *repodir, struct xbps_repo *repo, cleanup_repo(struct xbps_handle *xhp, const char *repodir, struct xbps_repo *repo,
const char *reponame, bool hashcheck) { const char *reponame, bool hashcheck, const char *compression)
{
int rv = 0; int rv = 0;
xbps_array_t allkeys; xbps_array_t allkeys;
struct CleanerCbInfo info = { struct CleanerCbInfo info = {
@ -109,20 +111,20 @@ cleanup_repo(struct xbps_handle *xhp, const char *repodir, struct xbps_repo *rep
(void)xbps_array_foreach_cb_multi(xhp, allkeys, repo->idx, idx_cleaner_cb, &info); (void)xbps_array_foreach_cb_multi(xhp, allkeys, repo->idx, idx_cleaner_cb, &info);
xbps_object_release(allkeys); xbps_object_release(allkeys);
if(strcmp("stagedata", reponame) == 0 && xbps_dictionary_count(dest) == 0) { if (strcmp("stagedata", reponame) == 0 && xbps_dictionary_count(dest) == 0) {
char *stagefile = xbps_repo_path_with_name(xhp, repodir, "stagedata"); char *stagefile = xbps_repo_path_with_name(xhp, repodir, "stagedata");
unlink(stagefile); unlink(stagefile);
free(stagefile); free(stagefile);
} }
if (!xbps_dictionary_equals(dest, repo->idx)) { if (!xbps_dictionary_equals(dest, repo->idx)) {
if (!repodata_flush(xhp, repodir, reponame, dest, repo->idxmeta)) { if (!repodata_flush(xhp, repodir, reponame, dest, repo->idxmeta, compression)) {
rv = errno; rv = errno;
fprintf(stderr, "failed to write repodata: %s\n", fprintf(stderr, "failed to write repodata: %s\n",
strerror(errno)); strerror(errno));
return rv; return rv;
} }
} }
if(strcmp("stagedata", reponame) == 0) if (strcmp("stagedata", reponame) == 0)
printf("stage: %u packages registered.\n", xbps_dictionary_count(dest)); printf("stage: %u packages registered.\n", xbps_dictionary_count(dest));
else else
printf("index: %u packages registered.\n", xbps_dictionary_count(dest)); printf("index: %u packages registered.\n", xbps_dictionary_count(dest));
@ -134,7 +136,7 @@ cleanup_repo(struct xbps_handle *xhp, const char *repodir, struct xbps_repo *rep
* binary package cannot be read (unavailable, not enough perms, etc). * binary package cannot be read (unavailable, not enough perms, etc).
*/ */
int int
index_clean(struct xbps_handle *xhp, const char *repodir, const bool hashcheck) index_clean(struct xbps_handle *xhp, const char *repodir, const bool hashcheck, const char *compression)
{ {
struct xbps_repo *repo, *stage; struct xbps_repo *repo, *stage;
char *rlockfname = NULL; char *rlockfname = NULL;
@ -166,10 +168,11 @@ index_clean(struct xbps_handle *xhp, const char *repodir, const bool hashcheck)
} }
printf("Cleaning `%s' index, please wait...\n", repodir); printf("Cleaning `%s' index, please wait...\n", repodir);
if((rv = cleanup_repo(xhp, repodir, repo, "repodata", hashcheck))) if ((rv = cleanup_repo(xhp, repodir, repo, "repodata", hashcheck, compression))) {
goto out; goto out;
if(stage) { }
cleanup_repo(xhp, repodir, stage, "stagedata", hashcheck); if (stage) {
cleanup_repo(xhp, repodir, stage, "stagedata", hashcheck, compression);
} }
out: out:

14
bin/xbps-rindex/main.c
View File

@ -1,5 +1,5 @@
/*- /*-
* Copyright (c) 2012-2015 Juan Romero Pardines. * Copyright (c) 2012-2019 Juan Romero Pardines.
* All rights reserved. * All rights reserved.
* *
* Redistribution and use in source and binary forms, with or without * Redistribution and use in source and binary forms, with or without
@ -43,6 +43,7 @@ usage(bool fail)
" -v --verbose Verbose messages\n" " -v --verbose Verbose messages\n"
" -V --version Show XBPS version\n" " -V --version Show XBPS version\n"
" -C --hashcheck Consider file hashes for cleaning up packages\n" " -C --hashcheck Consider file hashes for cleaning up packages\n"
" --compression <fmt> Compression format: none, gzip, bzip2, lz4 (default), zstd, xz.\n"
" --privkey <key> Path to the private key for signing\n" " --privkey <key> Path to the private key for signing\n"
" --signedby <string> Signature details, i.e \"name <email>\"\n\n" " --signedby <string> Signature details, i.e \"name <email>\"\n\n"
"MODE\n" "MODE\n"
@ -72,9 +73,11 @@ main(int argc, char **argv)
{ "sign", no_argument, NULL, 's'}, { "sign", no_argument, NULL, 's'},
{ "sign-pkg", no_argument, NULL, 'S'}, { "sign-pkg", no_argument, NULL, 'S'},
{ "hashcheck", no_argument, NULL, 'C' }, { "hashcheck", no_argument, NULL, 'C' },
{ "compression", required_argument, NULL, 2},
{ NULL, 0, NULL, 0 } { NULL, 0, NULL, 0 }
}; };
struct xbps_handle xh; struct xbps_handle xh;
const char *compression = NULL;
const char *privkey = NULL, *signedby = NULL; const char *privkey = NULL, *signedby = NULL;
int rv, c, flags = 0; int rv, c, flags = 0;
bool add_mode, clean_mode, rm_mode, sign_mode, sign_pkg_mode, force, bool add_mode, clean_mode, rm_mode, sign_mode, sign_pkg_mode, force,
@ -91,6 +94,9 @@ main(int argc, char **argv)
case 1: case 1:
signedby = optarg; signedby = optarg;
break; break;
case 2:
compression = optarg;
break;
case 'a': case 'a':
add_mode = true; add_mode = true;
break; break;
@ -149,13 +155,13 @@ main(int argc, char **argv)
} }
if (add_mode) if (add_mode)
rv = index_add(&xh, optind, argc, argv, force); rv = index_add(&xh, optind, argc, argv, force, compression);
else if (clean_mode) else if (clean_mode)
rv = index_clean(&xh, argv[optind], hashcheck); rv = index_clean(&xh, argv[optind], hashcheck, compression);
else if (rm_mode) else if (rm_mode)
rv = remove_obsoletes(&xh, argv[optind]); rv = remove_obsoletes(&xh, argv[optind]);
else if (sign_mode) else if (sign_mode)
rv = sign_repo(&xh, argv[optind], privkey, signedby); rv = sign_repo(&xh, argv[optind], privkey, signedby, compression);
else if (sign_pkg_mode) else if (sign_pkg_mode)
rv = sign_pkgs(&xh, optind, argc, argv, privkey, force); rv = sign_pkgs(&xh, optind, argc, argv, privkey, force);

26
bin/xbps-rindex/repoflush.c
View File

@ -39,7 +39,8 @@
bool bool
repodata_flush(struct xbps_handle *xhp, const char *repodir, repodata_flush(struct xbps_handle *xhp, const char *repodir,
const char *reponame, xbps_dictionary_t idx, xbps_dictionary_t meta) const char *reponame, xbps_dictionary_t idx, xbps_dictionary_t meta,
const char *compression)
{ {
struct archive *ar; struct archive *ar;
char *repofile, *tname, *buf; char *repofile, *tname, *buf;
@ -57,9 +58,28 @@ repodata_flush(struct xbps_handle *xhp, const char *repodir,
/* Create and write our repository archive */ /* Create and write our repository archive */
ar = archive_write_new(); ar = archive_write_new();
assert(ar); assert(ar);
archive_write_set_compression_gzip(ar);
if (compression == NULL || strcmp(compression, "zstd") == 0) {
archive_write_add_filter_zstd(ar);
archive_write_set_options(ar, "compression-level=19");
} else if (strcmp(compression, "gzip") == 0) {
archive_write_add_filter_gzip(ar);
archive_write_set_options(ar, "compression-level=9");
} else if (strcmp(compression, "bzip2") == 0) {
archive_write_add_filter_bzip2(ar);
archive_write_set_options(ar, "compression-level=9");
} else if (strcmp(compression, "lz4") == 0) {
archive_write_add_filter_lz4(ar);
archive_write_set_options(ar, "compression-level=9");
} else if (strcmp(compression, "xz") == 0) {
archive_write_add_filter_xz(ar);
} else if (strcmp(compression, "none") == 0) {
/* empty */
} else {
return false;
}
archive_write_set_format_pax_restricted(ar); archive_write_set_format_pax_restricted(ar);
archive_write_set_options(ar, "compression-level=9");
archive_write_open_fd(ar, repofd); archive_write_open_fd(ar, repofd);
/* XBPS_REPOIDX */ /* XBPS_REPOIDX */

6
bin/xbps-rindex/sign.c
View File

@ -1,5 +1,5 @@
/*- /*-
* Copyright (c) 2013-2015 Juan Romero Pardines. * Copyright (c) 2013-2019 Juan Romero Pardines.
* All rights reserved. * All rights reserved.
* *
* Redistribution and use in source and binary forms, with or without * Redistribution and use in source and binary forms, with or without
@ -152,7 +152,7 @@ ssl_init(void)
int int
sign_repo(struct xbps_handle *xhp, const char *repodir, sign_repo(struct xbps_handle *xhp, const char *repodir,
const char *privkey, const char *signedby) const char *privkey, const char *signedby, const char *compression)
{ {
struct xbps_repo *repo = NULL; struct xbps_repo *repo = NULL;
xbps_dictionary_t meta = NULL; xbps_dictionary_t meta = NULL;
@ -231,7 +231,7 @@ sign_repo(struct xbps_handle *xhp, const char *repodir,
_XBPS_RINDEX, strerror(errno)); _XBPS_RINDEX, strerror(errno));
goto out; goto out;
} }
flush_failed = repodata_flush(xhp, repodir, "repodata", repo->idx, meta); flush_failed = repodata_flush(xhp, repodir, "repodata", repo->idx, meta, compression);
xbps_repo_unlock(rlockfd, rlockfname); xbps_repo_unlock(rlockfd, rlockfname);
if (!flush_failed) { if (!flush_failed) {
fprintf(stderr, "failed to write repodata: %s\n", strerror(errno)); fprintf(stderr, "failed to write repodata: %s\n", strerror(errno));

5
bin/xbps-rindex/xbps-rindex.1
View File

@ -1,4 +1,4 @@
.Dd June 12, 2019 .Dd June 17, 2019
.Dt XBPS-RINDEX 1 .Dt XBPS-RINDEX 1
.Sh NAME .Sh NAME
.Nm xbps-rindex .Nm xbps-rindex
@ -17,6 +17,9 @@ in local repositories.
.Bl -tag -width November 6-x .Bl -tag -width November 6-x
.It Fl d, Fl -debug .It Fl d, Fl -debug
Enables extra debugging shown to stderr. Enables extra debugging shown to stderr.
.It Fl -compression Ar none | gzip | bzip2 | xz | lz4 | zstd
Set the repodata compression format. If unset, defaults to
.Ar zstd .
.It Fl C -hashcheck .It Fl C -hashcheck
Check not only for file existence but for the correct file hash while cleaning. Check not only for file existence but for the correct file hash while cleaning.
This flag is only useful with the This flag is only useful with the