From 6c1759862e0cc6b2218c1d13301902a05b027c55 Mon Sep 17 00:00:00 2001 From: Juan RP Date: Mon, 17 Jun 2019 07:24:05 +0200 Subject: [PATCH] xbps-rindex: added --compression option and set zstd by default. Multiple compression formats are now supported for repodata, none, gzip, bzip2, xz, lz4 and zstd. --- bin/xbps-rindex/defs.h | 8 ++++---- bin/xbps-rindex/index-add.c | 12 +++++++----- bin/xbps-rindex/index-clean.c | 21 ++++++++++++--------- bin/xbps-rindex/main.c | 14 ++++++++++---- bin/xbps-rindex/repoflush.c | 26 +++++++++++++++++++++++--- bin/xbps-rindex/sign.c | 6 +++--- bin/xbps-rindex/xbps-rindex.1 | 5 ++++- 7 files changed, 63 insertions(+), 29 deletions(-) diff --git a/bin/xbps-rindex/defs.h b/bin/xbps-rindex/defs.h index ada46df8..1545ed23 100644 --- a/bin/xbps-rindex/defs.h +++ b/bin/xbps-rindex/defs.h @@ -67,21 +67,21 @@ #define _XBPS_RINDEX "xbps-rindex" /* From index-add.c */ -int index_add(struct xbps_handle *, int, int, char **, bool); +int index_add(struct xbps_handle *, int, int, char **, bool, const char *); /* From index-clean.c */ -int index_clean(struct xbps_handle *, const char *, bool); +int index_clean(struct xbps_handle *, const char *, bool, const char *); /* From remove-obsoletes.c */ int remove_obsoletes(struct xbps_handle *, const char *); /* From sign.c */ int sign_repo(struct xbps_handle *, const char *, const char *, - const char *); + const char *, const char *); int sign_pkgs(struct xbps_handle *, int, int, char **, const char *, bool); /* From repoflush.c */ bool repodata_flush(struct xbps_handle *, const char *, const char *, - xbps_dictionary_t, xbps_dictionary_t); + xbps_dictionary_t, xbps_dictionary_t, const char *); #endif /* !_XBPS_RINDEX_DEFS_H_ */ diff --git a/bin/xbps-rindex/index-add.c b/bin/xbps-rindex/index-add.c index 5f5460ba..886727d8 100644 --- a/bin/xbps-rindex/index-add.c +++ b/bin/xbps-rindex/index-add.c @@ -56,7 +56,9 @@ set_build_date(const xbps_dictionary_t pkgd, time_t timestamp) static bool repodata_commit(struct xbps_handle *xhp, const char *repodir, - xbps_dictionary_t idx, xbps_dictionary_t meta, xbps_dictionary_t stage) { + xbps_dictionary_t idx, xbps_dictionary_t meta, xbps_dictionary_t stage, + const char *compression) +{ xbps_object_iterator_t iter; xbps_object_t keysym; int rv; @@ -187,7 +189,7 @@ repodata_commit(struct xbps_handle *xhp, const char *repodir, printf("stage: added `%s' (%s)\n", pkgver, arch); } xbps_object_iterator_release(iter); - rv = repodata_flush(xhp, repodir, "stagedata", stage, NULL); + rv = repodata_flush(xhp, repodir, "stagedata", stage, NULL, compression); } else { char *stagefile; @@ -205,7 +207,7 @@ repodata_commit(struct xbps_handle *xhp, const char *repodir, stagefile = xbps_repo_path_with_name(xhp, repodir, "stagedata"); unlink(stagefile); free(stagefile); - rv = repodata_flush(xhp, repodir, "repodata", idx, meta); + rv = repodata_flush(xhp, repodir, "repodata", idx, meta, compression); } xbps_object_release(usedshlibs); xbps_object_release(oldshlibs); @@ -213,7 +215,7 @@ repodata_commit(struct xbps_handle *xhp, const char *repodir, } int -index_add(struct xbps_handle *xhp, int args, int argmax, char **argv, bool force) +index_add(struct xbps_handle *xhp, int args, int argmax, char **argv, bool force, const char *compression) { xbps_dictionary_t idx, idxmeta, idxstage, binpkgd, curpkgd; struct xbps_repo *repo = NULL, *stage = NULL; @@ -405,7 +407,7 @@ index_add(struct xbps_handle *xhp, int args, int argmax, char **argv, bool force /* * Generate repository data files. */ - if (!repodata_commit(xhp, repodir, idx, idxmeta, idxstage)) { + if (!repodata_commit(xhp, repodir, idx, idxmeta, idxstage, compression)) { fprintf(stderr, "%s: failed to write repodata: %s\n", _XBPS_RINDEX, strerror(errno)); goto out; diff --git a/bin/xbps-rindex/index-clean.c b/bin/xbps-rindex/index-clean.c index 27cedd6e..79f099d6 100644 --- a/bin/xbps-rindex/index-clean.c +++ b/bin/xbps-rindex/index-clean.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2012-2015 Juan Romero Pardines. + * Copyright (c) 2012-2019 Juan Romero Pardines. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -44,6 +44,7 @@ struct CleanerCbInfo { const char *repourl; bool hashcheck; }; + static int idx_cleaner_cb(struct xbps_handle *xhp, xbps_object_t obj, @@ -94,7 +95,8 @@ out: static int cleanup_repo(struct xbps_handle *xhp, const char *repodir, struct xbps_repo *repo, - const char *reponame, bool hashcheck) { + const char *reponame, bool hashcheck, const char *compression) +{ int rv = 0; xbps_array_t allkeys; struct CleanerCbInfo info = { @@ -109,20 +111,20 @@ cleanup_repo(struct xbps_handle *xhp, const char *repodir, struct xbps_repo *rep (void)xbps_array_foreach_cb_multi(xhp, allkeys, repo->idx, idx_cleaner_cb, &info); xbps_object_release(allkeys); - if(strcmp("stagedata", reponame) == 0 && xbps_dictionary_count(dest) == 0) { + if (strcmp("stagedata", reponame) == 0 && xbps_dictionary_count(dest) == 0) { char *stagefile = xbps_repo_path_with_name(xhp, repodir, "stagedata"); unlink(stagefile); free(stagefile); } if (!xbps_dictionary_equals(dest, repo->idx)) { - if (!repodata_flush(xhp, repodir, reponame, dest, repo->idxmeta)) { + if (!repodata_flush(xhp, repodir, reponame, dest, repo->idxmeta, compression)) { rv = errno; fprintf(stderr, "failed to write repodata: %s\n", strerror(errno)); return rv; } } - if(strcmp("stagedata", reponame) == 0) + if (strcmp("stagedata", reponame) == 0) printf("stage: %u packages registered.\n", xbps_dictionary_count(dest)); else printf("index: %u packages registered.\n", xbps_dictionary_count(dest)); @@ -134,7 +136,7 @@ cleanup_repo(struct xbps_handle *xhp, const char *repodir, struct xbps_repo *rep * binary package cannot be read (unavailable, not enough perms, etc). */ int -index_clean(struct xbps_handle *xhp, const char *repodir, const bool hashcheck) +index_clean(struct xbps_handle *xhp, const char *repodir, const bool hashcheck, const char *compression) { struct xbps_repo *repo, *stage; char *rlockfname = NULL; @@ -166,10 +168,11 @@ index_clean(struct xbps_handle *xhp, const char *repodir, const bool hashcheck) } printf("Cleaning `%s' index, please wait...\n", repodir); - if((rv = cleanup_repo(xhp, repodir, repo, "repodata", hashcheck))) + if ((rv = cleanup_repo(xhp, repodir, repo, "repodata", hashcheck, compression))) { goto out; - if(stage) { - cleanup_repo(xhp, repodir, stage, "stagedata", hashcheck); + } + if (stage) { + cleanup_repo(xhp, repodir, stage, "stagedata", hashcheck, compression); } out: diff --git a/bin/xbps-rindex/main.c b/bin/xbps-rindex/main.c index b3ede2e2..57cc3c48 100644 --- a/bin/xbps-rindex/main.c +++ b/bin/xbps-rindex/main.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2012-2015 Juan Romero Pardines. + * Copyright (c) 2012-2019 Juan Romero Pardines. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -43,6 +43,7 @@ usage(bool fail) " -v --verbose Verbose messages\n" " -V --version Show XBPS version\n" " -C --hashcheck Consider file hashes for cleaning up packages\n" + " --compression Compression format: none, gzip, bzip2, lz4 (default), zstd, xz.\n" " --privkey Path to the private key for signing\n" " --signedby Signature details, i.e \"name \"\n\n" "MODE\n" @@ -72,9 +73,11 @@ main(int argc, char **argv) { "sign", no_argument, NULL, 's'}, { "sign-pkg", no_argument, NULL, 'S'}, { "hashcheck", no_argument, NULL, 'C' }, + { "compression", required_argument, NULL, 2}, { NULL, 0, NULL, 0 } }; struct xbps_handle xh; + const char *compression = NULL; const char *privkey = NULL, *signedby = NULL; int rv, c, flags = 0; bool add_mode, clean_mode, rm_mode, sign_mode, sign_pkg_mode, force, @@ -91,6 +94,9 @@ main(int argc, char **argv) case 1: signedby = optarg; break; + case 2: + compression = optarg; + break; case 'a': add_mode = true; break; @@ -149,13 +155,13 @@ main(int argc, char **argv) } if (add_mode) - rv = index_add(&xh, optind, argc, argv, force); + rv = index_add(&xh, optind, argc, argv, force, compression); else if (clean_mode) - rv = index_clean(&xh, argv[optind], hashcheck); + rv = index_clean(&xh, argv[optind], hashcheck, compression); else if (rm_mode) rv = remove_obsoletes(&xh, argv[optind]); else if (sign_mode) - rv = sign_repo(&xh, argv[optind], privkey, signedby); + rv = sign_repo(&xh, argv[optind], privkey, signedby, compression); else if (sign_pkg_mode) rv = sign_pkgs(&xh, optind, argc, argv, privkey, force); diff --git a/bin/xbps-rindex/repoflush.c b/bin/xbps-rindex/repoflush.c index ca3ac1f8..5365d4fa 100644 --- a/bin/xbps-rindex/repoflush.c +++ b/bin/xbps-rindex/repoflush.c @@ -39,7 +39,8 @@ bool repodata_flush(struct xbps_handle *xhp, const char *repodir, - const char *reponame, xbps_dictionary_t idx, xbps_dictionary_t meta) + const char *reponame, xbps_dictionary_t idx, xbps_dictionary_t meta, + const char *compression) { struct archive *ar; char *repofile, *tname, *buf; @@ -57,9 +58,28 @@ repodata_flush(struct xbps_handle *xhp, const char *repodir, /* Create and write our repository archive */ ar = archive_write_new(); assert(ar); - archive_write_set_compression_gzip(ar); + + if (compression == NULL || strcmp(compression, "zstd") == 0) { + archive_write_add_filter_zstd(ar); + archive_write_set_options(ar, "compression-level=19"); + } else if (strcmp(compression, "gzip") == 0) { + archive_write_add_filter_gzip(ar); + archive_write_set_options(ar, "compression-level=9"); + } else if (strcmp(compression, "bzip2") == 0) { + archive_write_add_filter_bzip2(ar); + archive_write_set_options(ar, "compression-level=9"); + } else if (strcmp(compression, "lz4") == 0) { + archive_write_add_filter_lz4(ar); + archive_write_set_options(ar, "compression-level=9"); + } else if (strcmp(compression, "xz") == 0) { + archive_write_add_filter_xz(ar); + } else if (strcmp(compression, "none") == 0) { + /* empty */ + } else { + return false; + } + archive_write_set_format_pax_restricted(ar); - archive_write_set_options(ar, "compression-level=9"); archive_write_open_fd(ar, repofd); /* XBPS_REPOIDX */ diff --git a/bin/xbps-rindex/sign.c b/bin/xbps-rindex/sign.c index b6ccd6ba..4a714bb9 100644 --- a/bin/xbps-rindex/sign.c +++ b/bin/xbps-rindex/sign.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2013-2015 Juan Romero Pardines. + * Copyright (c) 2013-2019 Juan Romero Pardines. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -152,7 +152,7 @@ ssl_init(void) int sign_repo(struct xbps_handle *xhp, const char *repodir, - const char *privkey, const char *signedby) + const char *privkey, const char *signedby, const char *compression) { struct xbps_repo *repo = NULL; xbps_dictionary_t meta = NULL; @@ -231,7 +231,7 @@ sign_repo(struct xbps_handle *xhp, const char *repodir, _XBPS_RINDEX, strerror(errno)); goto out; } - flush_failed = repodata_flush(xhp, repodir, "repodata", repo->idx, meta); + flush_failed = repodata_flush(xhp, repodir, "repodata", repo->idx, meta, compression); xbps_repo_unlock(rlockfd, rlockfname); if (!flush_failed) { fprintf(stderr, "failed to write repodata: %s\n", strerror(errno)); diff --git a/bin/xbps-rindex/xbps-rindex.1 b/bin/xbps-rindex/xbps-rindex.1 index 25ab6fde..b393a71c 100644 --- a/bin/xbps-rindex/xbps-rindex.1 +++ b/bin/xbps-rindex/xbps-rindex.1 @@ -1,4 +1,4 @@ -.Dd June 12, 2019 +.Dd June 17, 2019 .Dt XBPS-RINDEX 1 .Sh NAME .Nm xbps-rindex @@ -17,6 +17,9 @@ in local repositories. .Bl -tag -width November 6-x .It Fl d, Fl -debug Enables extra debugging shown to stderr. +.It Fl -compression Ar none | gzip | bzip2 | xz | lz4 | zstd +Set the repodata compression format. If unset, defaults to +.Ar zstd . .It Fl C -hashcheck Check not only for file existence but for the correct file hash while cleaning. This flag is only useful with the