From 0aa996b0ae4e5455a28d5110e55224548abed3d6 Mon Sep 17 00:00:00 2001 From: Duncan Overbruck Date: Tue, 12 May 2020 13:52:52 +0200 Subject: [PATCH 1/3] bin/xbps-repodb: add new tool to manage multiple repositories at once This currently only supports -p/--purge to remove obsolete binary package files from repositories with the awareness that different noarch package version can co-exist within a repository in different architectures repository indexes. --- .gitignore | 1 + bin/Makefile | 1 + bin/xbps-repodb/Makefile | 7 + bin/xbps-repodb/defs.h | 2 + bin/xbps-repodb/main.c | 122 +++++++++++++++++ bin/xbps-repodb/purge.c | 284 +++++++++++++++++++++++++++++++++++++++ 6 files changed, 417 insertions(+) create mode 100644 bin/xbps-repodb/Makefile create mode 100644 bin/xbps-repodb/defs.h create mode 100644 bin/xbps-repodb/main.c create mode 100644 bin/xbps-repodb/purge.c diff --git a/.gitignore b/.gitignore index 66249eedc..2bdb5ebb3 100644 --- a/.gitignore +++ b/.gitignore @@ -18,6 +18,7 @@ bin/xbps-checkvers/xbps-checkvers bin/xbps-uunshare/xbps-uunshare bin/xbps-digest/xbps-digest bin/xbps-fetch/xbps-fetch +bin/xbps-repodb/xbps-repodb *.static *.so* *.o diff --git a/bin/Makefile b/bin/Makefile index c0586aabb..611a0474d 100644 --- a/bin/Makefile +++ b/bin/Makefile @@ -14,6 +14,7 @@ SUBDIRS += xbps-checkvers SUBDIRS += xbps-fbulk SUBDIRS += xbps-digest SUBDIRS += xbps-fetch +SUBDIRS += xbps-repodb ifeq (${XBPS_OS},linux) SUBDIRS += xbps-uchroot diff --git a/bin/xbps-repodb/Makefile b/bin/xbps-repodb/Makefile new file mode 100644 index 000000000..7d4c937b3 --- /dev/null +++ b/bin/xbps-repodb/Makefile @@ -0,0 +1,7 @@ +TOPDIR = ../.. +-include $(TOPDIR)/config.mk + +BIN = xbps-repodb +OBJS = main.o purge.o + +include $(TOPDIR)/mk/prog.mk diff --git a/bin/xbps-repodb/defs.h b/bin/xbps-repodb/defs.h new file mode 100644 index 000000000..6ed5088dd --- /dev/null +++ b/bin/xbps-repodb/defs.h @@ -0,0 +1,2 @@ +/* clean repositories */ +int purge_repos(struct xbps_handle *, int, char *[], bool); diff --git a/bin/xbps-repodb/main.c b/bin/xbps-repodb/main.c new file mode 100644 index 000000000..721645b99 --- /dev/null +++ b/bin/xbps-repodb/main.c @@ -0,0 +1,122 @@ +/*- + * Copyright (c) 2020 Duncan Overbruck + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "defs.h" + + +static void __attribute__((noreturn)) +usage(void) +{ + fprintf(stdout, + "Usage: xbps-repodb [OPTIONS] MODE ...\n\n" + "OPTIONS:\n" + " -d, --debug Enable debug messages to stderr\n" + " -n, --dry-run Dry-run mode\n" + " -v, --verbose Enable verbose output\n" + " -V, --version Prints the xbps release version\n" + "MODE:\n" + " -p, --purge Remove obsolete binary packages from repositories\n"); + exit(EXIT_FAILURE); +} + +int +main(int argc, char **argv) +{ + int c = 0, rv = 0; + struct xbps_handle xh; + const struct option longopts[] = { + { "debug", no_argument, NULL, 'd' }, + { "dry-run", no_argument, NULL, 'n' }, + { "verbose", no_argument, NULL, 'v' }, + { "version", no_argument, NULL, 'V' }, + { NULL, 0, NULL, 0 } + }; + bool dry = false; + enum { + MODE_NIL, + MODE_PURGE, + } mode; + + memset(&xh, 0, sizeof xh); + + while ((c = getopt_long(argc, argv, "dhnpVv", longopts, NULL)) != -1) { + switch (c) { + case 'd': + xh.flags |= XBPS_FLAG_DEBUG; + break; + case 'p': + mode = MODE_PURGE; + break; + case 'n': + dry = true; + break; + case 'v': + xh.flags |= XBPS_FLAG_VERBOSE; + break; + case 'V': + printf("%s\n", XBPS_RELVER); + exit(EXIT_SUCCESS); + case '?': + case 'h': + default: + usage(); + } + } + argc -= optind; + argv += optind; + + if (argc == 0 || mode == MODE_NIL) + usage(); + + /* + * Initialize libxbps. + */ + xh.flags |= XBPS_FLAG_IGNORE_CONF_REPOS; + if ((rv = xbps_init(&xh)) != 0) { + xbps_error_printf("failed to initialize libxbps: %s\n", + strerror(rv)); + exit(EXIT_FAILURE); + } + + switch (mode) { + case MODE_PURGE: + rv = purge_repos(&xh, argc, argv, dry); + case MODE_NIL: + break; + } + + xbps_end(&xh); + exit(rv ? EXIT_FAILURE : EXIT_SUCCESS); +} diff --git a/bin/xbps-repodb/purge.c b/bin/xbps-repodb/purge.c new file mode 100644 index 000000000..f0f6c4df6 --- /dev/null +++ b/bin/xbps-repodb/purge.c @@ -0,0 +1,284 @@ +/*- * Copyright (c) 2020 Duncan Overbruck + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "uthash.h" +#include "defs.h" + +struct arch { + char arch[64]; + struct arch *next; + struct xbps_repo *repo, *stage; +}; + +struct repo { + char path[PATH_MAX]; + struct repo *next; + struct dirent **namelist; + int nnames; + struct arch *archs; +}; + +static struct repo *repos; + +static void +add_repo(struct xbps_handle *xhp, const char *path) +{ + struct repo *repo = calloc(1, sizeof (struct repo)); + if (repo == NULL) { + perror("calloc"); + exit(EXIT_FAILURE); + } + xbps_strlcpy(repo->path, path, sizeof repo->path); + repo->namelist = NULL; + repo->nnames = 0; + repo->archs = NULL; + repo->next = repos; + repos = repo; + xbps_dbg_printf(xhp, "Scanning repository: %s\n", path); + + repo->nnames = scandir(path, &repo->namelist, NULL, NULL); + if (repo->nnames == -1) { + perror("scandir"); + exit(EXIT_FAILURE); + } + + for (int i = 0; i < repo->nnames; i++) { + const char *name = repo->namelist[i]->d_name, *d; + if (*name == '.') + continue; + if ((d = strrchr(name, '-')) == NULL) + continue; + if (strcmp(d+1, "repodata") == 0) { + struct arch *arch = calloc(1, sizeof (struct arch)); + if (arch == NULL) { + perror("calloc"); + exit(1); + } + if ((size_t)(d-name) >= sizeof arch->arch) { + xbps_error_printf("invalid repodata: %s\n", name); + exit(1); + } + strncpy(arch->arch, name, d-name); + arch->next = repo->archs; + repo->archs = arch; + xbps_dbg_printf(xhp, " found architecture: %s\n", arch->arch); + + xhp->target_arch = arch->arch; + arch->repo = xbps_repo_public_open(xhp, path); + if (arch->repo == NULL) { + xbps_error_printf("Failed to read repodata: %s", + strerror(errno)); + exit(1); + } + arch->stage = xbps_repo_stage_open(xhp, path); + if (arch->repo == NULL && errno != ENOENT) { + xbps_error_printf("Failed to read stagedata: %s", + strerror(errno)); + exit(1); + } + } + } +} + +static bool +same_pkgver(xbps_dictionary_t pkgd, const char *pkgver) +{ + const char *rpkgver = NULL; + xbps_dictionary_get_cstring_nocopy(pkgd, "pkgver", &rpkgver); + return strcmp(pkgver, rpkgver) == 0; +} + +/* + * Return true if this pkgver is not in any of the repodata + * or stagedata repository indexes. + */ +static bool +check_obsolete_noarch(struct xbps_handle *xhp, struct repo *repo, + const char *pkgver) +{ + char name[XBPS_NAME_SIZE]; + xbps_dictionary_t pkgd; + + if (!xbps_pkg_name(name, sizeof name, pkgver)) { + xbps_error_printf("invalid pkgver: %s\n", pkgver); + return false; + } + for (struct arch *a = repo->archs; a; a = a->next) { + if (a->stage != NULL) { + if ((pkgd = xbps_dictionary_get(a->stage->idx, name)) != NULL) { + if (same_pkgver(pkgd, pkgver)) { + xbps_dbg_printf(xhp, "found package " + "`%s' in `%s/%s-stagedata'\n", pkgver, repo->path, + a->arch); + return false; + } + } + } + if ((pkgd = xbps_dictionary_get(a->repo->idx, name)) != NULL) { + if (same_pkgver(pkgd, pkgver)) { + xbps_dbg_printf(xhp, "found package " + "`%s' in `%s/%s-repodata'\n", pkgver, repo->path, a->arch); + return false; + } + } + } + + xbps_dbg_printf(xhp, "package `%s' is obsolete\n", pkgver); + return true; +} + +/* + * If the package is noarch, check all indexes using `check_obsolete_noarch`, + * otherwise return true if the repodata version doesn't match + * the supplied pkgver. + */ +static bool +check_obsolete(struct xbps_handle *xhp, struct repo *repo, + const char *pkgver, const char *arch) +{ + char name[XBPS_NAME_SIZE]; + struct arch *found = NULL; + xbps_dictionary_t pkgd; + + if (strcmp(arch, "noarch") == 0) + return check_obsolete_noarch(xhp, repo, pkgver); + + for (struct arch *a = repo->archs; a; a = a->next) + if (strcmp(a->arch, arch) == 0) + found = a; + + if (found == NULL) { + /* XXX: found package for architecture without repodata, delete? */ + xbps_error_printf("package `%s' with architecture `%s' without repository index\n", + pkgver, arch); + return false; + } + + if (!xbps_pkg_name(name, sizeof name, pkgver)) { + /* XXX: delete invalid packages? */ + xbps_error_printf("invalid pkgver: %s\n", pkgver); + return false; + } + + if (found->stage != NULL) { + if ((pkgd = xbps_dictionary_get(found->stage->idx, name)) != NULL) { + if (same_pkgver(pkgd, pkgver)) { + xbps_dbg_printf(xhp, "found package " + "`%s' in `%s/%s-stagedata'\n", pkgver, repo->path, + found->arch); + return false; + } + } + } + if ((pkgd = xbps_dictionary_get(found->repo->idx, name)) != NULL) { + if (same_pkgver(pkgd, pkgver)) { + xbps_dbg_printf(xhp, "found package " + "`%s' in `%s/%s-repodata'\n", pkgver, repo->path, found->arch); + return false; + } + } + + xbps_dbg_printf(xhp, "package `%s' is obsolete\n", pkgver); + return true; +} + +static int +purge_repo(struct xbps_handle *xhp, struct repo *repo, bool dry) +{ + char buf[PATH_MAX], path[PATH_MAX]; + const char *pkgver, *arch; + size_t pathlen = strlen(repo->path); + + xbps_strlcpy(path, repo->path, sizeof path); + + for (int i = 0; i < repo->nnames; i++) { + const char *name = repo->namelist[i]->d_name; + char *d; + xbps_strlcpy(buf, name, sizeof buf); + if ((d = strrchr(buf, '.')) == NULL) + continue; + if (strcmp(d+1, "xbps") != 0) + continue; + *d = '\0'; + if ((d = strrchr(buf, '.')) == NULL) + continue; + *d = '\0'; + arch = d+1; + pkgver = buf; + + if (!check_obsolete(xhp, repo, pkgver, arch)) { + /* the package is not obsolete */ + continue; + } + + /* reset path to append the file */ + path[pathlen] = '\0'; + + if (xbps_path_append(path, sizeof path, name) == -1) { + xbps_error_printf("path too long"); + exit(1); + } + if (dry || xhp->flags & XBPS_FLAG_VERBOSE) + fprintf(stdout, "removing %s...\n", path); + if (!dry) { + if (unlink(path) == -1) + xbps_error_printf("unlink: %s: %s", path, strerror(errno)); + } + + /* try to remove signature file */ + if (xbps_strlcat(path, ".sig", sizeof path) >= sizeof path) { + xbps_error_printf("path too long"); + exit(1); + } + if (dry || xhp->flags & XBPS_FLAG_VERBOSE) + fprintf(stdout, "removing %s...\n", path); + if (!dry) { + if (unlink(path) == -1 && errno != ENOENT) + xbps_error_printf("unlink: %s: %s", path, strerror(errno)); + } + + } + return 0; +} + +int +purge_repos(struct xbps_handle *xhp, int argc, char *argv[], bool dry) +{ + for (int i = 0; i < argc; i++) + add_repo(xhp, argv[i]); + for (struct repo *repo = repos; repo; repo = repo->next) + purge_repo(xhp, repo, dry); + return 0; +} From 767a117a2f2cb8c8555c14ea0660c781b9a5e79a Mon Sep 17 00:00:00 2001 From: Duncan Overbruck Date: Tue, 12 May 2020 14:30:21 +0200 Subject: [PATCH 2/3] bin/xbps-repodb: add stupid header guard for lgtm --- bin/xbps-repodb/defs.h | 32 +++++++++++++++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/bin/xbps-repodb/defs.h b/bin/xbps-repodb/defs.h index 6ed5088dd..5a1a5f9fb 100644 --- a/bin/xbps-repodb/defs.h +++ b/bin/xbps-repodb/defs.h @@ -1,2 +1,32 @@ -/* clean repositories */ +/*- + * Copyright (c) 2020 Duncan Overbruck + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _XBPS_REPODB_DEFS_H_ +#define _XBPS_REPODB_DEFS_H_ + +/* Form purge.c */ int purge_repos(struct xbps_handle *, int, char *[], bool); + +#endif /* !_XBPS_REPODB_DEFS_H_ */ From dd265f91c5cf876d8e12c44475c15cdccb358ea7 Mon Sep 17 00:00:00 2001 From: Duncan Overbruck Date: Fri, 15 May 2020 17:21:28 +0200 Subject: [PATCH 3/3] bin/xbps-repodb: initialize mode --- bin/xbps-repodb/main.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/xbps-repodb/main.c b/bin/xbps-repodb/main.c index 721645b99..fb0402871 100644 --- a/bin/xbps-repodb/main.c +++ b/bin/xbps-repodb/main.c @@ -67,7 +67,7 @@ main(int argc, char **argv) enum { MODE_NIL, MODE_PURGE, - } mode; + } mode = MODE_NIL; memset(&xh, 0, sizeof xh);