From de07d618beac312558dec766756568b3a2de1786 Mon Sep 17 00:00:00 2001 From: Duncan Overbruck Date: Sun, 9 Nov 2025 02:10:05 +0100 Subject: [PATCH 01/17] tests: add xbps-rindex multi repo staging test --- tests/xbps/xbps-rindex/add_test.sh | 108 +++++++++++++++++++++++++++++ 1 file changed, 108 insertions(+) diff --git a/tests/xbps/xbps-rindex/add_test.sh b/tests/xbps/xbps-rindex/add_test.sh index 6fa99b5e..780220d9 100644 --- a/tests/xbps/xbps-rindex/add_test.sh +++ b/tests/xbps/xbps-rindex/add_test.sh @@ -194,10 +194,118 @@ stage_stacked_body() { } +atf_test_case stage_multi_repos + +stage_multi_repos_head() { + atf_set "descr" "xbps-rindex(1) -a: staging multiple repositories" +} + +stage_multi_repos_body() { + mkdir -p repo1 repo2 root pkg + + cd repo1 + atf_check -o ignore -- xbps-create -A noarch -n flac-1.4.3_1 -s "flac pkg" --shlib-provides "libFLAC.so.12" ../pkg + atf_check -o ignore -- xbps-create -A noarch -n ruby-3.3.8_1 -s "ruby pkg" --shlib-provides "libruby.so.3.3" ../pkg + atf_check -o ignore -- xbps-create -A noarch -n A-1.0_1 -s "A pkg" --shlib-requires "libFLAC.so.12" ../pkg + atf_check -o ignore -- xbps-create -A noarch -n B-1.0_1 -s "B pkg" --shlib-requires "libruby.so.3.3" ../pkg + cd .. + + cd repo2 + atf_check -o ignore -- xbps-create -A noarch -n AA-1.0_1 -s "AA pkg" --shlib-requires "libFLAC.so.12" ../pkg + atf_check -o ignore -- xbps-create -A noarch -n BB-1.0_1 -s "BB pkg" --shlib-requires "libruby.so.3.3" ../pkg + cd .. + + atf_check -e ignore \ + -o match:"index: added \`A-1.0_1'" \ + -o match:"index: added \`B-1.0_1'" \ + -o match:"index: added \`flac-1.4.3_1'" \ + -o match:"index: added \`ruby-3.3.8_1'" \ + -- xbps-rindex -v -R repo1 -R repo2 -a \ + repo1/flac-1.4.3_1.noarch.xbps \ + repo1/ruby-3.3.8_1.noarch.xbps \ + repo1/A-1.0_1.noarch.xbps \ + repo1/B-1.0_1.noarch.xbps + atf_check -e ignore \ + -o match:"index: added \`AA-1.0_1'" \ + -o match:"index: added \`BB-1.0_1'" \ + -- xbps-rindex -v -R repo1 -R repo2 -a \ + repo2/AA-1.0_1.noarch.xbps \ + repo2/BB-1.0_1.noarch.xbps + + cd repo1 + atf_check -o ignore -- xbps-create -A noarch -n flac-1.5.0_1 -s "flac pkg" --shlib-provides "libFLAC.so.14" ../pkg + cd .. + + atf_check -e ignore -o match:"stage: added \`flac-1.5.0_1'" \ + -- xbps-rindex -v -R repo1 -R repo2 -a repo1/flac-1.5.0_1.noarch.xbps + + atf_check \ + -o match:"repo1 \(Staged\)" \ + -o not-match:"repo2 \(Staged\)" \ + -- xbps-query -r ../root -i --repository=repo1 --repository=repo2 -L + + cd repo2 + atf_check -o ignore -- xbps-create -A noarch -n AA-1.0_2 -s "AA pkg" --shlib-requires "libFLAC.so.14" ../pkg + cd .. + + atf_check -o match:"stage: added \`AA-1.0_2'" \ + -- xbps-rindex -v -R repo1 -R repo2 -a repo2/AA-1.0_2.noarch.xbps + + atf_check \ + -o match:"repo1 \(Staged\)" \ + -o match:"repo2 \(Staged\)" \ + -- xbps-query -r ../root -i --repository=repo1 --repository=repo2 -L + + cd repo1 + atf_check -o ignore -- xbps-create -A noarch -n ruby-3.4.5_1 -s "ruby pkg" --shlib-provides "libruby.so.3.4" ../pkg + cd .. + atf_check -o match:"stage: added \`ruby-3.4.5_1'" \ + -- xbps-rindex -v -R repo1 -R repo2 -a repo1/ruby-3.4.5_1.noarch.xbps + + atf_check \ + -o match:"repo1 \(Staged\)" \ + -o match:"repo2 \(Staged\)" \ + -- xbps-query -r ../root -i --repository=repo1 --repository=repo2 -L + + cd repo2 + atf_check -o ignore -- xbps-create -A noarch -n BB-1.0_2 -s "BB pkg" --shlib-requires "libruby.so.3.4" ../pkg + cd .. + atf_check -o match:"stage: added \`BB-1.0_2'" \ + -- xbps-rindex -v -R repo1 -R repo2 -a repo2/BB-1.0_2.noarch.xbps + + cd repo1 + atf_check -o ignore -- xbps-create -A noarch -n A-1.0_2 -s "A pkg" --shlib-requires "libFLAC.so.14" ../pkg + atf_check -o ignore -- xbps-create -A noarch -n B-1.0_2 -s "B pkg" --shlib-requires "libruby.so.3.4" ../pkg + cd .. + atf_check \ + -o match:"stage: added \`A-1.0_2'" \ + -- xbps-rindex -v -R repo1 -R repo2 -a repo1/A-1.0_2.noarch.xbps + + atf_check \ + -o match:"repo1 \(Staged\)" \ + -o match:"repo2 \(Staged\)" \ + -- xbps-query -r ../root -i --repository=repo1 --repository=repo2 -L + + atf_check \ + -o match:"index: added \`A-1\.0_2'" \ + -o match:"index: added \`B-1\.0_2'" \ + -o match:"index: added \`flac-1\.5\.0_1'" \ + -o match:"index: added \`ruby-3\.4\.5_1'" \ + -o match:"index: added \`AA-1\.0_2'" \ + -o match:"index: added \`BB-1\.0_2'" \ + -- xbps-rindex -v -R repo1 -R repo2 -a repo1/B-1.0_2.noarch.xbps + + atf_check \ + -o not-match:"repo1 \(Staged\)" \ + -o not-match:"repo2 \(Staged\)" \ + -- xbps-query -r ../root -i --repository=repo1 --repository=repo2 -L +} + atf_init_test_cases() { atf_add_test_case update atf_add_test_case revert atf_add_test_case stage atf_add_test_case stage_resolve_bug atf_add_test_case stage_stacked + atf_add_test_case stage_multi_repos } From a0517947a166ca1e71ea1bf6d327ccb9c861c0db Mon Sep 17 00:00:00 2001 From: Duncan Overbruck Date: Sun, 9 Nov 2025 02:07:17 +0100 Subject: [PATCH 02/17] bin/xbps-rindex: add --repository arg --- bin/xbps-rindex/main.c | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/bin/xbps-rindex/main.c b/bin/xbps-rindex/main.c index 0f9ed316..dfbacb0b 100644 --- a/bin/xbps-rindex/main.c +++ b/bin/xbps-rindex/main.c @@ -30,6 +30,7 @@ #include #include "defs.h" +#include "xbps/xbps_array.h" static void __attribute__((noreturn)) usage(bool fail) @@ -46,6 +47,7 @@ usage(bool fail) " --compression Compression format: none, gzip, bzip2, lz4, xz, zstd (default)\n" " --privkey Path to the private key for signing\n" " --signedby Signature details, i.e \"name \"\n\n" + " -R, --repository Add a local repository\n" "MODE\n" " -a, --add ... Add package(s) to repository index\n" " -c, --clean Clean repository index\n" @@ -58,7 +60,7 @@ usage(bool fail) int main(int argc, char **argv) { - const char *shortopts = "acdfhrsCSVv"; + const char *shortopts = "acdfhrR:sCSVv"; struct option longopts[] = { { "add", no_argument, NULL, 'a' }, { "clean", no_argument, NULL, 'c' }, @@ -74,12 +76,14 @@ main(int argc, char **argv) { "sign-pkg", no_argument, NULL, 'S'}, { "hashcheck", no_argument, NULL, 'C' }, { "compression", required_argument, NULL, 2}, + { "repository", required_argument, NULL, 'R'}, { NULL, 0, NULL, 0 } }; struct xbps_handle xh; const char *compression = NULL; const char *privkey = NULL, *signedby = NULL; int rv, c, flags = 0; + xbps_array_t repos = NULL; bool add_mode, clean_mode, rm_mode, sign_mode, sign_pkg_mode, force, hashcheck; @@ -121,6 +125,14 @@ main(int argc, char **argv) case 'C': hashcheck = true; break; + case 'R': + if (!repos) + repos = xbps_array_create(); + if (!repos || !xbps_array_add_cstring(repos, optarg)) { + xbps_error_oom(); + exit(1); + } + break; case 'S': sign_pkg_mode = true; break; From 40b0a052ac6e31e3d2869f8b150d7be1264acc6f Mon Sep 17 00:00:00 2001 From: Duncan Overbruck Date: Tue, 9 Dec 2025 17:49:36 +0100 Subject: [PATCH 03/17] tests: cleanup staging test --- tests/xbps/xbps-rindex/add_test.sh | 39 +++++++++++++++--------------- 1 file changed, 19 insertions(+), 20 deletions(-) diff --git a/tests/xbps/xbps-rindex/add_test.sh b/tests/xbps/xbps-rindex/add_test.sh index 780220d9..81b6b110 100644 --- a/tests/xbps/xbps-rindex/add_test.sh +++ b/tests/xbps/xbps-rindex/add_test.sh @@ -60,39 +60,38 @@ stage_head() { stage_body() { mkdir -p some_repo pkg_A pkg_B touch pkg_A/file00 pkg_B/file01 + cd some_repo - xbps-create -A noarch -n foo-1.0_1 -s "foo pkg" --shlib-provides "libfoo.so.1" ../pkg_A - atf_check_equal $? 0 - xbps-rindex -d -a $PWD/*.xbps - atf_check_equal $? 0 + atf_check -o ignore -- xbps-create -A noarch -n foo-1.0_1 -s "foo pkg" --shlib-provides "libfoo.so.1" ../pkg_A + + atf_check -e ignore -o match:"index: added \`foo-1.0_1'" -- \ + xbps-rindex -d -a $PWD/*.xbps + atf_check -o inline:" 1 $PWD (RSA unsigned)\n" -- \ xbps-query -r ../root -i --repository=$PWD -L - xbps-create -A noarch -n foo-1.1_1 -s "foo pkg" --shlib-provides "libfoo.so.2" ../pkg_A - atf_check_equal $? 0 - xbps-rindex -d -a $PWD/*.xbps - atf_check_equal $? 0 + atf_check -o ignore -- xbps-create -A noarch -n foo-1.1_1 -s "foo pkg" --shlib-provides "libfoo.so.2" ../pkg_A + atf_check -e ignore -o match:"index: added \`foo-1.1_1'" -- valgrind xbps-rindex -d -a $PWD/*.xbps atf_check -o inline:" 1 $PWD (RSA unsigned)\n" -- \ xbps-query -r ../root -i --repository=$PWD -L - xbps-create -A noarch -n bar-1.0_1 -s "foo pkg" --shlib-requires "libfoo.so.2" ../pkg_B - atf_check_equal $? 0 - xbps-rindex -d -a $PWD/*.xbps - atf_check_equal $? 0 + atf_check -o ignore -- xbps-create -A noarch -n bar-1.0_1 -s "foo pkg" --shlib-requires "libfoo.so.2" ../pkg_B + atf_check -e ignore -o match:"index: added \`bar-1.0_1'" -- \ + xbps-rindex -d -a $PWD/*.xbps atf_check -o inline:" 2 $PWD (RSA unsigned)\n" -- \ xbps-query -r ../root -i --repository=$PWD -L - xbps-create -A noarch -n foo-1.2_1 -s "foo pkg" --shlib-provides "libfoo.so.3" ../pkg_A - atf_check_equal $? 0 - xbps-rindex -d -a $PWD/*.xbps - atf_check_equal $? 0 + atf_check -o ignore -- xbps-create -A noarch -n foo-1.2_1 -s "foo pkg" --shlib-provides "libfoo.so.3" ../pkg_A + atf_check -e ignore -o not-match:"index: added \`foo-1.2_1'" -- \ + xbps-rindex -d -a $PWD/*.xbps atf_check -o inline:" 2 $PWD (Staged) (RSA unsigned)\n" -- \ xbps-query -r ../root -i --repository=$PWD -L - xbps-create -A noarch -n bar-1.1_1 -s "foo pkg" --shlib-requires "libfoo.so.3" ../pkg_A - atf_check_equal $? 0 - xbps-rindex -d -a $PWD/*.xbps - atf_check_equal $? 0 + atf_check -o ignore -- xbps-create -A noarch -n bar-1.1_1 -s "foo pkg" --shlib-requires "libfoo.so.3" ../pkg_A + atf_check -e ignore \ + -o match:"added \`bar-1.1_1'" \ + -o match:"added \`foo-1.2_1'" \ + -- xbps-rindex -d -a $PWD/*.xbps atf_check -o inline:" 2 $PWD (RSA unsigned)\n" -- \ xbps-query -r ../root -i --repository=$PWD -L } From 7aa95920a3e8ac7842a2084718d394737317d1aa Mon Sep 17 00:00:00 2001 From: Duncan Overbruck Date: Tue, 9 Dec 2025 18:30:41 +0100 Subject: [PATCH 04/17] bin/xbps-rindex: implement multi repo staging --- bin/xbps-rindex/defs.h | 2 +- bin/xbps-rindex/index-add.c | 659 ++++++++++++++++++++++++++---------- bin/xbps-rindex/main.c | 3 +- 3 files changed, 480 insertions(+), 184 deletions(-) diff --git a/bin/xbps-rindex/defs.h b/bin/xbps-rindex/defs.h index 0f3063bc..d97fe356 100644 --- a/bin/xbps-rindex/defs.h +++ b/bin/xbps-rindex/defs.h @@ -31,7 +31,7 @@ #define _XBPS_RINDEX "xbps-rindex" /* From index-add.c */ -int index_add(struct xbps_handle *, int, int, char **, bool, const char *); +int index_add(struct xbps_handle *xhp, int argc, char **argv, bool force, const char *compression, xbps_array_t repos); /* From index-clean.c */ int index_clean(struct xbps_handle *, const char *, bool, const char *); diff --git a/bin/xbps-rindex/index-add.c b/bin/xbps-rindex/index-add.c index 727053b5..385d1ffd 100644 --- a/bin/xbps-rindex/index-add.c +++ b/bin/xbps-rindex/index-add.c @@ -39,166 +39,174 @@ #include #include "defs.h" +struct repo_state { + int lockfd; + const char *repodir; + const char *arch; + struct xbps_repo *repo; + xbps_dictionary_t index; + xbps_dictionary_t stage; + xbps_dictionary_t meta; +}; + +struct shared_state { + xbps_dictionary_t staged_shlibs; + xbps_dictionary_t used_shlibs; +}; + static int -repodata_commit(const char *repodir, const char *repoarch, - xbps_dictionary_t index, xbps_dictionary_t stage, xbps_dictionary_t meta, - const char *compression) +add_staged_shlib_providers(struct shared_state *shared, struct repo_state *state) { xbps_object_iterator_t iter; xbps_object_t keysym; - int r; - xbps_dictionary_t oldshlibs, usedshlibs; - if (xbps_dictionary_count(stage) == 0) - return 0; + iter = xbps_dictionary_iterator(state->stage); + if (!iter) + return xbps_error_oom(); - /* - * Find old shlibs-provides - */ - oldshlibs = xbps_dictionary_create(); - usedshlibs = xbps_dictionary_create(); - - iter = xbps_dictionary_iterator(stage); while ((keysym = xbps_object_iterator_next(iter))) { const char *pkgname = xbps_dictionary_keysym_cstring_nocopy(keysym); - xbps_dictionary_t pkg = xbps_dictionary_get(index, pkgname); + xbps_dictionary_t pkg = xbps_dictionary_get(state->index, pkgname); xbps_array_t pkgshlibs; pkgshlibs = xbps_dictionary_get(pkg, "shlib-provides"); for (unsigned int i = 0; i < xbps_array_count(pkgshlibs); i++) { const char *shlib = NULL; - xbps_array_get_cstring_nocopy(pkgshlibs, i, &shlib); - xbps_dictionary_set_cstring(oldshlibs, shlib, pkgname); + if (!xbps_array_get_cstring_nocopy(pkgshlibs, i, &shlib)) + abort(); + // fprintf(stderr, ">> %s = %s (%s)\n", shlib, pkgname, state->repodir); + if (!xbps_dictionary_set_cstring(shared->staged_shlibs, shlib, pkgname)) { + xbps_object_iterator_release(iter); + return xbps_error_oom(); + } } } + xbps_object_iterator_release(iter); + return 0; +} + +static int +dictionary_array_add_cstring(xbps_dictionary_t dict, const char *key, const char *value) +{ + xbps_array_t array; + bool alloc = false; + int r = 0; + + array = xbps_dictionary_get(dict, key); + if (!array) { + array = xbps_array_create(); + if (!xbps_dictionary_set(dict, key, array)) { + xbps_object_release(array); + return xbps_error_oom(); + } + alloc = true; + } + if (!xbps_array_add_cstring(array, value)) + r = xbps_error_oom(); + if (alloc) + xbps_object_release(array); + return r; +} + +static int +find_used_staged_shlibs(struct shared_state *shared, struct repo_state *state) +{ + xbps_object_iterator_t iter; + xbps_object_t keysym; + int r; + + iter = xbps_dictionary_iterator(state->index); + if (!iter) + return xbps_error_oom(); - /* - * throw away all unused shlibs - */ - iter = xbps_dictionary_iterator(index); while ((keysym = xbps_object_iterator_next(iter))) { const char *pkgname = xbps_dictionary_keysym_cstring_nocopy(keysym); - xbps_dictionary_t pkg = xbps_dictionary_get(stage, pkgname); + xbps_dictionary_t pkg; xbps_array_t pkgshlibs; + + pkg = xbps_dictionary_get(state->stage, pkgname); if (!pkg) - pkg = xbps_dictionary_get_keysym(index, keysym); - pkgshlibs = xbps_dictionary_get(pkg, "shlib-requires"); + pkg = xbps_dictionary_get_keysym(state->index, keysym); + pkgshlibs = xbps_dictionary_get(pkg, "shlib-requires"); for (unsigned int i = 0; i < xbps_array_count(pkgshlibs); i++) { const char *shlib = NULL; - bool alloc = false; - xbps_array_t users; - xbps_array_get_cstring_nocopy(pkgshlibs, i, &shlib); - if (!xbps_dictionary_get(oldshlibs, shlib)) + if (!xbps_array_get_cstring_nocopy(pkgshlibs, i, &shlib)) + abort(); + if (!xbps_dictionary_get(shared->staged_shlibs, shlib)) continue; - users = xbps_dictionary_get(usedshlibs, shlib); - if (!users) { - users = xbps_array_create(); - xbps_dictionary_set(usedshlibs, shlib, users); - alloc = true; + r = dictionary_array_add_cstring(shared->used_shlibs, shlib, pkgname); + if (r < 0) { + xbps_object_iterator_release(iter); + return r; } - xbps_array_add_cstring(users, pkgname); - if (alloc) - xbps_object_release(users); } } xbps_object_iterator_release(iter); + return 0; +} + +static int +purge_satisfied_by_index(struct shared_state *shared, struct repo_state *state) +{ + xbps_object_iterator_t iter; + xbps_object_t keysym; + + iter = xbps_dictionary_iterator(state->index); + if (!iter) + return xbps_error_oom(); - /* - * purge all packages that are fullfilled by the index and - * not in the stage. - */ - iter = xbps_dictionary_iterator(index); while ((keysym = xbps_object_iterator_next(iter))) { - xbps_dictionary_t pkg = xbps_dictionary_get_keysym(index, keysym); + xbps_dictionary_t pkg = xbps_dictionary_get_keysym(state->index, keysym); xbps_array_t pkgshlibs; + const char *pkgname = xbps_dictionary_keysym_cstring_nocopy(keysym); - - if (xbps_dictionary_get(stage, - xbps_dictionary_keysym_cstring_nocopy(keysym))) { + if (xbps_dictionary_get(state->stage, pkgname)) continue; - } pkgshlibs = xbps_dictionary_get(pkg, "shlib-provides"); for (unsigned int i = 0; i < xbps_array_count(pkgshlibs); i++) { const char *shlib = NULL; - xbps_array_get_cstring_nocopy(pkgshlibs, i, &shlib); - xbps_dictionary_remove(usedshlibs, shlib); + if (!xbps_array_get_cstring_nocopy(pkgshlibs, i, &shlib)) + abort(); + xbps_dictionary_remove(shared->used_shlibs, shlib); } } + xbps_object_iterator_release(iter); + return 0; +} + +static int +purge_satisfied_by_stage(struct shared_state *shared, struct repo_state *state) +{ + xbps_object_iterator_t iter; + xbps_object_t keysym; + + iter = xbps_dictionary_iterator(state->stage); + if (!iter) + return xbps_error_oom(); - /* - * purge all packages that are fullfilled by the stage - */ - iter = xbps_dictionary_iterator(stage); while ((keysym = xbps_object_iterator_next(iter))) { - xbps_dictionary_t pkg = xbps_dictionary_get_keysym(stage, keysym); + xbps_dictionary_t pkg = xbps_dictionary_get_keysym(state->stage, keysym); xbps_array_t pkgshlibs; pkgshlibs = xbps_dictionary_get(pkg, "shlib-provides"); for (unsigned int i = 0; i < xbps_array_count(pkgshlibs); i++) { const char *shlib = NULL; - xbps_array_get_cstring_nocopy(pkgshlibs, i, &shlib); - xbps_dictionary_remove(usedshlibs, shlib); - } - } - xbps_object_iterator_release(iter); - - if (xbps_dictionary_count(usedshlibs) != 0) { - printf("Inconsistent shlibs:\n"); - iter = xbps_dictionary_iterator(usedshlibs); - while ((keysym = xbps_object_iterator_next(iter))) { - const char *shlib = xbps_dictionary_keysym_cstring_nocopy(keysym), - *provider = NULL, *pre; - xbps_array_t users = xbps_dictionary_get(usedshlibs, shlib); - xbps_dictionary_get_cstring_nocopy(oldshlibs, shlib, &provider); - - printf(" %s (provided by: %s; used by: ", shlib, provider); - pre = ""; - for (unsigned int i = 0; i < xbps_array_count(users); i++) { - const char *user = NULL; - xbps_array_get_cstring_nocopy(users, i, &user); - printf("%s%s", pre, user); - pre = ", "; - } - printf(")\n"); + if (!xbps_array_get_cstring_nocopy(pkgshlibs, i, &shlib)) + abort(); + xbps_dictionary_remove(shared->used_shlibs, shlib); } - xbps_object_iterator_release(iter); - iter = xbps_dictionary_iterator(stage); - while ((keysym = xbps_object_iterator_next(iter))) { - xbps_dictionary_t pkg = xbps_dictionary_get_keysym(stage, keysym); - const char *pkgver = NULL, *arch = NULL; - xbps_dictionary_get_cstring_nocopy(pkg, "pkgver", &pkgver); - xbps_dictionary_get_cstring_nocopy(pkg, "architecture", &arch); - printf("stage: added `%s' (%s)\n", pkgver, arch); - } - xbps_object_iterator_release(iter); - } else { - iter = xbps_dictionary_iterator(stage); - while ((keysym = xbps_object_iterator_next(iter))) { - const char *pkgname = xbps_dictionary_keysym_cstring_nocopy(keysym); - xbps_dictionary_t pkg = xbps_dictionary_get_keysym(stage, keysym); - const char *pkgver = NULL, *arch = NULL; - xbps_dictionary_get_cstring_nocopy(pkg, "pkgver", &pkgver); - xbps_dictionary_get_cstring_nocopy(pkg, "architecture", &arch); - printf("index: added `%s' (%s).\n", pkgver, arch); - xbps_dictionary_set(index, pkgname, pkg); - } - xbps_object_iterator_release(iter); - stage = NULL; } - r = repodata_flush(repodir, repoarch, index, stage, meta, compression); - xbps_object_release(usedshlibs); - xbps_object_release(oldshlibs); - return r; + xbps_object_iterator_release(iter); + return 0; } static int -index_add_pkg(struct xbps_handle *xhp, xbps_dictionary_t index, xbps_dictionary_t stage, - const char *file, bool force) +index_add_pkg(struct repo_state *state, const char *file, bool force) { char sha256[XBPS_SHA256_SIZE]; char pkgname[XBPS_NAME_SIZE]; @@ -217,15 +225,33 @@ index_add_pkg(struct xbps_handle *xhp, xbps_dictionary_t index, xbps_dictionary_ "`%s', skipping!\n", XBPS_PKGPROPS, file); return 0; } - xbps_dictionary_get_cstring_nocopy(binpkgd, "architecture", &arch); - xbps_dictionary_get_cstring_nocopy(binpkgd, "pkgver", &pkgver); - if (!xbps_pkg_arch_match(xhp, arch, NULL)) { - fprintf(stderr, "index: ignoring %s, unmatched arch (%s)\n", pkgver, arch); - goto out; + + if (!xbps_dictionary_get_cstring_nocopy(binpkgd, "architecture", &arch)) { + xbps_error_printf( + "invalid binary packages: %s: missing property: " + "architecture\n", + file); + return -EINVAL; + } + if (!xbps_dictionary_get_cstring_nocopy(binpkgd, "pkgver", &pkgver)) { + xbps_error_printf( + "invalid binary packages: %s: missing property: pkgver\n", + file); + return -EINVAL; } + + if (strcmp(state->arch, arch) != 0 && strcmp("noarch", arch) != 0) { + xbps_warn_printf("ignoring %s, unmatched arch (%s)\n", pkgver, arch); + xbps_object_release(binpkgd); + return 0; + } + if (!xbps_pkg_name(pkgname, sizeof(pkgname), pkgver)) { - r = -EINVAL; - goto err; + xbps_error_printf( + "invalid binary package: %s: invalid property: pkgver: " + "%s\n", file, pkgver); + xbps_object_release(binpkgd); + return -EINVAL; } /* @@ -234,16 +260,18 @@ index_add_pkg(struct xbps_handle *xhp, xbps_dictionary_t index, xbps_dictionary_ * than current registered package, update the index; otherwise * pass to the next one. */ - curpkgd = xbps_dictionary_get(stage, pkgname); + curpkgd = xbps_dictionary_get(state->stage, pkgname); if (!curpkgd) - curpkgd = xbps_dictionary_get(index, pkgname); + curpkgd = xbps_dictionary_get(state->index, pkgname); if (curpkgd && !force) { const char *opkgver = NULL, *oarch = NULL; int cmp; - xbps_dictionary_get_cstring_nocopy(curpkgd, "pkgver", &opkgver); - xbps_dictionary_get_cstring_nocopy(curpkgd, "architecture", &oarch); + if (!xbps_dictionary_get_cstring_nocopy(curpkgd, "pkgver", &opkgver)) + abort(); + if (!xbps_dictionary_get_cstring_nocopy(curpkgd, "architecture", &oarch)) + abort(); cmp = xbps_cmpver(pkgver, opkgver); if (cmp < 0 && xbps_pkg_reverts(binpkgd, opkgver)) { @@ -261,18 +289,29 @@ index_add_pkg(struct xbps_handle *xhp, xbps_dictionary_t index, xbps_dictionary_ } if (cmp <= 0) { fprintf(stderr, "index: skipping `%s' (%s), already registered.\n", pkgver, arch); - goto out; + xbps_object_release(binpkgd); + return 0; } } - if (!xbps_file_sha256(sha256, sizeof(sha256), file)) - goto err_errno; - if (!xbps_dictionary_set_cstring(binpkgd, "filename-sha256", sha256)) - goto err_errno; - if (stat(file, &st) == -1) - goto err_errno; - if (!xbps_dictionary_set_uint64(binpkgd, "filename-size", (uint64_t)st.st_size)) - goto err_errno; + if (!xbps_file_sha256(sha256, sizeof(sha256), file)) { + r = -errno; + xbps_error_printf("failed to checksum binary package: %s: %s\n", file, strerror(-r)); + goto err; + } + if (!xbps_dictionary_set_cstring(binpkgd, "filename-sha256", sha256)) { + r = xbps_error_oom(); + goto err; + } + if (stat(file, &st) == -1) { + r = -errno; + xbps_error_printf("failed to stat binary package: %s: %s\n", file, strerror(-r)); + goto err; + } + if (!xbps_dictionary_set_uint64(binpkgd, "filename-size", (uint64_t)st.st_size)) { + r = xbps_error_oom(); + goto err; + } xbps_dictionary_remove(binpkgd, "pkgname"); xbps_dictionary_remove(binpkgd, "version"); @@ -281,86 +320,342 @@ index_add_pkg(struct xbps_handle *xhp, xbps_dictionary_t index, xbps_dictionary_ /* * Add new pkg dictionary into the stage index */ - if (!xbps_dictionary_set(stage, pkgname, binpkgd)) - goto err_errno; + if (!xbps_dictionary_set(state->stage, pkgname, binpkgd)) { + r = xbps_error_oom(); + goto err; + } -out: - xbps_object_release(binpkgd); - return 0; -err_errno: - r = -errno; + r = 0; err: xbps_object_release(binpkgd); return r; } -int -index_add(struct xbps_handle *xhp, int args, int argc, char **argv, bool force, const char *compression) +static int +repo_add_package( + struct repo_state *states, unsigned nstates, const char *file, bool force) { - xbps_dictionary_t index, stage, meta; - struct xbps_repo *repo; - char *tmprepodir = NULL, *repodir = NULL; - int lockfd; + char tmp[PATH_MAX]; + const char *dir; int r; - const char *repoarch = xhp->target_arch ? xhp->target_arch : xhp->native_arch; - if ((tmprepodir = strdup(argv[args])) == NULL) - return EXIT_FAILURE; - repodir = dirname(tmprepodir); + if (strlcpy(tmp, file, sizeof(tmp)) >= sizeof(tmp)) { + xbps_error_printf( + "failed to copy path: %s: %s\n", file, strerror(ENOBUFS)); + return -ENOBUFS; + } - lockfd = xbps_repo_lock(repodir, repoarch); - if (lockfd < 0) { - xbps_error_printf("xbps-rindex: cannot lock repository " - "%s: %s\n", repodir, strerror(-lockfd)); - free(tmprepodir); - return EXIT_FAILURE; + dir = dirname(tmp); + if (!dir) { + r = -errno; + xbps_error_printf("failed to get directory from path: %s: %s\n", + file, strerror(-r)); + return r; + } + + for (unsigned i = 0; i < nstates; i++) { + struct repo_state *state = &states[i]; + if (strcmp(state->repodir, dir) != 0) + continue; + r = index_add_pkg(state, file, force); + if (r < 0) + return r; + return 0; } - repo = xbps_repo_open(xhp, repodir); - if (!repo && errno != ENOENT) { - free(tmprepodir); + xbps_error_printf("repository not in repository list: %s\n", dir); + return -ENODEV; +} + +static int +repo_state_open(struct xbps_handle *xhp, struct repo_state *state, const char *repodir, const char *arch) +{ + state->repodir = repodir; + state->arch = arch; + state->lockfd = xbps_repo_lock(state->repodir, arch); + if (state->lockfd < 0) { + xbps_error_printf( + "failed to lock repository: %s: %s\n", + state->repodir, strerror(-state->lockfd)); return EXIT_FAILURE; } + state->repo = xbps_repo_open(xhp, repodir); + if (!state->repo) { + if (errno != ENOENT) + return EXIT_FAILURE; + state->index = xbps_dictionary_create(); + state->stage = xbps_dictionary_create(); + state->meta = NULL; + } else { + state->index = xbps_dictionary_copy_mutable(state->repo->index); + state->stage = xbps_dictionary_copy_mutable(state->repo->stage); + state->meta = xbps_dictionary_copy_mutable(state->repo->idxmeta); + } + return 0; +} + +static void +repo_state_release(struct repo_state *state) +{ + if (state->index) + xbps_object_release(state->index); + if (state->stage) + xbps_object_release(state->stage); + if (state->meta) + xbps_object_release(state->meta); + xbps_repo_release(state->repo); + xbps_repo_unlock(state->repodir, state->arch, state->lockfd); +} + +static int +print_inconsistent_shlibs(struct shared_state *shared) +{ + xbps_object_iterator_t iter; + xbps_object_t keysym; + + iter = xbps_dictionary_iterator(shared->used_shlibs); + if (!iter) + return xbps_error_oom(); + + printf("Inconsistent shlibs:\n"); + while ((keysym = xbps_object_iterator_next(iter))) { + const char *shlib = xbps_dictionary_keysym_cstring_nocopy(keysym); + const char *provider = NULL; + xbps_array_t users; + + if (!xbps_dictionary_get_cstring_nocopy(shared->staged_shlibs, shlib, &provider)) + abort(); + + users = xbps_dictionary_get(shared->used_shlibs, shlib); + printf(" %s (provided by: %s; used by: ", shlib, provider); + for (unsigned int i = 0; i < xbps_array_count(users); i++) { + const char *user = NULL; + if (!xbps_array_get_cstring_nocopy(users, i, &user)) + abort(); + printf("%s%s", i > 0 ? ", " : "", user); + } + printf(")\n"); + } + + xbps_object_iterator_release(iter); + return 0; +} + +static int +print_staged_packages(struct repo_state *states, unsigned nstates) +{ + xbps_object_iterator_t iter; + xbps_object_t keysym; + + // XXX: should this really print all staged packages every time? + + for (unsigned int i = 0; i < nstates; i++) { + const struct repo_state *state = &states[i]; + + iter = xbps_dictionary_iterator(state->stage); + if (!iter) + return xbps_error_oom(); + + while ((keysym = xbps_object_iterator_next(iter))) { + xbps_dictionary_t pkg = xbps_dictionary_get_keysym(state->stage, keysym); + const char *pkgver = NULL, *arch = NULL; + xbps_dictionary_get_cstring_nocopy(pkg, "pkgver", &pkgver); + xbps_dictionary_get_cstring_nocopy(pkg, "architecture", &arch); + printf("stage: added `%s' (%s)\n", pkgver, arch); + } + + xbps_object_iterator_release(iter); + } + + return 0; +} + +static int +unstage(struct repo_state *state) +{ + xbps_object_iterator_t iter; + xbps_object_t keysym; + + iter = xbps_dictionary_iterator(state->stage); + if (!iter) + return xbps_error_oom(); + + while ((keysym = xbps_object_iterator_next(iter))) { + const char *pkgver = NULL, *arch = NULL; + const char *pkgname = xbps_dictionary_keysym_cstring_nocopy(keysym); + xbps_dictionary_t pkg = xbps_dictionary_get_keysym(state->stage, keysym); + + if (!xbps_dictionary_get_cstring_nocopy(pkg, "pkgver", &pkgver)) + abort(); + if (!xbps_dictionary_get_cstring_nocopy(pkg, "architecture", &arch)) + abort(); + + printf("index: added `%s' (%s).\n", pkgver, arch); + + if (!xbps_dictionary_set(state->index, pkgname, pkg)) { + xbps_object_iterator_release(iter); + return xbps_error_oom(); + } + } + xbps_object_iterator_release(iter); - if (repo) { - index = xbps_dictionary_copy_mutable(repo->index); - stage = xbps_dictionary_copy_mutable(repo->stage); - meta = xbps_dictionary_copy_mutable(repo->idxmeta); + xbps_object_release(state->stage); + state->stage = NULL; + + return 0; +} + +static int +commit(struct repo_state *states, unsigned nstates) +{ + struct shared_state shared; + int r; + + shared.staged_shlibs = xbps_dictionary_create(); + if (!shared.staged_shlibs) + return xbps_error_oom(); + + shared.used_shlibs = xbps_dictionary_create(); + if (!shared.used_shlibs) { + xbps_object_release(shared.staged_shlibs); + return xbps_error_oom(); + } + + // collect all the used shared libraries in the stage + for (unsigned i = 0; i < nstates; i++) { + r = add_staged_shlib_providers(&shared, &states[i]); + if (r < 0) + return r; + r = find_used_staged_shlibs(&shared, &states[i]); + if (r < 0) + return r; + } + + // throw out shared libraries that are already satisfied + for (unsigned i = 0; i < nstates; i++) { + r = purge_satisfied_by_index(&shared, &states[i]); + if (r < 0) + return r; + r = purge_satisfied_by_stage(&shared, &states[i]); + if (r < 0) + return r; + } + + // ... now if there are libraries left, there is an inconsistency + if (xbps_dictionary_count(shared.used_shlibs) != 0) { + r = print_inconsistent_shlibs(&shared); + if (r < 0) + goto err; + r = print_staged_packages(states, nstates); + if (r < 0) + goto err; } else { - index = xbps_dictionary_create(); - stage = xbps_dictionary_create(); - meta = NULL; + for (unsigned i = 0; i < nstates; i++) { + r = unstage(&states[i]); + if (r < 0) + goto err; + } + } + + for (unsigned i = 0; i < nstates; i++) { + struct repo_state *state = &states[i]; + // XXX: kinda useless to print for all repos, also print stage?. + printf("index: %u packages registered.\n", xbps_dictionary_count(state->index)); + } + + r = 0; +err: + xbps_object_release(shared.staged_shlibs); + xbps_object_release(shared.used_shlibs); + return r; +} + +static int +repo_write(struct repo_state *state, const char *compression) +{ + int r; + + r = repodata_flush(state->repodir, state->arch, state->index, state->stage, state->meta, compression); + if (r < 0) + return r; + return 0; +} + +int +index_add(struct xbps_handle *xhp, int argc, char **argv, bool force, const char *compression, xbps_array_t repos) +{ + const char *arch = xhp->target_arch ? xhp->target_arch : xhp->native_arch; + struct repo_state *states; + unsigned nstates; + int r; + + if (!repos) { + char tmp[PATH_MAX]; + const char *repodir; + repos = xbps_array_create(); + if (!repos) { + xbps_error_oom(); + return EXIT_FAILURE; + } + if (strlcpy(tmp, argv[0], sizeof(tmp)) >= sizeof(tmp)) { + xbps_error_printf("failed to copy path: %s: %s\n", argv[0], + strerror(ENOBUFS)); + return EXIT_FAILURE; + } + repodir = dirname(tmp); + if (!repodir) { + xbps_error_printf("failed to get dirname: %s: %s\n", tmp, strerror(errno)); + xbps_object_release(repos); + return EXIT_FAILURE; + } + if (!xbps_array_add_cstring(repos, repodir)) { + xbps_error_oom(); + xbps_object_release(repos); + return EXIT_FAILURE; + } + } + + nstates = xbps_array_count(repos); + states = calloc(nstates, sizeof(*states)); + if (!states) { + xbps_error_oom(); + return EXIT_FAILURE; + } + + for (unsigned i = 0; i < nstates; i++) { + struct repo_state *state = &states[i]; + const char *repodir = NULL; + if (!xbps_array_get_cstring_nocopy(repos, i, &repodir)) + abort(); + r = repo_state_open(xhp, state, repodir, arch); + if (r < 0) + goto err; } - for (int i = args; i < argc; i++) { - r = index_add_pkg(xhp, index, stage, argv[i], force); + for (int i = 0; i < argc; i++) { + r = repo_add_package(states, nstates, argv[i], force); if (r < 0) - goto err2; + goto err; } - r = repodata_commit(repodir, repoarch, index, stage, meta, compression); - if (r < 0) { - xbps_error_printf("failed to write repodata: %s\n", strerror(-r)); - goto err2; + r = commit(states, nstates); + if (r < 0) + goto err; + + for (unsigned i = 0; i < nstates; i++) { + r = repo_write(&states[i], compression); + if (r < 0) + return r; } - printf("index: %u packages registered.\n", xbps_dictionary_count(index)); - xbps_object_release(index); - xbps_object_release(stage); - if (meta) - xbps_object_release(meta); - xbps_repo_release(repo); - xbps_repo_unlock(repodir, repoarch, lockfd); - free(tmprepodir); + for (unsigned i = 0; i < xbps_array_count(repos); i++) + repo_state_release(&states[i]); + free(states); + return EXIT_SUCCESS; -err2: - xbps_object_release(index); - xbps_object_release(stage); - if (meta) - xbps_object_release(meta); - xbps_repo_release(repo); - xbps_repo_unlock(repodir, repoarch, lockfd); - free(tmprepodir); +err: + for (unsigned i = 0; i < xbps_array_count(repos); i++) + repo_state_release(&states[i]); return EXIT_FAILURE; } diff --git a/bin/xbps-rindex/main.c b/bin/xbps-rindex/main.c index dfbacb0b..f3d5edc5 100644 --- a/bin/xbps-rindex/main.c +++ b/bin/xbps-rindex/main.c @@ -148,6 +148,7 @@ main(int argc, char **argv) /* NOTREACHED */ } } + if ((argc == optind) || (!add_mode && !clean_mode && !rm_mode && !sign_mode && !sign_pkg_mode)) { usage(true); @@ -172,7 +173,7 @@ main(int argc, char **argv) } if (add_mode) - rv = index_add(&xh, optind, argc, argv, force, compression); + rv = index_add(&xh, argc-optind, argv+optind, force, compression, repos); else if (clean_mode) rv = index_clean(&xh, argv[optind], hashcheck, compression); else if (rm_mode) From b5bff3f1ef3b64e3a9eb149bbb11c1ee249bd680 Mon Sep 17 00:00:00 2001 From: Duncan Overbruck Date: Tue, 9 Dec 2025 18:34:19 +0100 Subject: [PATCH 05/17] fixup! tests: cleanup staging test --- tests/xbps/xbps-rindex/add_test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/xbps/xbps-rindex/add_test.sh b/tests/xbps/xbps-rindex/add_test.sh index 81b6b110..d213f002 100644 --- a/tests/xbps/xbps-rindex/add_test.sh +++ b/tests/xbps/xbps-rindex/add_test.sh @@ -71,7 +71,7 @@ stage_body() { xbps-query -r ../root -i --repository=$PWD -L atf_check -o ignore -- xbps-create -A noarch -n foo-1.1_1 -s "foo pkg" --shlib-provides "libfoo.so.2" ../pkg_A - atf_check -e ignore -o match:"index: added \`foo-1.1_1'" -- valgrind xbps-rindex -d -a $PWD/*.xbps + atf_check -e ignore -o match:"index: added \`foo-1.1_1'" -- xbps-rindex -d -a $PWD/*.xbps atf_check -o inline:" 1 $PWD (RSA unsigned)\n" -- \ xbps-query -r ../root -i --repository=$PWD -L From 2a4a097cf9cc2dc30f2675e0d2036617e3f27e2b Mon Sep 17 00:00:00 2001 From: Duncan Overbruck Date: Tue, 9 Dec 2025 20:00:53 +0100 Subject: [PATCH 06/17] fixup! bin/xbps-rindex: implement multi repo staging --- bin/xbps-rindex/index-add.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/bin/xbps-rindex/index-add.c b/bin/xbps-rindex/index-add.c index 385d1ffd..aef5ed3b 100644 --- a/bin/xbps-rindex/index-add.c +++ b/bin/xbps-rindex/index-add.c @@ -30,6 +30,7 @@ #include #include #include +#include #include #include #include @@ -525,20 +526,20 @@ commit(struct repo_state *states, unsigned nstates) for (unsigned i = 0; i < nstates; i++) { r = add_staged_shlib_providers(&shared, &states[i]); if (r < 0) - return r; + goto err; r = find_used_staged_shlibs(&shared, &states[i]); if (r < 0) - return r; + goto err; } // throw out shared libraries that are already satisfied for (unsigned i = 0; i < nstates; i++) { r = purge_satisfied_by_index(&shared, &states[i]); if (r < 0) - return r; + goto err; r = purge_satisfied_by_stage(&shared, &states[i]); if (r < 0) - return r; + goto err; } // ... now if there are libraries left, there is an inconsistency From 7251df84c70fc931715117a2cae81088e9797832 Mon Sep 17 00:00:00 2001 From: Duncan Overbruck Date: Tue, 9 Dec 2025 21:21:18 +0100 Subject: [PATCH 07/17] fixup! bin/xbps-rindex: implement multi repo staging --- bin/xbps-rindex/index-add.c | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/bin/xbps-rindex/index-add.c b/bin/xbps-rindex/index-add.c index aef5ed3b..07314fba 100644 --- a/bin/xbps-rindex/index-add.c +++ b/bin/xbps-rindex/index-add.c @@ -462,7 +462,7 @@ print_staged_packages(struct repo_state *states, unsigned nstates) const char *pkgver = NULL, *arch = NULL; xbps_dictionary_get_cstring_nocopy(pkg, "pkgver", &pkgver); xbps_dictionary_get_cstring_nocopy(pkg, "architecture", &arch); - printf("stage: added `%s' (%s)\n", pkgver, arch); + printf("%s: stage: added `%s' (%s)\n", state->repodir, pkgver, arch); } xbps_object_iterator_release(iter); @@ -491,7 +491,8 @@ unstage(struct repo_state *state) if (!xbps_dictionary_get_cstring_nocopy(pkg, "architecture", &arch)) abort(); - printf("index: added `%s' (%s).\n", pkgver, arch); + printf("%s: index: added `%s' (%s).\n", state->repodir, + pkgver, arch); if (!xbps_dictionary_set(state->index, pkgname, pkg)) { xbps_object_iterator_release(iter); @@ -560,8 +561,13 @@ commit(struct repo_state *states, unsigned nstates) for (unsigned i = 0; i < nstates; i++) { struct repo_state *state = &states[i]; - // XXX: kinda useless to print for all repos, also print stage?. - printf("index: %u packages registered.\n", xbps_dictionary_count(state->index)); + printf("%s: index: %u packages registered.\n", + state->repodir, xbps_dictionary_count(state->index)); + if (xbps_dictionary_count(state->stage) != 0) { + printf("%s: stage: %u packages registered.\n", + state->repodir, + xbps_dictionary_count(state->stage)); + } } r = 0; From 472df8e1f3074253992db727dd710650fa02cd6b Mon Sep 17 00:00:00 2001 From: Duncan Overbruck Date: Tue, 9 Dec 2025 21:38:15 +0100 Subject: [PATCH 08/17] fixup! bin/xbps-rindex: implement multi repo staging --- bin/xbps-rindex/index-add.c | 65 ++++++++++++++++++++++++++----------- 1 file changed, 46 insertions(+), 19 deletions(-) diff --git a/bin/xbps-rindex/index-add.c b/bin/xbps-rindex/index-add.c index 07314fba..64143855 100644 --- a/bin/xbps-rindex/index-add.c +++ b/bin/xbps-rindex/index-add.c @@ -42,12 +42,14 @@ struct repo_state { int lockfd; + bool changed; const char *repodir; const char *arch; struct xbps_repo *repo; xbps_dictionary_t index; xbps_dictionary_t stage; xbps_dictionary_t meta; + xbps_array_t added; }; struct shared_state { @@ -326,6 +328,12 @@ index_add_pkg(struct repo_state *state, const char *file, bool force) goto err; } + state->changed = true; + if (!xbps_array_add_cstring(state->added, pkgname)) { + r = xbps_error_oom(); + goto err; + } + r = 0; err: xbps_object_release(binpkgd); @@ -371,6 +379,7 @@ repo_add_package( static int repo_state_open(struct xbps_handle *xhp, struct repo_state *state, const char *repodir, const char *arch) { + state->changed = false; state->repodir = repodir; state->arch = arch; state->lockfd = xbps_repo_lock(state->repodir, arch); @@ -378,19 +387,36 @@ repo_state_open(struct xbps_handle *xhp, struct repo_state *state, const char *r xbps_error_printf( "failed to lock repository: %s: %s\n", state->repodir, strerror(-state->lockfd)); - return EXIT_FAILURE; + return -state->lockfd; } + + state->added = xbps_array_create(); + if (!state->added) + return xbps_error_oom(); + state->repo = xbps_repo_open(xhp, repodir); if (!state->repo) { if (errno != ENOENT) - return EXIT_FAILURE; + return -errno; state->index = xbps_dictionary_create(); + if (!state->index) + return xbps_error_oom(); state->stage = xbps_dictionary_create(); + if (!state->stage) + return xbps_error_oom(); state->meta = NULL; } else { state->index = xbps_dictionary_copy_mutable(state->repo->index); + if (!state->index) + return xbps_error_oom(); state->stage = xbps_dictionary_copy_mutable(state->repo->stage); - state->meta = xbps_dictionary_copy_mutable(state->repo->idxmeta); + if (!state->stage) + return xbps_error_oom(); + if (state->repo->idxmeta) { + state->meta = xbps_dictionary_copy_mutable(state->repo->idxmeta); + if (!state->meta) + return xbps_error_oom(); + } } return 0; } @@ -445,27 +471,22 @@ print_inconsistent_shlibs(struct shared_state *shared) static int print_staged_packages(struct repo_state *states, unsigned nstates) { - xbps_object_iterator_t iter; - xbps_object_t keysym; - - // XXX: should this really print all staged packages every time? - for (unsigned int i = 0; i < nstates; i++) { const struct repo_state *state = &states[i]; - iter = xbps_dictionary_iterator(state->stage); - if (!iter) - return xbps_error_oom(); - - while ((keysym = xbps_object_iterator_next(iter))) { - xbps_dictionary_t pkg = xbps_dictionary_get_keysym(state->stage, keysym); - const char *pkgver = NULL, *arch = NULL; - xbps_dictionary_get_cstring_nocopy(pkg, "pkgver", &pkgver); - xbps_dictionary_get_cstring_nocopy(pkg, "architecture", &arch); + for (unsigned int j = 0; j < xbps_array_count(state->added); j++) { + xbps_dictionary_t pkg; + const char *pkgname = NULL, *pkgver = NULL, *arch = NULL; + if (!xbps_array_get_cstring_nocopy(state->added, j, &pkgname)) + abort(); + if (!xbps_dictionary_get_dict(state->stage, pkgname, &pkg)) + abort(); + if (!xbps_dictionary_get_cstring_nocopy(pkg, "pkgver", &pkgver)) + abort(); + if (!xbps_dictionary_get_cstring_nocopy(pkg, "architecture", &arch)) + abort(); printf("%s: stage: added `%s' (%s)\n", state->repodir, pkgver, arch); } - - xbps_object_iterator_release(iter); } return 0; @@ -477,6 +498,9 @@ unstage(struct repo_state *state) xbps_object_iterator_t iter; xbps_object_t keysym; + if (xbps_dictionary_count(state->stage) == 0) + return 0; + iter = xbps_dictionary_iterator(state->stage); if (!iter) return xbps_error_oom(); @@ -503,6 +527,7 @@ unstage(struct repo_state *state) xbps_object_release(state->stage); state->stage = NULL; + state->changed = true; return 0; } @@ -561,6 +586,8 @@ commit(struct repo_state *states, unsigned nstates) for (unsigned i = 0; i < nstates; i++) { struct repo_state *state = &states[i]; + if (!state->changed) + continue; printf("%s: index: %u packages registered.\n", state->repodir, xbps_dictionary_count(state->index)); if (xbps_dictionary_count(state->stage) != 0) { From 7bb570f0fd1adca52f0a49a9cc367fee468e2a34 Mon Sep 17 00:00:00 2001 From: Duncan Overbruck Date: Tue, 9 Dec 2025 22:12:44 +0100 Subject: [PATCH 09/17] fixup! bin/xbps-rindex: implement multi repo staging --- bin/xbps-rindex/index-add.c | 278 ++++++++++++++++++------------------ 1 file changed, 141 insertions(+), 137 deletions(-) diff --git a/bin/xbps-rindex/index-add.c b/bin/xbps-rindex/index-add.c index 64143855..ffa35c7c 100644 --- a/bin/xbps-rindex/index-add.c +++ b/bin/xbps-rindex/index-add.c @@ -40,12 +40,11 @@ #include #include "defs.h" -struct repo_state { +struct repo { int lockfd; bool changed; const char *repodir; const char *arch; - struct xbps_repo *repo; xbps_dictionary_t index; xbps_dictionary_t stage; xbps_dictionary_t meta; @@ -58,18 +57,18 @@ struct shared_state { }; static int -add_staged_shlib_providers(struct shared_state *shared, struct repo_state *state) +add_staged_shlib_providers(struct shared_state *state, struct repo *repo) { xbps_object_iterator_t iter; xbps_object_t keysym; - iter = xbps_dictionary_iterator(state->stage); + iter = xbps_dictionary_iterator(repo->stage); if (!iter) return xbps_error_oom(); while ((keysym = xbps_object_iterator_next(iter))) { const char *pkgname = xbps_dictionary_keysym_cstring_nocopy(keysym); - xbps_dictionary_t pkg = xbps_dictionary_get(state->index, pkgname); + xbps_dictionary_t pkg = xbps_dictionary_get(repo->index, pkgname); xbps_array_t pkgshlibs; pkgshlibs = xbps_dictionary_get(pkg, "shlib-provides"); @@ -77,8 +76,7 @@ add_staged_shlib_providers(struct shared_state *shared, struct repo_state *state const char *shlib = NULL; if (!xbps_array_get_cstring_nocopy(pkgshlibs, i, &shlib)) abort(); - // fprintf(stderr, ">> %s = %s (%s)\n", shlib, pkgname, state->repodir); - if (!xbps_dictionary_set_cstring(shared->staged_shlibs, shlib, pkgname)) { + if (!xbps_dictionary_set_cstring(state->staged_shlibs, shlib, pkgname)) { xbps_object_iterator_release(iter); return xbps_error_oom(); } @@ -113,13 +111,13 @@ dictionary_array_add_cstring(xbps_dictionary_t dict, const char *key, const char } static int -find_used_staged_shlibs(struct shared_state *shared, struct repo_state *state) +find_used_staged_shlibs(struct shared_state *state, struct repo *repo) { xbps_object_iterator_t iter; xbps_object_t keysym; int r; - iter = xbps_dictionary_iterator(state->index); + iter = xbps_dictionary_iterator(repo->index); if (!iter) return xbps_error_oom(); @@ -128,18 +126,18 @@ find_used_staged_shlibs(struct shared_state *shared, struct repo_state *state) xbps_dictionary_t pkg; xbps_array_t pkgshlibs; - pkg = xbps_dictionary_get(state->stage, pkgname); + pkg = xbps_dictionary_get(repo->stage, pkgname); if (!pkg) - pkg = xbps_dictionary_get_keysym(state->index, keysym); + pkg = xbps_dictionary_get_keysym(repo->index, keysym); pkgshlibs = xbps_dictionary_get(pkg, "shlib-requires"); for (unsigned int i = 0; i < xbps_array_count(pkgshlibs); i++) { const char *shlib = NULL; if (!xbps_array_get_cstring_nocopy(pkgshlibs, i, &shlib)) abort(); - if (!xbps_dictionary_get(shared->staged_shlibs, shlib)) + if (!xbps_dictionary_get(state->staged_shlibs, shlib)) continue; - r = dictionary_array_add_cstring(shared->used_shlibs, shlib, pkgname); + r = dictionary_array_add_cstring(state->used_shlibs, shlib, pkgname); if (r < 0) { xbps_object_iterator_release(iter); return r; @@ -151,21 +149,21 @@ find_used_staged_shlibs(struct shared_state *shared, struct repo_state *state) } static int -purge_satisfied_by_index(struct shared_state *shared, struct repo_state *state) +purge_satisfied_by_index(struct shared_state *state, struct repo *repo) { xbps_object_iterator_t iter; xbps_object_t keysym; - iter = xbps_dictionary_iterator(state->index); + iter = xbps_dictionary_iterator(repo->index); if (!iter) return xbps_error_oom(); while ((keysym = xbps_object_iterator_next(iter))) { - xbps_dictionary_t pkg = xbps_dictionary_get_keysym(state->index, keysym); + xbps_dictionary_t pkg = xbps_dictionary_get_keysym(repo->index, keysym); xbps_array_t pkgshlibs; const char *pkgname = xbps_dictionary_keysym_cstring_nocopy(keysym); - if (xbps_dictionary_get(state->stage, pkgname)) + if (xbps_dictionary_get(repo->stage, pkgname)) continue; pkgshlibs = xbps_dictionary_get(pkg, "shlib-provides"); @@ -173,7 +171,7 @@ purge_satisfied_by_index(struct shared_state *shared, struct repo_state *state) const char *shlib = NULL; if (!xbps_array_get_cstring_nocopy(pkgshlibs, i, &shlib)) abort(); - xbps_dictionary_remove(shared->used_shlibs, shlib); + xbps_dictionary_remove(state->used_shlibs, shlib); } } @@ -182,17 +180,17 @@ purge_satisfied_by_index(struct shared_state *shared, struct repo_state *state) } static int -purge_satisfied_by_stage(struct shared_state *shared, struct repo_state *state) +purge_satisfied_by_stage(struct shared_state *state, struct repo *repo) { xbps_object_iterator_t iter; xbps_object_t keysym; - iter = xbps_dictionary_iterator(state->stage); + iter = xbps_dictionary_iterator(repo->stage); if (!iter) return xbps_error_oom(); while ((keysym = xbps_object_iterator_next(iter))) { - xbps_dictionary_t pkg = xbps_dictionary_get_keysym(state->stage, keysym); + xbps_dictionary_t pkg = xbps_dictionary_get_keysym(repo->stage, keysym); xbps_array_t pkgshlibs; pkgshlibs = xbps_dictionary_get(pkg, "shlib-provides"); @@ -200,7 +198,7 @@ purge_satisfied_by_stage(struct shared_state *shared, struct repo_state *state) const char *shlib = NULL; if (!xbps_array_get_cstring_nocopy(pkgshlibs, i, &shlib)) abort(); - xbps_dictionary_remove(shared->used_shlibs, shlib); + xbps_dictionary_remove(state->used_shlibs, shlib); } } @@ -209,7 +207,7 @@ purge_satisfied_by_stage(struct shared_state *shared, struct repo_state *state) } static int -index_add_pkg(struct repo_state *state, const char *file, bool force) +repo_add_pkg(struct repo *repo, const char *file, bool force) { char sha256[XBPS_SHA256_SIZE]; char pkgname[XBPS_NAME_SIZE]; @@ -243,7 +241,7 @@ index_add_pkg(struct repo_state *state, const char *file, bool force) return -EINVAL; } - if (strcmp(state->arch, arch) != 0 && strcmp("noarch", arch) != 0) { + if (strcmp(repo->arch, arch) != 0 && strcmp("noarch", arch) != 0) { xbps_warn_printf("ignoring %s, unmatched arch (%s)\n", pkgver, arch); xbps_object_release(binpkgd); return 0; @@ -263,9 +261,9 @@ index_add_pkg(struct repo_state *state, const char *file, bool force) * than current registered package, update the index; otherwise * pass to the next one. */ - curpkgd = xbps_dictionary_get(state->stage, pkgname); + curpkgd = xbps_dictionary_get(repo->stage, pkgname); if (!curpkgd) - curpkgd = xbps_dictionary_get(state->index, pkgname); + curpkgd = xbps_dictionary_get(repo->index, pkgname); if (curpkgd && !force) { const char *opkgver = NULL, *oarch = NULL; @@ -323,13 +321,13 @@ index_add_pkg(struct repo_state *state, const char *file, bool force) /* * Add new pkg dictionary into the stage index */ - if (!xbps_dictionary_set(state->stage, pkgname, binpkgd)) { + if (!xbps_dictionary_set(repo->stage, pkgname, binpkgd)) { r = xbps_error_oom(); goto err; } - state->changed = true; - if (!xbps_array_add_cstring(state->added, pkgname)) { + repo->changed = true; + if (!xbps_array_add_cstring(repo->added, pkgname)) { r = xbps_error_oom(); goto err; } @@ -341,8 +339,7 @@ index_add_pkg(struct repo_state *state, const char *file, bool force) } static int -repo_add_package( - struct repo_state *states, unsigned nstates, const char *file, bool force) +find_repo(struct repo *repos, unsigned int nrepos, const char *file, struct repo **outp) { char tmp[PATH_MAX]; const char *dir; @@ -362,23 +359,24 @@ repo_add_package( return r; } - for (unsigned i = 0; i < nstates; i++) { - struct repo_state *state = &states[i]; - if (strcmp(state->repodir, dir) != 0) - continue; - r = index_add_pkg(state, file, force); - if (r < 0) - return r; - return 0; + for (unsigned i = 0; i < nrepos; i++) { + if (strcmp(repos[i].repodir, dir) == 0) { + *outp = &repos[i]; + return 1; + } } - xbps_error_printf("repository not in repository list: %s\n", dir); - return -ENODEV; + *outp = NULL; + xbps_error_printf("repository not found: %s\n", dir); + return -ENOENT; } static int -repo_state_open(struct xbps_handle *xhp, struct repo_state *state, const char *repodir, const char *arch) +repo_open(struct xbps_handle *xhp, struct repo *state, const char *repodir, const char *arch) { + struct xbps_repo *src; + int r; + state->changed = false; state->repodir = repodir; state->arch = arch; @@ -394,8 +392,27 @@ repo_state_open(struct xbps_handle *xhp, struct repo_state *state, const char *r if (!state->added) return xbps_error_oom(); - state->repo = xbps_repo_open(xhp, repodir); - if (!state->repo) { + src = xbps_repo_open(xhp, repodir); + if (src) { + state->index = xbps_dictionary_copy_mutable(src->index); + if (!state->index) { + r = xbps_error_oom(); + goto err; + } + state->stage = xbps_dictionary_copy_mutable(src->stage); + if (!state->stage) { + r = xbps_error_oom(); + goto err; + } + if (src->idxmeta) { + state->meta = xbps_dictionary_copy_mutable(src->idxmeta); + if (!state->meta) { + r = xbps_error_oom(); + goto err; + } + } + xbps_repo_release(src); + } else { if (errno != ENOENT) return -errno; state->index = xbps_dictionary_create(); @@ -405,42 +422,34 @@ repo_state_open(struct xbps_handle *xhp, struct repo_state *state, const char *r if (!state->stage) return xbps_error_oom(); state->meta = NULL; - } else { - state->index = xbps_dictionary_copy_mutable(state->repo->index); - if (!state->index) - return xbps_error_oom(); - state->stage = xbps_dictionary_copy_mutable(state->repo->stage); - if (!state->stage) - return xbps_error_oom(); - if (state->repo->idxmeta) { - state->meta = xbps_dictionary_copy_mutable(state->repo->idxmeta); - if (!state->meta) - return xbps_error_oom(); - } } return 0; +err: + xbps_repo_release(src); + return r; } static void -repo_state_release(struct repo_state *state) +repo_state_release(struct repo *repo) { - if (state->index) - xbps_object_release(state->index); - if (state->stage) - xbps_object_release(state->stage); - if (state->meta) - xbps_object_release(state->meta); - xbps_repo_release(state->repo); - xbps_repo_unlock(state->repodir, state->arch, state->lockfd); + if (repo->index) + xbps_object_release(repo->index); + if (repo->stage) + xbps_object_release(repo->stage); + if (repo->meta) + xbps_object_release(repo->meta); + if (repo->added) + xbps_object_release(repo->added); + xbps_repo_unlock(repo->repodir, repo->arch, repo->lockfd); } static int -print_inconsistent_shlibs(struct shared_state *shared) +print_inconsistent_shlibs(struct shared_state *state) { xbps_object_iterator_t iter; xbps_object_t keysym; - iter = xbps_dictionary_iterator(shared->used_shlibs); + iter = xbps_dictionary_iterator(state->used_shlibs); if (!iter) return xbps_error_oom(); @@ -450,10 +459,10 @@ print_inconsistent_shlibs(struct shared_state *shared) const char *provider = NULL; xbps_array_t users; - if (!xbps_dictionary_get_cstring_nocopy(shared->staged_shlibs, shlib, &provider)) + if (!xbps_dictionary_get_cstring_nocopy(state->staged_shlibs, shlib, &provider)) abort(); - users = xbps_dictionary_get(shared->used_shlibs, shlib); + users = xbps_dictionary_get(state->used_shlibs, shlib); printf(" %s (provided by: %s; used by: ", shlib, provider); for (unsigned int i = 0; i < xbps_array_count(users); i++) { const char *user = NULL; @@ -469,10 +478,10 @@ print_inconsistent_shlibs(struct shared_state *shared) } static int -print_staged_packages(struct repo_state *states, unsigned nstates) +print_staged_packages(struct repo *repos, unsigned nrepos) { - for (unsigned int i = 0; i < nstates; i++) { - const struct repo_state *state = &states[i]; + for (unsigned int i = 0; i < nrepos; i++) { + const struct repo *state = &repos[i]; for (unsigned int j = 0; j < xbps_array_count(state->added); j++) { xbps_dictionary_t pkg; @@ -493,7 +502,7 @@ print_staged_packages(struct repo_state *states, unsigned nstates) } static int -unstage(struct repo_state *state) +repo_unstage(struct repo *state) { xbps_object_iterator_t iter; xbps_object_t keysym; @@ -533,101 +542,89 @@ unstage(struct repo_state *state) } static int -commit(struct repo_state *states, unsigned nstates) +repos_check_stage(struct repo *repos, unsigned nrepos) { - struct shared_state shared; + struct shared_state state; int r; - shared.staged_shlibs = xbps_dictionary_create(); - if (!shared.staged_shlibs) + state.staged_shlibs = xbps_dictionary_create(); + if (!state.staged_shlibs) return xbps_error_oom(); - shared.used_shlibs = xbps_dictionary_create(); - if (!shared.used_shlibs) { - xbps_object_release(shared.staged_shlibs); + state.used_shlibs = xbps_dictionary_create(); + if (!state.used_shlibs) { + xbps_object_release(state.staged_shlibs); return xbps_error_oom(); } // collect all the used shared libraries in the stage - for (unsigned i = 0; i < nstates; i++) { - r = add_staged_shlib_providers(&shared, &states[i]); + for (unsigned i = 0; i < nrepos; i++) { + r = add_staged_shlib_providers(&state, &repos[i]); if (r < 0) goto err; - r = find_used_staged_shlibs(&shared, &states[i]); + r = find_used_staged_shlibs(&state, &repos[i]); if (r < 0) goto err; } // throw out shared libraries that are already satisfied - for (unsigned i = 0; i < nstates; i++) { - r = purge_satisfied_by_index(&shared, &states[i]); + for (unsigned i = 0; i < nrepos; i++) { + r = purge_satisfied_by_index(&state, &repos[i]); if (r < 0) goto err; - r = purge_satisfied_by_stage(&shared, &states[i]); + r = purge_satisfied_by_stage(&state, &repos[i]); if (r < 0) goto err; } // ... now if there are libraries left, there is an inconsistency - if (xbps_dictionary_count(shared.used_shlibs) != 0) { - r = print_inconsistent_shlibs(&shared); + if (xbps_dictionary_count(state.used_shlibs) != 0) { + r = print_inconsistent_shlibs(&state); if (r < 0) goto err; - r = print_staged_packages(states, nstates); + r = print_staged_packages(repos, nrepos); if (r < 0) goto err; } else { - for (unsigned i = 0; i < nstates; i++) { - r = unstage(&states[i]); + for (unsigned i = 0; i < nrepos; i++) { + r = repo_unstage(&repos[i]); if (r < 0) goto err; } } - for (unsigned i = 0; i < nstates; i++) { - struct repo_state *state = &states[i]; - if (!state->changed) + for (unsigned i = 0; i < nrepos; i++) { + struct repo *repo = &repos[i]; + if (!repo->changed) continue; printf("%s: index: %u packages registered.\n", - state->repodir, xbps_dictionary_count(state->index)); - if (xbps_dictionary_count(state->stage) != 0) { + repo->repodir, xbps_dictionary_count(repo->index)); + if (xbps_dictionary_count(repo->stage) != 0) { printf("%s: stage: %u packages registered.\n", - state->repodir, - xbps_dictionary_count(state->stage)); + repo->repodir, xbps_dictionary_count(repo->stage)); } } r = 0; err: - xbps_object_release(shared.staged_shlibs); - xbps_object_release(shared.used_shlibs); + xbps_object_release(state.staged_shlibs); + xbps_object_release(state.used_shlibs); return r; } -static int -repo_write(struct repo_state *state, const char *compression) -{ - int r; - - r = repodata_flush(state->repodir, state->arch, state->index, state->stage, state->meta, compression); - if (r < 0) - return r; - return 0; -} - int -index_add(struct xbps_handle *xhp, int argc, char **argv, bool force, const char *compression, xbps_array_t repos) +index_add(struct xbps_handle *xhp, int argc, char **argv, bool force, const char *compression, xbps_array_t repo_args) { const char *arch = xhp->target_arch ? xhp->target_arch : xhp->native_arch; - struct repo_state *states; - unsigned nstates; + struct repo *repos; + unsigned nrepos; int r; - if (!repos) { + if (!repo_args) { char tmp[PATH_MAX]; const char *repodir; - repos = xbps_array_create(); - if (!repos) { + repo_args = xbps_array_create(); + if (!repo_args) { xbps_error_oom(); return EXIT_FAILURE; } @@ -639,57 +636,64 @@ index_add(struct xbps_handle *xhp, int argc, char **argv, bool force, const char repodir = dirname(tmp); if (!repodir) { xbps_error_printf("failed to get dirname: %s: %s\n", tmp, strerror(errno)); - xbps_object_release(repos); + xbps_object_release(repo_args); return EXIT_FAILURE; } - if (!xbps_array_add_cstring(repos, repodir)) { + if (!xbps_array_add_cstring(repo_args, repodir)) { xbps_error_oom(); - xbps_object_release(repos); + xbps_object_release(repo_args); return EXIT_FAILURE; } } - nstates = xbps_array_count(repos); - states = calloc(nstates, sizeof(*states)); - if (!states) { + nrepos = xbps_array_count(repo_args); + repos = calloc(nrepos, sizeof(*repos)); + if (!repos) { xbps_error_oom(); return EXIT_FAILURE; } - for (unsigned i = 0; i < nstates; i++) { - struct repo_state *state = &states[i]; + for (unsigned i = 0; i < nrepos; i++) { const char *repodir = NULL; - if (!xbps_array_get_cstring_nocopy(repos, i, &repodir)) + if (!xbps_array_get_cstring_nocopy(repo_args, i, &repodir)) abort(); - r = repo_state_open(xhp, state, repodir, arch); + r = repo_open(xhp, &repos[i], repodir, arch); if (r < 0) goto err; } for (int i = 0; i < argc; i++) { - r = repo_add_package(states, nstates, argv[i], force); + struct repo *repo = NULL; + r = find_repo(repos, nrepos, argv[i], &repo); + if (r < 0) + goto err; + r = repo_add_pkg(repo, argv[i], force); if (r < 0) goto err; } - r = commit(states, nstates); + r = repos_check_stage(repos, nrepos); if (r < 0) goto err; - for (unsigned i = 0; i < nstates; i++) { - r = repo_write(&states[i], compression); + for (unsigned i = 0; i < nrepos; i++) { + struct repo *repo = &repos[i]; + if (!repo->changed) + continue; + r = repodata_flush(repo->repodir, repo->arch, repo->index, + repo->stage, repo->meta, compression); if (r < 0) - return r; + goto err; } - for (unsigned i = 0; i < xbps_array_count(repos); i++) - repo_state_release(&states[i]); - free(states); + for (unsigned i = 0; i < nrepos; i++) + repo_state_release(&repos[i]); + free(repos); return EXIT_SUCCESS; err: - for (unsigned i = 0; i < xbps_array_count(repos); i++) - repo_state_release(&states[i]); + for (unsigned i = 0; i < nrepos; i++) + repo_state_release(&repos[i]); return EXIT_FAILURE; } From 37f826ed3bb74ebfa602cbf7c01665313fd02ac3 Mon Sep 17 00:00:00 2001 From: Duncan Overbruck Date: Tue, 9 Dec 2025 22:33:03 +0100 Subject: [PATCH 10/17] fixup! bin/xbps-rindex: implement multi repo staging --- bin/xbps-rindex/index-add.c | 80 +++++++++++++++++++++++-------------- 1 file changed, 51 insertions(+), 29 deletions(-) diff --git a/bin/xbps-rindex/index-add.c b/bin/xbps-rindex/index-add.c index ffa35c7c..0c0e969f 100644 --- a/bin/xbps-rindex/index-add.c +++ b/bin/xbps-rindex/index-add.c @@ -338,6 +338,16 @@ repo_add_pkg(struct repo *repo, const char *file, bool force) return r; } +static const char * +safe_dirname(char *dst, size_t dstsz, const char *path) +{ + if (strlcpy(dst, path, dstsz) >= dstsz) { + errno = ENOBUFS; + return NULL; + } + return dirname(dst); +} + static int find_repo(struct repo *repos, unsigned int nrepos, const char *file, struct repo **outp) { @@ -345,13 +355,7 @@ find_repo(struct repo *repos, unsigned int nrepos, const char *file, struct repo const char *dir; int r; - if (strlcpy(tmp, file, sizeof(tmp)) >= sizeof(tmp)) { - xbps_error_printf( - "failed to copy path: %s: %s\n", file, strerror(ENOBUFS)); - return -ENOBUFS; - } - - dir = dirname(tmp); + dir = safe_dirname(tmp, sizeof(tmp), file); if (!dir) { r = -errno; xbps_error_printf("failed to get directory from path: %s: %s\n", @@ -612,6 +616,42 @@ repos_check_stage(struct repo *repos, unsigned nrepos) return r; } +static xbps_array_t +repos_from_argv(int argc, char **argv) +{ + char tmp[PATH_MAX]; + xbps_array_t res; + + res = xbps_array_create(); + if (!res) { + errno = -xbps_error_oom(); + return NULL; + } + + for (int i = 0; i < argc; i++) { + const char *dir; + int r; + dir = safe_dirname(tmp, sizeof(tmp), argv[0]); + if (!dir) { + r = -errno; + xbps_error_printf( + "failed to get dirname: %s: %s\n", argv[0], strerror(-r)); + xbps_object_release(res); + errno = -r; + return NULL; + } + if (xbps_match_string_in_array(res, dir)) + continue; + if (!xbps_array_add_cstring(res, dir)) { + xbps_object_release(res); + errno = -xbps_error_oom(); + return NULL; + } + } + + return res; +} + int index_add(struct xbps_handle *xhp, int argc, char **argv, bool force, const char *compression, xbps_array_t repo_args) { @@ -620,30 +660,12 @@ index_add(struct xbps_handle *xhp, int argc, char **argv, bool force, const char unsigned nrepos; int r; + // Backwards compatibiltiy if no repo args are given, in this case + // add the repos based on the supplied packages... if (!repo_args) { - char tmp[PATH_MAX]; - const char *repodir; - repo_args = xbps_array_create(); - if (!repo_args) { - xbps_error_oom(); + repo_args = repos_from_argv(argc, argv); + if (!repo_args) return EXIT_FAILURE; - } - if (strlcpy(tmp, argv[0], sizeof(tmp)) >= sizeof(tmp)) { - xbps_error_printf("failed to copy path: %s: %s\n", argv[0], - strerror(ENOBUFS)); - return EXIT_FAILURE; - } - repodir = dirname(tmp); - if (!repodir) { - xbps_error_printf("failed to get dirname: %s: %s\n", tmp, strerror(errno)); - xbps_object_release(repo_args); - return EXIT_FAILURE; - } - if (!xbps_array_add_cstring(repo_args, repodir)) { - xbps_error_oom(); - xbps_object_release(repo_args); - return EXIT_FAILURE; - } } nrepos = xbps_array_count(repo_args); From 496bf14e2b9669dfe255a7421de29d82f9741276 Mon Sep 17 00:00:00 2001 From: Duncan Overbruck Date: Tue, 9 Dec 2025 23:03:50 +0100 Subject: [PATCH 11/17] bin/xbps-rindex: write all repodata's to tempfiles before renaming them all --- bin/xbps-rindex/defs.h | 11 ++- bin/xbps-rindex/index-add.c | 28 ++++++- bin/xbps-rindex/index-clean.c | 2 +- bin/xbps-rindex/repoflush.c | 137 ++++++++++++++++++++-------------- bin/xbps-rindex/sign.c | 2 +- 5 files changed, 118 insertions(+), 62 deletions(-) diff --git a/bin/xbps-rindex/defs.h b/bin/xbps-rindex/defs.h index d97fe356..8ca90eac 100644 --- a/bin/xbps-rindex/defs.h +++ b/bin/xbps-rindex/defs.h @@ -45,8 +45,13 @@ int sign_repo(struct xbps_handle *, const char *, const char *, int sign_pkgs(struct xbps_handle *, int, int, char **, const char *, bool); /* From repoflush.c */ -int repodata_flush(const char *repodir, const char *arch, - xbps_dictionary_t index, xbps_dictionary_t stage, xbps_dictionary_t meta, - const char *compression); +int repodata_write_file(const char *repodir, const char *arch, + xbps_dictionary_t index, xbps_dictionary_t stage, xbps_dictionary_t meta, + const char *compression); +int repodata_write_tmpfile(char *tmp, size_t tmpsz, char *path, size_t pathsz, + const char *repodir, const char *arch, xbps_dictionary_t index, + xbps_dictionary_t stage, xbps_dictionary_t meta, const char *compression); +int repodata_write_fd(int fd, xbps_dictionary_t index, xbps_dictionary_t stage, xbps_dictionary_t meta, + const char *compression); #endif /* !_XBPS_RINDEX_DEFS_H_ */ diff --git a/bin/xbps-rindex/index-add.c b/bin/xbps-rindex/index-add.c index 0c0e969f..75dce7d7 100644 --- a/bin/xbps-rindex/index-add.c +++ b/bin/xbps-rindex/index-add.c @@ -1,5 +1,6 @@ /*- * Copyright (c) 2012-2015 Juan Romero Pardines. + * Copyright (c) 2025 Duncan Overbruck . * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -41,6 +42,8 @@ #include "defs.h" struct repo { + char path[PATH_MAX]; + char tmp[PATH_MAX]; int lockfd; bool changed; const char *repodir; @@ -698,16 +701,32 @@ index_add(struct xbps_handle *xhp, int argc, char **argv, bool force, const char if (r < 0) goto err; + // write all changed repodata's to tempfiles for (unsigned i = 0; i < nrepos; i++) { struct repo *repo = &repos[i]; if (!repo->changed) continue; - r = repodata_flush(repo->repodir, repo->arch, repo->index, - repo->stage, repo->meta, compression); + r = repodata_write_tmpfile(repo->path, sizeof(repo->path), + repo->tmp, sizeof(repo->tmp), repo->repodir, repo->arch, + repo->index, repo->stage, repo->meta, compression); if (r < 0) goto err; } + // rename all changed repodata's tempfiles + for (unsigned i = 0; i < nrepos; i++) { + struct repo *repo = &repos[i]; + if (!repo->changed) + continue; + if (rename(repo->tmp, repo->path) == -2) { + xbps_error_printf("failed to rename tempfile: %s: %s: %s\n", + repo->tmp, repo->path, strerror(-errno)); + // XXX: maybe better to abort here, but either way + // we'll end up with inconsistent staging... + } + + } + for (unsigned i = 0; i < nrepos; i++) repo_state_release(&repos[i]); free(repos); @@ -715,7 +734,10 @@ index_add(struct xbps_handle *xhp, int argc, char **argv, bool force, const char return EXIT_SUCCESS; err: - for (unsigned i = 0; i < nrepos; i++) + for (unsigned i = 0; i < nrepos; i++) { + if (repos[i].tmp[0] != 0) + unlink(repos[i].tmp); repo_state_release(&repos[i]); + } return EXIT_FAILURE; } diff --git a/bin/xbps-rindex/index-clean.c b/bin/xbps-rindex/index-clean.c index 27a105aa..ed0992d4 100644 --- a/bin/xbps-rindex/index-clean.c +++ b/bin/xbps-rindex/index-clean.c @@ -134,7 +134,7 @@ cleanup_repo(struct xbps_handle *xhp, const char *repodir, struct xbps_repo *rep return 0; } - r = repodata_flush(repodir, repoarch, index, stage, repo->idxmeta, compression); + r = repodata_write_file(repodir, repoarch, index, stage, repo->idxmeta, compression); if (r < 0) { xbps_error_printf("failed to write repodata: %s\n", strerror(-r)); xbps_object_release(index); diff --git a/bin/xbps-rindex/repoflush.c b/bin/xbps-rindex/repoflush.c index 3bdd474c..904e0181 100644 --- a/bin/xbps-rindex/repoflush.c +++ b/bin/xbps-rindex/repoflush.c @@ -1,6 +1,6 @@ /*- * Copyright (c) 2013-2019 Juan Romero Pardines. - * Copyright (c) 2023 Duncan Overbruck . + * Copyright (c) 2023-2025 Duncan Overbruck . * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -131,48 +131,17 @@ archive_dict(struct archive *ar, const char *filename, xbps_dictionary_t dict) } int -repodata_flush(const char *repodir, - const char *arch, - xbps_dictionary_t index, - xbps_dictionary_t stage, - xbps_dictionary_t meta, - const char *compression) +repodata_write_fd(int fd, xbps_dictionary_t index, xbps_dictionary_t stage, + xbps_dictionary_t meta, const char *compression) { - char path[PATH_MAX]; - char tmp[PATH_MAX]; - struct archive *ar = NULL; - mode_t prevumask; + struct archive *ar; int r; - int fd; - - r = snprintf(path, sizeof(path), "%s/%s-repodata", repodir, arch); - if (r < 0 || (size_t)r >= sizeof(tmp)) { - xbps_error_printf("repodata path too long: %s: %s\n", path, - strerror(ENAMETOOLONG)); - return -ENAMETOOLONG; - } - - r = snprintf(tmp, sizeof(tmp), "%s.XXXXXXX", path); - if (r < 0 || (size_t)r >= sizeof(tmp)) { - xbps_error_printf("repodata tmp path too long: %s: %s\n", path, - strerror(ENAMETOOLONG)); - return -ENAMETOOLONG; - } - - prevumask = umask(S_IXUSR|S_IRWXG|S_IRWXO); - fd = mkstemp(tmp); - if (fd == -1) { - r = -errno; - xbps_error_printf("failed to open temp file: %s: %s", tmp, strerror(-r)); - umask(prevumask); - goto err; - } - umask(prevumask); ar = open_archive(fd, compression); if (!ar) { r = -errno; - goto err; + xbps_error_printf("failed to open archive: %s\n", strerror(-r)); + return r; } r = archive_dict(ar, XBPS_REPODATA_INDEX, index); @@ -185,18 +154,20 @@ repodata_flush(const char *repodir, if (r < 0) goto err; - /* Write data to tempfile and rename */ + if (archive_write_close(ar) == ARCHIVE_FATAL) { r = -archive_errno(ar); - if (r == 1) - r = -EINVAL; - xbps_error_printf("failed to close archive: %s\n", archive_error_string(ar)); - goto err; + xbps_error_printf( + "failed to close archive: %s\n", archive_error_string(ar)); + archive_write_free(ar); + return r; } if (archive_write_free(ar) == ARCHIVE_FATAL) { r = -errno; - xbps_error_printf("failed to free archive: %s\n", strerror(-r)); - goto err; + xbps_error_printf( + "failed to free archive: %s\n", strerror(-r)); + archive_write_free(ar); + return r; } #ifdef HAVE_FDATASYNC @@ -204,6 +175,51 @@ repodata_flush(const char *repodir, #else fsync(fd); #endif + return 0; + +err: + archive_write_free(ar); + return r; +} + +int +repodata_write_tmpfile(char *path, size_t pathsz, char *tmp, size_t tmpsz, + const char *repodir, const char *arch, xbps_dictionary_t index, + xbps_dictionary_t stage, xbps_dictionary_t meta, const char *compression) +{ + mode_t prevumask; + int fd; + int r; + + r = snprintf(path, pathsz, "%s/%s-repodata", repodir, arch); + if (r < 0 || (size_t)r >= pathsz) { + xbps_error_printf("repodata path too long: %s: %s\n", path, + strerror(ENAMETOOLONG)); + return -ENAMETOOLONG; + } + + r = snprintf(tmp, tmpsz, "%s.XXXXXXX", path); + if (r < 0 || (size_t)r >= tmpsz) { + xbps_error_printf("repodata tmp path too long: %s: %s\n", path, + strerror(ENAMETOOLONG)); + return -ENAMETOOLONG; + } + + prevumask = umask(S_IXUSR|S_IRWXG|S_IRWXO); + + fd = mkstemp(tmp); + if (fd == -1) { + r = -errno; + xbps_error_printf("failed to open temp file: %s: %s", tmp, strerror(-r)); + umask(prevumask); + goto err; + } + + umask(prevumask); + + repodata_write_fd(fd, index, stage, meta, compression); + if (r < 0) + goto err; if (fchmod(fd, 0664) == -1) { errno = -r; @@ -215,6 +231,28 @@ repodata_flush(const char *repodir, } close(fd); + return 0; +err: + if (fd != -1) + close(fd); + unlink(tmp); + return r; +} + +int +repodata_write_file(const char *repodir, const char *arch, + xbps_dictionary_t index, xbps_dictionary_t stage, xbps_dictionary_t meta, + const char *compression) +{ + char path[PATH_MAX]; + char tmp[PATH_MAX]; + int r; + + r = repodata_write_tmpfile(path, sizeof(path), tmp, sizeof(tmp), repodir, + arch, index, stage, meta, compression); + if (r < 0) + return r; + if (rename(tmp, path) == -1) { r = -errno; xbps_error_printf("failed to rename repodata: %s: %s: %s\n", @@ -222,15 +260,6 @@ repodata_flush(const char *repodir, unlink(tmp); return r; } - return 0; -err: - if (ar) { - archive_write_close(ar); - archive_write_free(ar); - } - if (fd != -1) - close(fd); - unlink(tmp); - return r; + return 0; } diff --git a/bin/xbps-rindex/sign.c b/bin/xbps-rindex/sign.c index 4acb203b..35797e42 100644 --- a/bin/xbps-rindex/sign.c +++ b/bin/xbps-rindex/sign.c @@ -230,7 +230,7 @@ sign_repo(struct xbps_handle *xhp, const char *repodir, xbps_error_printf("cannot lock repository: %s\n", strerror(errno)); goto out; } - r = repodata_flush(repodir, repoarch, repo->index, repo->stage, meta, compression); + r = repodata_write_file(repodir, repoarch, repo->index, repo->stage, meta, compression); xbps_repo_unlock(repodir, repoarch, lockfd); if (r < 0) { xbps_error_printf("failed to write repodata: %s\n", strerror(errno)); From 934b9e92d8c79e0bde2331b34c40e75ef9b9451f Mon Sep 17 00:00:00 2001 From: Duncan Overbruck Date: Tue, 9 Dec 2025 23:04:42 +0100 Subject: [PATCH 12/17] fixup! bin/xbps-rindex: implement multi repo staging --- tests/xbps/xbps-rindex/add_test.sh | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/tests/xbps/xbps-rindex/add_test.sh b/tests/xbps/xbps-rindex/add_test.sh index d213f002..7a590259 100644 --- a/tests/xbps/xbps-rindex/add_test.sh +++ b/tests/xbps/xbps-rindex/add_test.sh @@ -207,11 +207,13 @@ stage_multi_repos_body() { atf_check -o ignore -- xbps-create -A noarch -n ruby-3.3.8_1 -s "ruby pkg" --shlib-provides "libruby.so.3.3" ../pkg atf_check -o ignore -- xbps-create -A noarch -n A-1.0_1 -s "A pkg" --shlib-requires "libFLAC.so.12" ../pkg atf_check -o ignore -- xbps-create -A noarch -n B-1.0_1 -s "B pkg" --shlib-requires "libruby.so.3.3" ../pkg + atf_check -o ignore -- xbps-create -A noarch -n unaffected-1.0_1 -s "unaffected pkg" --shlib-provides "libfoo.so.1" ../pkg cd .. cd repo2 atf_check -o ignore -- xbps-create -A noarch -n AA-1.0_1 -s "AA pkg" --shlib-requires "libFLAC.so.12" ../pkg atf_check -o ignore -- xbps-create -A noarch -n BB-1.0_1 -s "BB pkg" --shlib-requires "libruby.so.3.3" ../pkg + atf_check -o ignore -- xbps-create -A noarch -n other-unaffected-1.0_1 -s "other unaffected pkg" --shlib-requires "libfoo.so.1" ../pkg cd .. atf_check -e ignore \ @@ -223,19 +225,21 @@ stage_multi_repos_body() { repo1/flac-1.4.3_1.noarch.xbps \ repo1/ruby-3.3.8_1.noarch.xbps \ repo1/A-1.0_1.noarch.xbps \ - repo1/B-1.0_1.noarch.xbps + repo1/B-1.0_1.noarch.xbps \ + repo1/unaffected-1.0_1.noarch.xbps atf_check -e ignore \ -o match:"index: added \`AA-1.0_1'" \ -o match:"index: added \`BB-1.0_1'" \ -- xbps-rindex -v -R repo1 -R repo2 -a \ repo2/AA-1.0_1.noarch.xbps \ - repo2/BB-1.0_1.noarch.xbps + repo2/BB-1.0_1.noarch.xbps \ + repo2/other-unaffected-1.0_1.noarch.xbps cd repo1 atf_check -o ignore -- xbps-create -A noarch -n flac-1.5.0_1 -s "flac pkg" --shlib-provides "libFLAC.so.14" ../pkg cd .. - atf_check -e ignore -o match:"stage: added \`flac-1.5.0_1'" \ + atf_check -e ignore -o match:"repo1: stage: added \`flac-1.5.0_1'" \ -- xbps-rindex -v -R repo1 -R repo2 -a repo1/flac-1.5.0_1.noarch.xbps atf_check \ @@ -247,7 +251,7 @@ stage_multi_repos_body() { atf_check -o ignore -- xbps-create -A noarch -n AA-1.0_2 -s "AA pkg" --shlib-requires "libFLAC.so.14" ../pkg cd .. - atf_check -o match:"stage: added \`AA-1.0_2'" \ + atf_check -o match:"repo2: stage: added \`AA-1.0_2'" \ -- xbps-rindex -v -R repo1 -R repo2 -a repo2/AA-1.0_2.noarch.xbps atf_check \ @@ -258,7 +262,7 @@ stage_multi_repos_body() { cd repo1 atf_check -o ignore -- xbps-create -A noarch -n ruby-3.4.5_1 -s "ruby pkg" --shlib-provides "libruby.so.3.4" ../pkg cd .. - atf_check -o match:"stage: added \`ruby-3.4.5_1'" \ + atf_check -o match:"repo1: stage: added \`ruby-3.4.5_1'" \ -- xbps-rindex -v -R repo1 -R repo2 -a repo1/ruby-3.4.5_1.noarch.xbps atf_check \ @@ -269,7 +273,7 @@ stage_multi_repos_body() { cd repo2 atf_check -o ignore -- xbps-create -A noarch -n BB-1.0_2 -s "BB pkg" --shlib-requires "libruby.so.3.4" ../pkg cd .. - atf_check -o match:"stage: added \`BB-1.0_2'" \ + atf_check -o match:"repo2: stage: added \`BB-1.0_2'" \ -- xbps-rindex -v -R repo1 -R repo2 -a repo2/BB-1.0_2.noarch.xbps cd repo1 @@ -277,7 +281,7 @@ stage_multi_repos_body() { atf_check -o ignore -- xbps-create -A noarch -n B-1.0_2 -s "B pkg" --shlib-requires "libruby.so.3.4" ../pkg cd .. atf_check \ - -o match:"stage: added \`A-1.0_2'" \ + -o match:"repo1: stage: added \`A-1.0_2'" \ -- xbps-rindex -v -R repo1 -R repo2 -a repo1/A-1.0_2.noarch.xbps atf_check \ @@ -286,12 +290,12 @@ stage_multi_repos_body() { -- xbps-query -r ../root -i --repository=repo1 --repository=repo2 -L atf_check \ - -o match:"index: added \`A-1\.0_2'" \ - -o match:"index: added \`B-1\.0_2'" \ - -o match:"index: added \`flac-1\.5\.0_1'" \ - -o match:"index: added \`ruby-3\.4\.5_1'" \ - -o match:"index: added \`AA-1\.0_2'" \ - -o match:"index: added \`BB-1\.0_2'" \ + -o match:"repo1: index: added \`A-1\.0_2'" \ + -o match:"repo1: index: added \`B-1\.0_2'" \ + -o match:"repo1: index: added \`flac-1\.5\.0_1'" \ + -o match:"repo1: index: added \`ruby-3\.4\.5_1'" \ + -o match:"repo2: index: added \`AA-1\.0_2'" \ + -o match:"repo2: index: added \`BB-1\.0_2'" \ -- xbps-rindex -v -R repo1 -R repo2 -a repo1/B-1.0_2.noarch.xbps atf_check \ From 812897307f1e72a5112bc466a76eb7e668f326fa Mon Sep 17 00:00:00 2001 From: Duncan Overbruck Date: Tue, 9 Dec 2025 23:09:26 +0100 Subject: [PATCH 13/17] fixup! bin/xbps-rindex: write all repodata's to tempfiles before renaming them all --- bin/xbps-rindex/repoflush.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/bin/xbps-rindex/repoflush.c b/bin/xbps-rindex/repoflush.c index 904e0181..a1d3ba90 100644 --- a/bin/xbps-rindex/repoflush.c +++ b/bin/xbps-rindex/repoflush.c @@ -154,7 +154,6 @@ repodata_write_fd(int fd, xbps_dictionary_t index, xbps_dictionary_t stage, if (r < 0) goto err; - if (archive_write_close(ar) == ARCHIVE_FATAL) { r = -archive_errno(ar); xbps_error_printf( @@ -217,7 +216,7 @@ repodata_write_tmpfile(char *path, size_t pathsz, char *tmp, size_t tmpsz, umask(prevumask); - repodata_write_fd(fd, index, stage, meta, compression); + r = repodata_write_fd(fd, index, stage, meta, compression); if (r < 0) goto err; From d12a09de28cd3dd36dec04681f0343b02a4f766c Mon Sep 17 00:00:00 2001 From: Duncan Overbruck Date: Thu, 11 Dec 2025 18:26:09 +0100 Subject: [PATCH 14/17] fixup! bin/xbps-rindex: implement multi repo staging --- bin/xbps-rindex/index-add.c | 126 +++++++++++++++++++++++++----------- 1 file changed, 87 insertions(+), 39 deletions(-) diff --git a/bin/xbps-rindex/index-add.c b/bin/xbps-rindex/index-add.c index 75dce7d7..44530c65 100644 --- a/bin/xbps-rindex/index-add.c +++ b/bin/xbps-rindex/index-add.c @@ -41,6 +41,15 @@ #include #include "defs.h" +static void __attribute__((__noreturn__)) +unreachable_log_abort(const char *file, size_t line) +{ + xbps_error_printf("%s:%zu: code should not be reached\n", file, line); + abort(); +} + +#define unreachable() unreachable_log_abort(__FILE__, __LINE__) + struct repo { char path[PATH_MAX]; char tmp[PATH_MAX]; @@ -55,12 +64,15 @@ struct repo { }; struct shared_state { - xbps_dictionary_t staged_shlibs; + struct repo *repos; + unsigned int nrepos; + + xbps_dictionary_t old_shlibs; xbps_dictionary_t used_shlibs; }; static int -add_staged_shlib_providers(struct shared_state *state, struct repo *repo) +add_old_shlibs(struct shared_state *state, struct repo *repo) { xbps_object_iterator_t iter; xbps_object_t keysym; @@ -69,22 +81,47 @@ add_staged_shlib_providers(struct shared_state *state, struct repo *repo) if (!iter) return xbps_error_oom(); + // record all shlibs from the old version of the staged package while ((keysym = xbps_object_iterator_next(iter))) { const char *pkgname = xbps_dictionary_keysym_cstring_nocopy(keysym); - xbps_dictionary_t pkg = xbps_dictionary_get(repo->index, pkgname); - xbps_array_t pkgshlibs; + xbps_dictionary_t oldpkg; + xbps_array_t oldshlibs; - pkgshlibs = xbps_dictionary_get(pkg, "shlib-provides"); - for (unsigned int i = 0; i < xbps_array_count(pkgshlibs); i++) { + oldpkg = xbps_dictionary_get(repo->index, pkgname); + if (!oldpkg) + continue; + + oldshlibs = xbps_dictionary_get(oldpkg, "shlib-provides"); + for (unsigned int i = 0; i < xbps_array_count(oldshlibs); i++) { const char *shlib = NULL; - if (!xbps_array_get_cstring_nocopy(pkgshlibs, i, &shlib)) - abort(); - if (!xbps_dictionary_set_cstring(state->staged_shlibs, shlib, pkgname)) { + if (!xbps_array_get_cstring_nocopy(oldshlibs, i, &shlib)) + unreachable(); + if (!xbps_dictionary_set_cstring(state->old_shlibs, shlib, pkgname)) { xbps_object_iterator_release(iter); return xbps_error_oom(); } } } + xbps_object_iterator_reset(iter); + + // remove all shlibs that are still provided by a staged package + while ((keysym = xbps_object_iterator_next(iter))) { + // const char *pkgname = xbps_dictionary_keysym_cstring_nocopy(keysym); + xbps_dictionary_t newpkg; + xbps_array_t newshlibs; + + newpkg = xbps_dictionary_get_keysym(repo->stage, keysym); + if (!newpkg) + unreachable(); + + newshlibs = xbps_dictionary_get(newpkg, "shlib-provides"); + for (unsigned int i = 0; i < xbps_array_count(newshlibs); i++) { + const char *shlib = NULL; + if (!xbps_array_get_cstring_nocopy(newshlibs, i, &shlib)) + unreachable(); + xbps_dictionary_remove(state->old_shlibs, shlib); + } + } xbps_object_iterator_release(iter); return 0; @@ -97,9 +134,13 @@ dictionary_array_add_cstring(xbps_dictionary_t dict, const char *key, const char bool alloc = false; int r = 0; + assert(dict); + array = xbps_dictionary_get(dict, key); if (!array) { array = xbps_array_create(); + if (!array) + return xbps_error_oom(); if (!xbps_dictionary_set(dict, key, array)) { xbps_object_release(array); return xbps_error_oom(); @@ -127,18 +168,18 @@ find_used_staged_shlibs(struct shared_state *state, struct repo *repo) while ((keysym = xbps_object_iterator_next(iter))) { const char *pkgname = xbps_dictionary_keysym_cstring_nocopy(keysym); xbps_dictionary_t pkg; - xbps_array_t pkgshlibs; + xbps_array_t shlibreqs; pkg = xbps_dictionary_get(repo->stage, pkgname); if (!pkg) pkg = xbps_dictionary_get_keysym(repo->index, keysym); - pkgshlibs = xbps_dictionary_get(pkg, "shlib-requires"); - for (unsigned int i = 0; i < xbps_array_count(pkgshlibs); i++) { + shlibreqs = xbps_dictionary_get(pkg, "shlib-requires"); + for (unsigned int i = 0; i < xbps_array_count(shlibreqs); i++) { const char *shlib = NULL; - if (!xbps_array_get_cstring_nocopy(pkgshlibs, i, &shlib)) - abort(); - if (!xbps_dictionary_get(state->staged_shlibs, shlib)) + if (!xbps_array_get_cstring_nocopy(shlibreqs, i, &shlib)) + unreachable(); + if (!xbps_dictionary_get(state->old_shlibs, shlib)) continue; r = dictionary_array_add_cstring(state->used_shlibs, shlib, pkgname); if (r < 0) { @@ -151,7 +192,7 @@ find_used_staged_shlibs(struct shared_state *state, struct repo *repo) return 0; } -static int +static int UNUSED purge_satisfied_by_index(struct shared_state *state, struct repo *repo) { xbps_object_iterator_t iter; @@ -173,7 +214,7 @@ purge_satisfied_by_index(struct shared_state *state, struct repo *repo) for (unsigned int i = 0; i < xbps_array_count(pkgshlibs); i++) { const char *shlib = NULL; if (!xbps_array_get_cstring_nocopy(pkgshlibs, i, &shlib)) - abort(); + unreachable(); xbps_dictionary_remove(state->used_shlibs, shlib); } } @@ -182,7 +223,7 @@ purge_satisfied_by_index(struct shared_state *state, struct repo *repo) return 0; } -static int +static int UNUSED purge_satisfied_by_stage(struct shared_state *state, struct repo *repo) { xbps_object_iterator_t iter; @@ -200,7 +241,7 @@ purge_satisfied_by_stage(struct shared_state *state, struct repo *repo) for (unsigned int i = 0; i < xbps_array_count(pkgshlibs); i++) { const char *shlib = NULL; if (!xbps_array_get_cstring_nocopy(pkgshlibs, i, &shlib)) - abort(); + unreachable(); xbps_dictionary_remove(state->used_shlibs, shlib); } } @@ -273,9 +314,9 @@ repo_add_pkg(struct repo *repo, const char *file, bool force) int cmp; if (!xbps_dictionary_get_cstring_nocopy(curpkgd, "pkgver", &opkgver)) - abort(); + unreachable(); if (!xbps_dictionary_get_cstring_nocopy(curpkgd, "architecture", &oarch)) - abort(); + unreachable(); cmp = xbps_cmpver(pkgver, opkgver); if (cmp < 0 && xbps_pkg_reverts(binpkgd, opkgver)) { @@ -292,7 +333,9 @@ repo_add_pkg(struct repo *repo, const char *file, bool force) cmp = -1; } if (cmp <= 0) { - fprintf(stderr, "index: skipping `%s' (%s), already registered.\n", pkgver, arch); + xbps_warn_printf( + "%s: skipping `%s' (%s), already registered.\n", + repo->repodir, pkgver, arch); xbps_object_release(binpkgd); return 0; } @@ -466,15 +509,15 @@ print_inconsistent_shlibs(struct shared_state *state) const char *provider = NULL; xbps_array_t users; - if (!xbps_dictionary_get_cstring_nocopy(state->staged_shlibs, shlib, &provider)) - abort(); + if (!xbps_dictionary_get_cstring_nocopy(state->old_shlibs, shlib, &provider)) + unreachable(); users = xbps_dictionary_get(state->used_shlibs, shlib); printf(" %s (provided by: %s; used by: ", shlib, provider); for (unsigned int i = 0; i < xbps_array_count(users); i++) { const char *user = NULL; if (!xbps_array_get_cstring_nocopy(users, i, &user)) - abort(); + unreachable(); printf("%s%s", i > 0 ? ", " : "", user); } printf(")\n"); @@ -494,14 +537,14 @@ print_staged_packages(struct repo *repos, unsigned nrepos) xbps_dictionary_t pkg; const char *pkgname = NULL, *pkgver = NULL, *arch = NULL; if (!xbps_array_get_cstring_nocopy(state->added, j, &pkgname)) - abort(); + unreachable(); if (!xbps_dictionary_get_dict(state->stage, pkgname, &pkg)) - abort(); + unreachable(); if (!xbps_dictionary_get_cstring_nocopy(pkg, "pkgver", &pkgver)) - abort(); + unreachable(); if (!xbps_dictionary_get_cstring_nocopy(pkg, "architecture", &arch)) - abort(); - printf("%s: stage: added `%s' (%s)\n", state->repodir, pkgver, arch); + unreachable(); + printf("%s: stage: added `%s' (%s).\n", state->repodir, pkgver, arch); } } @@ -527,9 +570,9 @@ repo_unstage(struct repo *state) xbps_dictionary_t pkg = xbps_dictionary_get_keysym(state->stage, keysym); if (!xbps_dictionary_get_cstring_nocopy(pkg, "pkgver", &pkgver)) - abort(); + unreachable(); if (!xbps_dictionary_get_cstring_nocopy(pkg, "architecture", &arch)) - abort(); + unreachable(); printf("%s: index: added `%s' (%s).\n", state->repodir, pkgver, arch); @@ -551,24 +594,29 @@ repo_unstage(struct repo *state) static int repos_check_stage(struct repo *repos, unsigned nrepos) { - struct shared_state state; + struct shared_state state = { + .repos = repos, + .nrepos = nrepos, + }; int r; - state.staged_shlibs = xbps_dictionary_create(); - if (!state.staged_shlibs) + state.old_shlibs = xbps_dictionary_create(); + if (!state.old_shlibs) return xbps_error_oom(); state.used_shlibs = xbps_dictionary_create(); if (!state.used_shlibs) { - xbps_object_release(state.staged_shlibs); + xbps_object_release(state.old_shlibs); return xbps_error_oom(); } // collect all the used shared libraries in the stage for (unsigned i = 0; i < nrepos; i++) { - r = add_staged_shlib_providers(&state, &repos[i]); + r = add_old_shlibs(&state, &repos[i]); if (r < 0) goto err; + } + for (unsigned i = 0; i < nrepos; i++) { r = find_used_staged_shlibs(&state, &repos[i]); if (r < 0) goto err; @@ -614,7 +662,7 @@ repos_check_stage(struct repo *repos, unsigned nrepos) r = 0; err: - xbps_object_release(state.staged_shlibs); + xbps_object_release(state.old_shlibs); xbps_object_release(state.used_shlibs); return r; } @@ -681,7 +729,7 @@ index_add(struct xbps_handle *xhp, int argc, char **argv, bool force, const char for (unsigned i = 0; i < nrepos; i++) { const char *repodir = NULL; if (!xbps_array_get_cstring_nocopy(repo_args, i, &repodir)) - abort(); + unreachable(); r = repo_open(xhp, &repos[i], repodir, arch); if (r < 0) goto err; From 8ca5cbd32277983c13f15ce06c3678be3aea023e Mon Sep 17 00:00:00 2001 From: Duncan Overbruck Date: Thu, 11 Dec 2025 21:31:50 +0100 Subject: [PATCH 15/17] bin/xbps-rindex: refactor main --- bin/xbps-rindex/main.c | 92 +++++++++++++++++++++++++----------------- 1 file changed, 55 insertions(+), 37 deletions(-) diff --git a/bin/xbps-rindex/main.c b/bin/xbps-rindex/main.c index f3d5edc5..7f945c72 100644 --- a/bin/xbps-rindex/main.c +++ b/bin/xbps-rindex/main.c @@ -33,7 +33,7 @@ #include "xbps/xbps_array.h" static void __attribute__((noreturn)) -usage(bool fail) +usage(int status) { fprintf(stdout, "Usage: xbps-rindex [OPTIONS] MODE ARGUMENTS\n\n" @@ -54,7 +54,15 @@ usage(bool fail) " -r, --remove-obsoletes Removes obsolete packages from repository\n" " -s, --sign Initialize repository metadata signature\n" " -S, --sign-pkg ... Sign binary package archive\n"); - exit(fail ? EXIT_FAILURE : EXIT_SUCCESS); + exit(status); +} + +static void __attribute__((noreturn)) +multiple_mode_error(void) +{ + xbps_error_printf("only one mode can be specified: add, clean, " + "remove-obsoletes, sign or sign-pkg.\n"); + exit(EXIT_FAILURE); } int @@ -79,16 +87,19 @@ main(int argc, char **argv) { "repository", required_argument, NULL, 'R'}, { NULL, 0, NULL, 0 } }; - struct xbps_handle xh; + struct xbps_handle xh = {0}; const char *compression = NULL; const char *privkey = NULL, *signedby = NULL; - int rv, c, flags = 0; + int rv, c; xbps_array_t repos = NULL; - bool add_mode, clean_mode, rm_mode, sign_mode, sign_pkg_mode, force, - hashcheck; - - add_mode = clean_mode = rm_mode = sign_mode = sign_pkg_mode = force = - hashcheck = false; + enum { + INDEX_ADD = 1, + CLEAN_INDEX, + REMOVE_OBSOLETES, + SIGN_REPO, + SIGN_PACKAGE, + } mode = 0; + bool force = false, hashcheck = false; while ((c = getopt_long(argc, argv, shortopts, longopts, NULL)) != -1) { switch (c) { @@ -102,25 +113,33 @@ main(int argc, char **argv) compression = optarg; break; case 'a': - add_mode = true; + if (mode != 0) + multiple_mode_error(); + mode = INDEX_ADD; break; case 'c': - clean_mode = true; + if (mode != 0) + multiple_mode_error(); + mode = CLEAN_INDEX; break; case 'd': - flags |= XBPS_FLAG_DEBUG; + xh.flags |= XBPS_FLAG_DEBUG; break; case 'f': force = true; break; case 'h': - usage(false); + usage(EXIT_SUCCESS); /* NOTREACHED */ case 'r': - rm_mode = true; + if (mode != 0) + multiple_mode_error(); + mode = REMOVE_OBSOLETES; break; case 's': - sign_mode = true; + if (mode != 0) + multiple_mode_error(); + mode = SIGN_REPO; break; case 'C': hashcheck = true; @@ -134,54 +153,53 @@ main(int argc, char **argv) } break; case 'S': - sign_pkg_mode = true; + if (mode != 0) + multiple_mode_error(); + mode = SIGN_PACKAGE; break; case 'v': - flags |= XBPS_FLAG_VERBOSE; + xh.flags |= XBPS_FLAG_VERBOSE; break; case 'V': printf("%s\n", XBPS_RELVER); exit(EXIT_SUCCESS); case '?': default: - usage(true); + usage(EXIT_FAILURE); /* NOTREACHED */ } } - if ((argc == optind) || - (!add_mode && !clean_mode && !rm_mode && !sign_mode && !sign_pkg_mode)) { - usage(true); + if ((argc == optind) || mode == 0) { + usage(EXIT_FAILURE); /* NOTREACHED */ - } else if ((add_mode && (clean_mode || rm_mode || sign_mode || sign_pkg_mode)) || - (clean_mode && (add_mode || rm_mode || sign_mode || sign_pkg_mode)) || - (rm_mode && (add_mode || clean_mode || sign_mode || sign_pkg_mode)) || - (sign_mode && (add_mode || clean_mode || rm_mode || sign_pkg_mode)) || - (sign_pkg_mode && (add_mode || clean_mode || rm_mode || sign_mode))) { - xbps_error_printf("Only one mode can be specified: add, clean, " - "remove-obsoletes, sign or sign-pkg.\n"); - exit(EXIT_FAILURE); } /* initialize libxbps */ - memset(&xh, 0, sizeof(xh)); - xh.flags = flags; if ((rv = xbps_init(&xh)) != 0) { xbps_error_printf("failed to initialize libxbps: %s\n", strerror(rv)); exit(EXIT_FAILURE); } - if (add_mode) - rv = index_add(&xh, argc-optind, argv+optind, force, compression, repos); - else if (clean_mode) + switch (mode) { + case INDEX_ADD: + rv = index_add(&xh, argc - optind, argv + optind, force, + compression, repos); + break; + case CLEAN_INDEX: rv = index_clean(&xh, argv[optind], hashcheck, compression); - else if (rm_mode) + break; + case REMOVE_OBSOLETES: rv = remove_obsoletes(&xh, argv[optind]); - else if (sign_mode) + break; + case SIGN_REPO: rv = sign_repo(&xh, argv[optind], privkey, signedby, compression); - else if (sign_pkg_mode) + break; + case SIGN_PACKAGE: rv = sign_pkgs(&xh, optind, argc, argv, privkey, force); + break; + } exit(rv); } From 34e46aff31e98b6a0f57028c6564cec06b060f8b Mon Sep 17 00:00:00 2001 From: Duncan Overbruck Date: Sat, 13 Dec 2025 16:29:16 +0100 Subject: [PATCH 16/17] tests: cleanup xbps-rindex add regression test --- tests/xbps/xbps-rindex/add_test.sh | 53 ++++++++++++++++-------------- 1 file changed, 29 insertions(+), 24 deletions(-) diff --git a/tests/xbps/xbps-rindex/add_test.sh b/tests/xbps/xbps-rindex/add_test.sh index 7a590259..037f48cb 100644 --- a/tests/xbps/xbps-rindex/add_test.sh +++ b/tests/xbps/xbps-rindex/add_test.sh @@ -110,47 +110,52 @@ stage_resolve_bug_body() { cd some_repo # first add the provider and the requirer to the repo - xbps-create -A noarch -n provider-1.0_1 -s "foo pkg" --shlib-provides "libfoo.so.1 libbar.so.1" ../provider - atf_check_equal $? 0 - xbps-create -A noarch -n require-1.0_1 -s "foo pkg" --shlib-requires "libfoo.so.1" ../requirer - atf_check_equal $? 0 - xbps-create -A noarch -n stage-trigger-1.0_1 -s "foo pkg" --shlib-requires "libbar.so.1" ../stage-trigger - atf_check_equal $? 0 - xbps-rindex -d -a $PWD/*.xbps - atf_check_equal $? 0 + atf_check -o ignore -- xbps-create -A noarch -n provider-1.0_1 -s "foo pkg" --shlib-provides "libfoo.so.1 libbar.so.1" ../provider + atf_check -o ignore -- xbps-create -A noarch -n require-1.0_1 -s "foo pkg" --shlib-requires "libfoo.so.1" ../requirer + atf_check -o ignore -- xbps-create -A noarch -n stage-trigger-1.0_1 -s "foo pkg" --shlib-requires "libbar.so.1" ../stage-trigger + atf_check \ + -e ignore \ + -o match:"some_repo: index: added \`provider-1\.0_1' \(noarch\)\." \ + -o match:"some_repo: index: added \`require-1\.0_1' \(noarch\)\." \ + -o match:"some_repo: index: added \`stage-trigger-1\.0_1' \(noarch\)\." \ + -- xbps-rindex -a $PWD/*.xbps # then add libprovider that also provides the library - xbps-create -A noarch -n libprovider-1.0_2 -s "foo pkg" --shlib-provides "libfoo.so.1" ../libprovider - atf_check_equal $? 0 - xbps-rindex -d -a $PWD/*.xbps - atf_check_equal $? 0 + atf_check -o ignore -- xbps-create -A noarch -n libprovider-1.0_2 -s "foo pkg" --shlib-provides "libfoo.so.1" ../libprovider + atf_check \ + -e ignore \ + -o match:"some_repo: index: added \`libprovider-1\.0_2' \(noarch\)\." \ + -- xbps-rindex -a $PWD/*.xbps atf_check -o inline:" 4 $PWD (RSA unsigned)\n" -- \ xbps-query -r ../root -i --repository=$PWD -L # trigger staging - xbps-create -A noarch -n provider-1.0_2 -s "foo pkg" --shlib-provides "libfoo.so.1" ../provider - atf_check_equal $? 0 - xbps-rindex -d -a $PWD/*.xbps - atf_check_equal $? 0 + atf_check -o ignore -- xbps-create -A noarch -n provider-1.0_2 -s "foo pkg" --shlib-provides "libfoo.so.1" ../provider + atf_check \ + -e ignore \ + -o match:"some_repo: stage: added \`provider-1\.0_2' \(noarch\)\." \ + -- xbps-rindex -a $PWD/*.xbps atf_check -o inline:" 4 $PWD (Staged) (RSA unsigned)\n" -- \ xbps-query -r ../root -i --repository=$PWD -L # then add a new provider not containing the provides field. This resulted in # a stage state despites the library is resolved through libprovides - xbps-create -A noarch -n provider-1.0_3 -s "foo pkg" ../provider - atf_check_equal $? 0 - xbps-rindex -d -a $PWD/*.xbps - atf_check_equal $? 0 + atf_check -o ignore -- xbps-create -A noarch -n provider-1.0_3 -s "foo pkg" ../provider + atf_check \ + -e ignore \ + -o match:"some_repo: stage: added \`provider-1\.0_3' \(noarch\)\." \ + -- xbps-rindex -a $PWD/*.xbps atf_check -o inline:" 4 $PWD (Staged) (RSA unsigned)\n" -- \ xbps-query -r ../root -i --repository=$PWD -L # resolve staging # the actual bug appeared here: libfoo.so.1 is still provided by libprovider, but # xbps-rindex fails to register that. - xbps-create -A noarch -n stage-trigger-1.0_2 -s "foo pkg" ../stage-trigger - atf_check_equal $? 0 - xbps-rindex -d -a $PWD/*.xbps - atf_check_equal $? 0 + atf_check -o ignore -- xbps-create -A noarch -n stage-trigger-1.0_2 -s "foo pkg" ../stage-trigger + atf_check \ + -e ignore \ + -o match:"some_repo: index: added \`stage-trigger-1\.0_2' \(noarch\)\." \ + -- xbps-rindex -a $PWD/*.xbps atf_check -o inline:" 4 $PWD (RSA unsigned)\n" -- \ xbps-query -r ../root -i --repository=$PWD -L } From c5e827bd0492d7e6d3e8ddda4fa7e2f2b4369164 Mon Sep 17 00:00:00 2001 From: Duncan Overbruck Date: Sat, 13 Dec 2025 16:30:12 +0100 Subject: [PATCH 17/17] tests: add (expected failing) xbps-rindex dependency staging test --- tests/xbps/xbps-rindex/add_test.sh | 41 ++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/tests/xbps/xbps-rindex/add_test.sh b/tests/xbps/xbps-rindex/add_test.sh index 037f48cb..2307fb89 100644 --- a/tests/xbps/xbps-rindex/add_test.sh +++ b/tests/xbps/xbps-rindex/add_test.sh @@ -309,6 +309,46 @@ stage_multi_repos_body() { -- xbps-query -r ../root -i --repository=repo1 --repository=repo2 -L } +atf_test_case stage_dependency + +stage_dependency_head() { + atf_set "descr" "xbps-rindex(1) -a: stage repository due to broken dependency" +} + +stage_dependency_body() { + atf_expect_fail "not implemented" + mkdir -p repo1 repo2 repo3 repo4 root pkg + + cd repo1 + atf_check -o ignore -- xbps-create -A noarch -n A-1.0_1 -s "A pkg" --dependencies "B<2.0_1" ../pkg + atf_check -o ignore -- xbps-create -A noarch -n B-1.0_1 -s "B pkg" ../pkg + atf_check -o ignore -- xbps-create -A noarch -n unaffected-1.0_1 -s "unaffected pkg" --dependencies "B>=1.0_1" ../pkg + cd .. + + atf_check \ + -e ignore \ + -o match:"index: added \`A-1\.0_1'" \ + -o match:"index: added \`B-1\.0_1'" \ + -o match:"index: added \`unaffected-1\.0_1'" \ + -- xbps-rindex -v -R repo1 -a \ + repo1/A-1.0_1.noarch.xbps \ + repo1/B-1.0_1.noarch.xbps \ + repo1/unaffected-1.0_1.noarch.xbps + + cd repo1 + atf_check -o ignore -- xbps-create -A noarch -n B-2.0_1 -s "B pkg" ../pkg + cd .. + + atf_check \ + -e ignore \ + -o match:"repo1: stage: added \`B-2\.0_1'" \ + -- xbps-rindex -v -R repo1 -R repo2 -R repo3 -R repo4 -a repo1/B-2.0_1.noarch.xbps + + atf_check \ + -o match:"repo1 \(Staged\)" \ + -- xbps-query -r ../root -i --repository=repo1 --repository=repo2 -L +} + atf_init_test_cases() { atf_add_test_case update atf_add_test_case revert @@ -316,4 +356,5 @@ atf_init_test_cases() { atf_add_test_case stage_resolve_bug atf_add_test_case stage_stacked atf_add_test_case stage_multi_repos + atf_add_test_case stage_dependency }