diff --git a/ansible/host_vars/a-fsn-de.m.voidlinux.org.yml b/ansible/host_vars/a-fsn-de.m.voidlinux.org.yml index 5f0617bc..a362a9a6 100644 --- a/ansible/host_vars/a-fsn-de.m.voidlinux.org.yml +++ b/ansible/host_vars/a-fsn-de.m.voidlinux.org.yml @@ -31,7 +31,7 @@ nomad_host_volumes: path: /data/void-packages read_only: false - name: root-pkgs - path: /hostdir/binpkgs + path: /srv/www/void-repo/current read_only: false - name: root_mirror path: /srv/www/void-repo @@ -48,3 +48,6 @@ nomad_host_volumes: - name: ccache path: /hostdir/ccache read_only: true + - name: incoming_pkgs + path: /incoming + read_only: false diff --git a/ansible/roles/consul/tasks/main.yml b/ansible/roles/consul/tasks/main.yml index 51fb749f..4c96865d 100644 --- a/ansible/roles/consul/tasks/main.yml +++ b/ansible/roles/consul/tasks/main.yml @@ -3,6 +3,7 @@ xbps: pkg: consul state: present + tags: [install_consul] - name: Install Core Configuration template: diff --git a/ansible/roles/docker/tasks/main.yml b/ansible/roles/docker/tasks/main.yml index 00764189..206e973e 100644 --- a/ansible/roles/docker/tasks/main.yml +++ b/ansible/roles/docker/tasks/main.yml @@ -3,6 +3,7 @@ xbps: pkg: docker state: present + tags: [install_docker] - name: Create config directory file: @@ -11,6 +12,7 @@ owner: root group: root mode: 0755 + tags: [install_docker] - name: Install daemon.json template: @@ -21,6 +23,7 @@ mode: 0644 notify: - docker + tags: [install_docker, configure_docker] - name: Install Firewall Rules copy: @@ -31,9 +34,11 @@ mode: 0640 notify: - iptables + tags: [install_docker, configure_docker] - name: Enable Service service: name: docker state: started enabled: true + tags: [install_docker, enable_docker] diff --git a/ansible/roles/nomad/tasks/main.yml b/ansible/roles/nomad/tasks/main.yml index 0d7917ee..ff51b5da 100644 --- a/ansible/roles/nomad/tasks/main.yml +++ b/ansible/roles/nomad/tasks/main.yml @@ -3,6 +3,7 @@ xbps: pkg: nomad state: present + tags: [install_nomad] - name: Install base configuration template: diff --git a/services/nomad/build/build-rsyncd.nomad b/services/nomad/build/build-rsyncd.nomad index f9c3e300..58cb1c5f 100644 --- a/services/nomad/build/build-rsyncd.nomad +++ b/services/nomad/build/build-rsyncd.nomad @@ -19,6 +19,18 @@ job "build-rsyncd" { read_only = false } + volume "root_mirror" { + type = "host" + source = "root_mirror" + read_only = false + } + + volume "incoming_pkgs" { + type = "host" + source = "incoming_pkgs" + read_only = false + } + service { provider = "nomad" name = "build-rsyncd" @@ -29,8 +41,7 @@ job "build-rsyncd" { driver = "docker" config { - image = "ghcr.io/void-linux/infra-rsync:20240709R1" - volumes = [ "local/buildsync.conf:/etc/rsyncd.conf.d/buildsync.conf" ] + image = "ghcr.io/void-linux/infra-rsync:20251102R1" } resources { @@ -43,17 +54,26 @@ job "build-rsyncd" { destination = "/hostdir" } + volume_mount { + volume = "root_mirror" + destination = "/mirror" + } + + volume_mount { + volume = "incoming_pkgs" + destination = "/incoming" + } + template { - data = file("xbps-clean-sigs") - destination = "local/xbps-clean-sigs" + data = file("rsync-post-xfer") + destination = "local/rsync-post-xfer" perms = "0755" } template { data = < Syncing packages to the shadow repository..." +rsync -vurk --delete-after \ +--filter='+ */' --filter='+ *.%(prop:target)s.xbps' \ +--filter='- .*' --filter='- *' \ +--password-file=/secrets/rsync/password /hostdir/binpkgs/ \ +{{ range nomadService 1 $allocID "build-rsyncd" -}} +rsync://buildsync@{{ .Address }}:{{ .Port }}/incoming-%(prop:target)s +{{ end -}} +case "$?" in + 0|23|24) exit 0 ;; + *) exit 1 ;; +esac +""")] + + +@util.renderer +def make_clean_cmd(props): + return [ + 'find', '/hostdir/binpkgs', '(', + '-name', util.Interpolate('*.%(prop:target)s.xbps'), '-o', '-name', util.Interpolate('%(prop:target)s-repodata'), + ')', '-print', '-delete', + ], + + @util.renderer def build_packages(props): cmds = [] @@ -220,6 +254,11 @@ def build_packages(props): logname=f'pkg:{p}', haltOnFailure=True, )) + cmds.append(util.ShellArg( + command=make_rsync_cmd, + logname=f'sync:{p}', + haltOnFailure=True, + )) if cmds: cmds.append(util.ShellArg( command=['make', 'clean'], @@ -229,34 +268,6 @@ def build_packages(props): return cmds -@util.renderer -def make_prune_cmd(props): - return ['bash', '-c', - util.Interpolate(f""" -export XBPS_TARGET_ARCH="%(prop:target)s" -for repo in / /debug /nonfree /bootstrap; do - xbps-rindex -r "{hostdir}/binpkgs/$repo" -done -if [ "$XBPS_TARGET_ARCH" = i686 ]; then - for repo in /multilib /multilib/nonfree /multilib/bootstrap; do - XBPS_TARGET_ARCH=x86_64 xbps-rindex -r "{hostdir}/binpkgs/$repo" - done -fi -""")] - -@util.renderer -def make_rsync_cmd(props): - return ['bash', '-c', - util.Interpolate(""" -rsync -vurk --delete-after --delay-updates \ ---filter='+ */' --filter='+ %(prop:target)s-repodata' --filter='+ *.%(prop:target)s.xbps' \ ---filter='+ otime' --filter='- .*' --filter='- *' \ ---password-file=/secrets/rsync/password /hostdir/binpkgs/ \ -{{ range nomadService 1 $allocID "build-rsyncd" -}} -rsync://buildsync-%(prop:worker)s@{{ .Address }}:{{ .Port }}/%(prop:worker)s -{{ end -}}""")] - - factory.addStep(GitWithDiff( repourl='https://github.com/void-linux/void-packages.git', mode='incremental', @@ -351,34 +362,15 @@ factory.addStep(steps.ShellSequence( )) factory.addStep(steps.ShellCommand( - command=make_prune_cmd, - name='prune_packages', - description='removing obsolete packages', - descriptionDone='removed obsolete packages', - haltOnFailure=True, - logEnviron=False, - usePTY=True, - workdir='.', - timeout=14400, -)) - -factory.addStep(steps.ShellCommand( - command=make_rsync_cmd, - name='sync_packages', - description='syncing packages to the shadow repository', - descriptionDone='synced packages to the shadow repository', + command=make_clean_cmd, + name='clean_packages', + description='cleaning packages from hostdir', + descriptionDone='cleaned packages from hostdir', alwaysRun=True, logEnviron=False, usePTY=True, workdir='.', - doStepIf=do_sync(), - hideStepIf=hide_skipped, timeout=14400, - decodeRC={ - 0: SUCCESS, - 23: SUCCESS, - 24: SUCCESS, - }, )) for b in builders: @@ -393,8 +385,7 @@ for b in builders: 'target': targetarch, 'cross': str(hostarch != targetarch), 'worker': b['worker'], - 'sync': str(b['sync']), - 'bootstrap_args': b.get('bootstrap_args', '-N'), + 'bootstrap_args': b.get('bootstrap_args', ''), } c['builders'].append(util.BuilderConfig( diff --git a/services/nomad/build/buildbot.nomad b/services/nomad/build/buildbot.nomad index 3b7e5d25..7ee1b02e 100644 --- a/services/nomad/build/buildbot.nomad +++ b/services/nomad/build/buildbot.nomad @@ -107,15 +107,15 @@ EOF { name = "aarch64", max-builds = 2 }, ], builders = [ - { name = "x86_64", host = "x86_64", worker = "glibc", sync = false }, - { name = "i686", host = "i686", worker = "glibc", sync = false }, - { name = "armv7l", host = "x86_64", target = "armv7l", worker = "glibc", sync = false }, - { name = "armv6l", host = "x86_64", target = "armv6l", worker = "glibc", sync = false }, - { name = "x86_64-musl", host = "x86_64-musl", worker = "musl", sync = true }, - { name = "armv7l-musl", host = "x86_64-musl", target = "armv7l-musl", worker = "musl", sync = true }, - { name = "armv6l-musl", host = "x86_64-musl", target = "armv6l-musl", worker = "musl", sync = true }, - { name = "aarch64", host = "x86_64", target = "aarch64", worker = "aarch64", sync = true, bootstrap_args = "" }, - { name = "aarch64-musl", host = "x86_64-musl", target = "aarch64-musl", worker = "aarch64", sync = true, bootstrap_args = "" }, + { name = "x86_64", host = "x86_64", worker = "glibc", }, + { name = "i686", host = "i686", worker = "glibc", }, + { name = "armv7l", host = "x86_64", target = "armv7l", worker = "glibc", }, + { name = "armv6l", host = "x86_64", target = "armv6l", worker = "glibc", }, + { name = "x86_64-musl", host = "x86_64-musl", worker = "musl", }, + { name = "armv7l-musl", host = "x86_64-musl", target = "armv7l-musl", worker = "musl", }, + { name = "armv6l-musl", host = "x86_64-musl", target = "armv6l-musl", worker = "musl", }, + { name = "aarch64", host = "x86_64", target = "aarch64", worker = "aarch64" }, + { name = "aarch64-musl", host = "x86_64-musl", target = "aarch64-musl", worker = "aarch64" }, ], }) destination = "local/workers.json" diff --git a/services/nomad/build/buildsync.nomad b/services/nomad/build/buildsync.nomad index 86a1fef1..4a8db1c2 100644 --- a/services/nomad/build/buildsync.nomad +++ b/services/nomad/build/buildsync.nomad @@ -37,7 +37,7 @@ job "buildsync" { template { data = <"/hostdir/binpkgs/$dir/otime" - done - sleep 60 + t="$(date +%s)" + for dir in / /nonfree /debug /multilib /multilib/nonfree /nonfree; do + [ -e "/mirror/current/$dir" ] && echo "$t">"/mirror/current/$dir/otime" + [ -e "/mirror/current/musl/$dir" ] && echo "$t">"/mirror/current/musl/$dir/otime" + [ -e "/mirror/current/aarch64/$dir" ] && echo "$t">"/mirror/current/aarch64/$dir/otime" + done + sleep 60 done EOF - destination = "local/run.sh" - perms = "0755" - } + destination = "local/run.sh" + perms = "0755" } } } diff --git a/services/nomad/build/xbps-clean-sigs b/services/nomad/build/xbps-clean-sigs deleted file mode 100644 index 85c8f3b0..00000000 --- a/services/nomad/build/xbps-clean-sigs +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/sh - -# Remove signatures that don't have a corresponding package. -find /hostdir/binpkgs \( -name '*.xbps.sig' -o -name '*.xbps.sig2' \) -exec sh -c 'for x in "$@"; do [ -e "${x%.sig*}" ] || rm -- $x; done' _ {} + - -exit 0