native aarch64 builder, remote-only sync architecture

ansible/host_vars/a-fsn-de.m.voidlinux.org.yml

@@ -31,7 +31,7 @@ nomad_host_volumes:
     path: /data/void-packages
     read_only: false
   - name: root-pkgs
-    path: /hostdir/binpkgs
+    path: /srv/www/void-repo/current
     read_only: false
   - name: root_mirror
     path: /srv/www/void-repo
@@ -48,3 +48,6 @@ nomad_host_volumes:
   - name: ccache
     path: /hostdir/ccache
     read_only: true
+  - name: incoming_pkgs
+    path: /incoming
+    read_only: false

ansible/roles/consul/tasks/main.yml

@@ -3,6 +3,7 @@
   xbps:
     pkg: consul
     state: present
+  tags: [install_consul]
 
 - name: Install Core Configuration
   template:

ansible/roles/docker/tasks/main.yml

@@ -3,6 +3,7 @@
   xbps:
     pkg: docker
     state: present
+  tags: [install_docker]
 
 - name: Create config directory
   file:
@@ -11,6 +12,7 @@
     owner: root
     group: root
     mode: 0755
+  tags: [install_docker]
 
 - name: Install daemon.json
   template:
@@ -21,6 +23,7 @@
     mode: 0644
   notify:
     - docker
+  tags: [install_docker, configure_docker]
 
 - name: Install Firewall Rules
   copy:
@@ -31,9 +34,11 @@
     mode: 0640
   notify:
     - iptables
+  tags: [install_docker, configure_docker]
 
 - name: Enable Service
   service:
     name: docker
     state: started
     enabled: true
+  tags: [install_docker, enable_docker]

ansible/roles/nomad/tasks/main.yml

@@ -3,6 +3,7 @@
   xbps:
     pkg: nomad
     state: present
+  tags: [install_nomad]
 
 - name: Install base configuration
   template:

services/nomad/build/build-rsyncd.nomad

@@ -19,6 +19,18 @@ job "build-rsyncd" {
       read_only = false
     }
 
+    volume "root_mirror" {
+      type = "host"
+      source = "root_mirror"
+      read_only = false
+    }
+
+    volume "incoming_pkgs" {
+      type = "host"
+      source = "incoming_pkgs"
+      read_only = false
+    }
+
     service {
       provider = "nomad"
       name = "build-rsyncd"
@@ -29,8 +41,7 @@ job "build-rsyncd" {
       driver = "docker"
 
       config {
-        image = "ghcr.io/void-linux/infra-rsync:20240709R1"
-        volumes = [ "local/buildsync.conf:/etc/rsyncd.conf.d/buildsync.conf" ]
+        image = "ghcr.io/void-linux/infra-rsync:20251102R1"
       }
 
       resources {
@@ -43,17 +54,26 @@ job "build-rsyncd" {
         destination = "/hostdir"
       }
 
+      volume_mount {
+        volume = "root_mirror"
+        destination = "/mirror"
+      }
+
+      volume_mount {
+        volume = "incoming_pkgs"
+        destination = "/incoming"
+      }
+
       template {
-        data = file("xbps-clean-sigs")
-        destination = "local/xbps-clean-sigs"
+        data = file("rsync-post-xfer")
+        destination = "local/rsync-post-xfer"
         perms = "0755"
       }
 
       template {
         data = <<EOF
 {{- with nomadVar "nomad/jobs/buildsync" }}
-buildsync-aarch64:{{ .aarch64_password }}
-buildsync-musl:{{ .musl_password }}
+buildsync:{{ .password }}
 {{- end }}
 EOF
         destination = "secrets/buildsync.secrets"
@@ -75,21 +95,30 @@ incoming chmod = D0755,F0644
 [sources]
 path = /hostdir/sources
 filter = - by_sha256/ - .* - *.part
-auth users = buildsync-*:rw
-
-[aarch64]
-path = /hostdir/binpkgs/aarch64
-auth users = buildsync-aarch64:rw
-filter = + */ + *-repodata + otime + *.xbps - *.sig - *.sig2 - *-repodata.* - *-stagedata.* - *.x86_64* - x86_64*-repodata - .*
-post-xfer exec = /local/xbps-clean-sigs
-
-[musl]
-path = /hostdir/binpkgs/musl
-auth users = buildsync-musl:rw
-filter = + */ + *-repodata + otime + *.xbps - *.sig - *.sig2 - *-repodata.* - *-stagedata.* - .*
-post-xfer exec = /local/xbps-clean-sigs
+auth users = buildsync:rw
+
+&merge /local/rsyncd.conf.d
+EOF
+        destination = "local/rsyncd.conf.d/buildsync.conf"
+      }
+
+      dynamic "template" {
+        for_each = [
+          "x86_64", "i686", "armv7l", "armv6l",
+          "x86_64-musl", "armv7l-musl", "armv6l-musl",
+          "aarch64", "aarch64-musl",
+        ]
+
+        content {
+          data = <<EOF
+[incoming-${template.value}]
+path = /incoming/${template.value}
+auth users = buildsync:rw
+filter = + */ + *.${template.value}.xbps - *.sig - *.sig2 - *-repodata* - .*
+post-xfer exec = /local/rsync-post-xfer
 EOF
-        destination = "local/buildsync.conf"
+          destination = "local/rsyncd.conf.d/${template.value}.conf.inc"
+        }
       }
     }
   }

services/nomad/build/buildbot-worker.nomad

@@ -160,22 +160,31 @@ EOF
           destination = "local/info/host"
         }
 
-        // the builders should use local repos
-        // except for aarch64, which must be able to get hostmakedepends from repo-default
+        // the builders should use the internal mirror
         template {
           data = <<EOF
-repository=/hostdir/binpkgs/bootstrap
-repository=/hostdir/binpkgs
-repository=/hostdir/binpkgs/nonfree
-{{ if eq "${group.value.name}" "glibc" }}
-repository=/hostdir/binpkgs/multilib/bootstrap
-repository=/hostdir/binpkgs/multilib
-repository=/hostdir/binpkgs/multilib/nonfree
-{{ end }}
-{{ if eq "${group.value.name}" "aarch64" }}
 {{ range service "root-pkgs-internal" }}
+{{ if eq "${group.value.name}" "glibc" }}
+repository=http://{{ .Address }}:{{ .Port }}/bootstrap
+repository=http://{{ .Address }}:{{ .Port }}
+repository=http://{{ .Address }}:{{ .Port }}/nonfree
+repository=http://{{ .Address }}:{{ .Port }}/multilib/bootstrap
+repository=http://{{ .Address }}:{{ .Port }}/multilib
+repository=http://{{ .Address }}:{{ .Port }}/multilib/nonfree
+{{ else if eq "${group.value.name}" "aarch64" }}
+repository=http://{{ .Address }}:{{ .Port }}/bootstrap
 repository=http://{{ .Address }}:{{ .Port }}
+repository=http://{{ .Address }}:{{ .Port }}/nonfree
+repository=http://{{ .Address }}:{{ .Port }}/musl/bootstrap
 repository=http://{{ .Address }}:{{ .Port }}/musl
+repository=http://{{ .Address }}:{{ .Port }}/musl/nonfree
+repository=http://{{ .Address }}:{{ .Port }}/aarch64/bootstrap
+repository=http://{{ .Address }}:{{ .Port }}/aarch64
+repository=http://{{ .Address }}:{{ .Port }}/aarch64/nonfree
+{{ else }}
+repository=http://{{ .Address }}:{{ .Port }}/${group.value.name}/bootstrap
+repository=http://{{ .Address }}:{{ .Port }}/${group.value.name}
+repository=http://{{ .Address }}:{{ .Port }}/${group.value.name}/nonfree
 {{ end }}
 {{ end }}
 EOF
@@ -205,7 +214,7 @@ EOF
         template {
           data = <<EOF
 {{- with nomadVar "nomad/jobs/buildsync" -}}
-{{ .${group.value.name}_password }}
+{{ .password }}
 {{- end -}}
 EOF
           destination = "secrets/rsync/password"

services/nomad/build/buildbot.cfg

@@ -10,7 +10,7 @@ from pathlib import Path
 from twisted.internet import defer
 from twisted.python import log
 
-from buildbot.process.results import SUCCESS, SKIPPED
+from buildbot.process.results import SUCCESS
 from buildbot.plugins import util, secrets, reporters, worker, schedulers
 from buildbot.plugins import steps
 
@@ -129,13 +129,19 @@ c['schedulers'].append(schedulers.ForceScheduler(
 
 # ###### BUILDERS
 
-distdir = lambda f: util.Interpolate(f'/workdir/%(prop:name)s/void-packages/{f}')
-bulkdir = lambda f: util.Interpolate(f'/workdir/%(prop:name)s/xbps-bulk/{f}')
+def distdir(f):
+    return util.Interpolate(f'/workdir/%(prop:name)s/void-packages/{f}')
+
+def bulkdir(f):
+    return util.Interpolate(f'/workdir/%(prop:name)s/xbps-bulk/{f}')
+
 hostdir = '/hostdir'
-buildroot = lambda: util.Interpolate(f'/buildroots/%(prop:name)s/buildroot')
-builddir = lambda: util.Interpolate('builddir-%(prop:buildnumber)s')
-do_sync = lambda: util.Interpolate('%(prop:sync)s') == "True"
-hide_skipped = lambda results, _: results == SKIPPED
+
+def buildroot():
+    return util.Interpolate('/buildroots/%(prop:name)s/buildroot')
+
+def builddir():
+    return util.Interpolate('builddir-%(prop:buildnumber)s')
 
 
 factory = util.BuildFactory()
@@ -198,7 +204,7 @@ def make_xbps_bulk_cmd(props):
         '-h', hostdir,
         '-d', distdir(''),
         '-m', buildroot(),
-        '-t',
+        '-t', '-s', '-S',
     ]
     if props.getProperty('cross') == 'True':
         command += ['-a', props.getProperty('target')]
@@ -211,6 +217,34 @@ def make_xbps_bulk_cmd(props):
     return command
 
 
+@util.renderer
+def make_rsync_cmd(props):
+    return ['bash', '-e', '-c',
+            util.Interpolate("""
+echo "=> Syncing packages to the shadow repository..."
+rsync -vurk --delete-after \
+--filter='+ */' --filter='+ *.%(prop:target)s.xbps' \
+--filter='- .*' --filter='- *' \
+--password-file=/secrets/rsync/password /hostdir/binpkgs/ \
+{{ range nomadService 1 $allocID "build-rsyncd" -}}
+rsync://buildsync@{{ .Address }}:{{ .Port }}/incoming-%(prop:target)s
+{{ end -}}
+case "$?" in
+    0|23|24) exit 0 ;;
+    *) exit 1 ;;
+esac
+""")]
+
+
+@util.renderer
+def make_clean_cmd(props):
+    return [
+        'find', '/hostdir/binpkgs', '(',
+            '-name', util.Interpolate('*.%(prop:target)s.xbps'), '-o', '-name', util.Interpolate('%(prop:target)s-repodata'),
+        ')', '-print', '-delete',
+    ],
+
+
 @util.renderer
 def build_packages(props):
     cmds = []
@@ -220,6 +254,11 @@ def build_packages(props):
             logname=f'pkg:{p}',
             haltOnFailure=True,
         ))
+        cmds.append(util.ShellArg(
+            command=make_rsync_cmd,
+            logname=f'sync:{p}',
+            haltOnFailure=True,
+        ))
     if cmds:
         cmds.append(util.ShellArg(
             command=['make', 'clean'],
@@ -229,34 +268,6 @@ def build_packages(props):
     return cmds
 
 
-@util.renderer
-def make_prune_cmd(props):
-    return ['bash', '-c',
-            util.Interpolate(f"""
-export XBPS_TARGET_ARCH="%(prop:target)s"
-for repo in / /debug /nonfree /bootstrap; do
-    xbps-rindex -r "{hostdir}/binpkgs/$repo"
-done
-if [ "$XBPS_TARGET_ARCH" = i686 ]; then
-    for repo in /multilib /multilib/nonfree /multilib/bootstrap; do
-        XBPS_TARGET_ARCH=x86_64 xbps-rindex -r "{hostdir}/binpkgs/$repo"
-    done
-fi
-""")]
-
-@util.renderer
-def make_rsync_cmd(props):
-    return ['bash', '-c',
-            util.Interpolate("""
-rsync -vurk --delete-after --delay-updates \
---filter='+ */' --filter='+ %(prop:target)s-repodata' --filter='+ *.%(prop:target)s.xbps' \
---filter='+ otime' --filter='- .*' --filter='- *' \
---password-file=/secrets/rsync/password /hostdir/binpkgs/ \
-{{ range nomadService 1 $allocID "build-rsyncd" -}}
-rsync://buildsync-%(prop:worker)s@{{ .Address }}:{{ .Port }}/%(prop:worker)s
-{{ end -}}""")]
-
-
 factory.addStep(GitWithDiff(
     repourl='https://github.com/void-linux/void-packages.git',
     mode='incremental',
@@ -351,34 +362,15 @@ factory.addStep(steps.ShellSequence(
 ))
 
 factory.addStep(steps.ShellCommand(
-    command=make_prune_cmd,
-    name='prune_packages',
-    description='removing obsolete packages',
-    descriptionDone='removed obsolete packages',
-    haltOnFailure=True,
-    logEnviron=False,
-    usePTY=True,
-    workdir='.',
-    timeout=14400,
-))
-
-factory.addStep(steps.ShellCommand(
-    command=make_rsync_cmd,
-    name='sync_packages',
-    description='syncing packages to the shadow repository',
-    descriptionDone='synced packages to the shadow repository',
+    command=make_clean_cmd,
+    name='clean_packages',
+    description='cleaning packages from hostdir',
+    descriptionDone='cleaned packages from hostdir',
     alwaysRun=True,
     logEnviron=False,
     usePTY=True,
     workdir='.',
-    doStepIf=do_sync(),
-    hideStepIf=hide_skipped,
     timeout=14400,
-    decodeRC={
-        0: SUCCESS,
-        23: SUCCESS,
-        24: SUCCESS,
-    },
 ))
 
 for b in builders:
@@ -393,8 +385,7 @@ for b in builders:
         'target': targetarch,
         'cross': str(hostarch != targetarch),
         'worker': b['worker'],
-        'sync': str(b['sync']),
-        'bootstrap_args': b.get('bootstrap_args', '-N'),
+        'bootstrap_args': b.get('bootstrap_args', ''),
     }
 
     c['builders'].append(util.BuilderConfig(