native aarch64 builder, remote-only sync architecture

ansible/host_vars/a-fsn-de.m.voidlinux.org.yml

@@ -31,7 +31,7 @@ nomad_host_volumes:
     path: /data/void-packages
     read_only: false
   - name: root-pkgs
-    path: /hostdir/binpkgs
+    path: /srv/www/void-repo/current
     read_only: false
   - name: root_mirror
     path: /srv/www/void-repo
@@ -48,3 +48,6 @@ nomad_host_volumes:
   - name: ccache
     path: /hostdir/ccache
     read_only: true
+  - name: incoming_pkgs
+    path: /incoming
+    read_only: false

ansible/host_vars/a-sea-us.m.voidlinux.org.yml

@@ -8,3 +8,24 @@ xbps_repository_nonfree: https://{{ xbps_repository_address }}/current/aarch64/n
 xbps_repoconf_repos_enabled:
   - 00-repository-main
   - 10-repository-nonfree
+
+sshd_AllowGroups:
+  - build-ops
+
+buildworker_archs:
+  - aarch64
+  - aarch64-musl
+
+nomad_host_volumes:
+  - name: ccache
+    path: /hostdir/ccache
+    read_only: true
+  - name: aarch64_hostdir
+    path: /hostdir
+    read_only: false
+  - name: aarch64_workdir
+    path: /builder/workdir
+    read_only: false
+  - name: aarch64_buildrootdir
+    path: /builder/buildroot
+    read_only: false

ansible/inventory

@@ -2,6 +2,7 @@
 a-hel-fi.m.voidlinux.org
 a-fsn-de.m.voidlinux.org
 b-fsn-de.m.voidlinux.org
+a-sea-us.m.voidlinux.org
 
 [root_mirror]
 a-fsn-de.m.voidlinux.org

services/nomad/build/build-rsyncd.nomad

@@ -9,7 +9,7 @@ job "build-rsyncd" {
       mode = "bridge"
       port "rsync" {
         to = 873
-        host_network = "internal"
+        # XXX: host_network = "internal"
       }
     }
 
@@ -19,6 +19,18 @@ job "build-rsyncd" {
       read_only = false
     }
 
+    volume "root_mirror" {
+      type = "host"
+      source = "root_mirror"
+      read_only = false
+    }
+
+    volume "incoming_pkgs" {
+      type = "host"
+      source = "incoming_pkgs"
+      read_only = false
+    }
+
     service {
       provider = "nomad"
       name = "build-rsyncd"
@@ -29,8 +41,7 @@ job "build-rsyncd" {
       driver = "docker"
 
       config {
-        image = "ghcr.io/void-linux/infra-rsync:20240709R1"
-        volumes = [ "local/buildsync.conf:/etc/rsyncd.conf.d/buildsync.conf" ]
+        image = "ghcr.io/void-linux/infra-rsync:20251102R1"
       }
 
       resources {
@@ -43,17 +54,26 @@ job "build-rsyncd" {
         destination = "/hostdir"
       }
 
+      volume_mount {
+        volume = "root_mirror"
+        destination = "/mirror"
+      }
+
+      volume_mount {
+        volume = "incoming_pkgs"
+        destination = "/incoming"
+      }
+
       template {
-        data = file("xbps-clean-sigs")
-        destination = "local/xbps-clean-sigs"
+        data = file("rsync-post-xfer")
+        destination = "local/rsync-post-xfer"
         perms = "0755"
       }
 
       template {
         data = <<EOF
 {{- with nomadVar "nomad/jobs/buildsync" }}
-buildsync-aarch64:{{ .aarch64_password }}
-buildsync-musl:{{ .musl_password }}
+buildsync:{{ .password }}
 {{- end }}
 EOF
         destination = "secrets/buildsync.secrets"
@@ -77,19 +97,28 @@ path = /hostdir/sources
 filter = - by_sha256/ - .* - *.part
 auth users = buildsync-*:rw
 
-[aarch64]
-path = /hostdir/binpkgs/aarch64
-auth users = buildsync-aarch64:rw
-filter = + */ + *-repodata + otime + *.xbps - *.sig - *.sig2 - *-repodata.* - *-stagedata.* - *.x86_64* - x86_64*-repodata - .*
-post-xfer exec = /local/xbps-clean-sigs
-
-[musl]
-path = /hostdir/binpkgs/musl
-auth users = buildsync-musl:rw
-filter = + */ + *-repodata + otime + *.xbps - *.sig - *.sig2 - *-repodata.* - *-stagedata.* - .*
-post-xfer exec = /local/xbps-clean-sigs
+&merge /local/rsyncd.conf.d
+EOF
+        destination = "local/rsyncd.conf.d/buildsync.conf"
+      }
+
+      dynamic "template" {
+        for_each = [
+          "x86_64", "i686", "armv7l", "armv6l",
+          "x86_64-musl", "armv7l-musl", "armv6l-musl",
+          "aarch64", "aarch64-musl",
+        ]
+
+        content {
+          data = <<EOF
+[incoming-${template.value}]
+path = /incoming/${template.value}
+auth users = buildsync:rw
+filter = + */ + *.${template.value}.xbps - *.sig - *.sig2 - *-repodata* - .*
+post-xfer exec = /local/rsync-post-xfer
 EOF
-        destination = "local/buildsync.conf"
+          destination = "local/rsyncd.conf.d/${template.value}.conf.inc"
+        }
       }
     }
   }

services/nomad/build/buildbot-worker.nomad

@@ -8,9 +8,13 @@ job "buildbot-worker" {
       // cpu is ~equivalent to a number of cores
       // memory is ~90% of capacity
       // memory_max is ~95% of capacity
-      { name = "glibc",   jobs = 10, cpu = 38100, mem = 115840, mem_max = 122270 },
-      { name = "musl",    jobs = 6,  cpu = 21700, mem = 57690,  mem_max = 60890  },
-      { name = "aarch64", jobs = 6,  cpu = 12000, mem = 28500,  mem_max = 30500  },
+      # XXX
+      # { name = "glibc",   jobs = 10, cpu = 38100, mem = 115840, mem_max = 122270 },
+      # { name = "musl",    jobs = 6,  cpu = 21700, mem = 57690,  mem_max = 60890  },
+      # { name = "aarch64", jobs = 12, cpu = 16000, mem = 27500,  mem_max = 29500  },
+      { name = "glibc",   jobs = 2, cpu = 10000, mem = 4096, mem_max = 8192 },
+      { name = "musl",    jobs = 2, cpu = 10000, mem = 4096, mem_max = 8192 },
+      { name = "aarch64", jobs = 2, cpu = 10000, mem = 4096, mem_max = 8192 },
     ]
     labels = [ "buildbot-worker-${group.value.name}" ]
 
@@ -118,9 +122,10 @@ job "buildbot-worker" {
           destination = "/buildroots"
         }
 
+        # XXX
         template {
           data = <<EOF
-{{ range service "buildbot-worker" -}}
+{{ range nomadService "buildbot-worker" -}}
 [buildbot]
 host = {{ .Address }}
 worker-port = {{ .Port }}
@@ -132,6 +137,7 @@ EOF
           destination = "local/config.ini"
         }
 
+        # XXX
         template {
           data = <<EOF
 XBPS_MAKEJOBS=${group.value.jobs}
@@ -141,7 +147,7 @@ XBPS_DEBUG_PKGS=yes
 XBPS_USE_GIT_REVS=yes
 XBPS_DISTFILES_MIRROR=https://sources.voidlinux.org
 XBPS_PRESERVE_PKGS=yes
-{{ range service "root-pkgs-internal" }}
+{{ range nomadService "root-pkgs-internal" }}
 XBPS_MIRROR=http://{{ .Address }}:{{ .Port }}
 {{ end }}
 EOF
@@ -161,21 +167,21 @@ EOF
         }
 
         // the builders should use local repos
-        // except for aarch64, which must be able to get hostmakedepends from repo-default
+        # XXX
         template {
           data = <<EOF
-repository=/hostdir/binpkgs/bootstrap
-repository=/hostdir/binpkgs
-repository=/hostdir/binpkgs/nonfree
+{{ range nomadService "root-pkgs-internal" }}
 {{ if eq "${group.value.name}" "glibc" }}
-repository=/hostdir/binpkgs/multilib/bootstrap
-repository=/hostdir/binpkgs/multilib
-repository=/hostdir/binpkgs/multilib/nonfree
-{{ end }}
-{{ if eq "${group.value.name}" "aarch64" }}
-{{ range service "root-pkgs-internal" }}
+repository=http://{{ .Address }}:{{ .Port }}/bootstrap
 repository=http://{{ .Address }}:{{ .Port }}
-repository=http://{{ .Address }}:{{ .Port }}/musl
+repository=http://{{ .Address }}:{{ .Port }}/nonfree
+repository=http://{{ .Address }}:{{ .Port }}/multilib/bootstrap
+repository=http://{{ .Address }}:{{ .Port }}/multilib
+repository=http://{{ .Address }}:{{ .Port }}/multilib/nonfree
+{{ else }}
+repository=http://{{ .Address }}:{{ .Port }}/${group.value.name}/bootstrap
+repository=http://{{ .Address }}:{{ .Port }}/${group.value.name}
+repository=http://{{ .Address }}:{{ .Port }}/${group.value.name}/nonfree
 {{ end }}
 {{ end }}
 EOF
@@ -205,7 +211,7 @@ EOF
         template {
           data = <<EOF
 {{- with nomadVar "nomad/jobs/buildsync" -}}
-{{ .${group.value.name}_password }}
+{{ .password }}
 {{- end -}}
 EOF
           destination = "secrets/rsync/password"

services/nomad/build/buildbot.cfg

@@ -10,7 +10,7 @@ from pathlib import Path
 from twisted.internet import defer
 from twisted.python import log
 
-from buildbot.process.results import SUCCESS, SKIPPED
+from buildbot.process.results import SUCCESS
 from buildbot.plugins import util, secrets, reporters, worker, schedulers
 from buildbot.plugins import steps
 
@@ -129,13 +129,19 @@ c['schedulers'].append(schedulers.ForceScheduler(
 
 # ###### BUILDERS
 
-distdir = lambda f: util.Interpolate(f'/workdir/%(prop:name)s/void-packages/{f}')
-bulkdir = lambda f: util.Interpolate(f'/workdir/%(prop:name)s/xbps-bulk/{f}')
+def distdir(f):
+    return util.Interpolate(f'/workdir/%(prop:name)s/void-packages/{f}')
+
+def bulkdir(f):
+    return util.Interpolate(f'/workdir/%(prop:name)s/xbps-bulk/{f}')
+
 hostdir = '/hostdir'
-buildroot = lambda: util.Interpolate(f'/buildroots/%(prop:name)s/buildroot')
-builddir = lambda: util.Interpolate('builddir-%(prop:buildnumber)s')
-do_sync = lambda: util.Interpolate('%(prop:sync)s') == "True"
-hide_skipped = lambda results, _: results == SKIPPED
+
+def buildroot():
+    return util.Interpolate('/buildroots/%(prop:name)s/buildroot')
+
+def builddir():
+    return util.Interpolate('builddir-%(prop:buildnumber)s')
 
 
 factory = util.BuildFactory()
@@ -198,7 +204,7 @@ def make_xbps_bulk_cmd(props):
         '-h', hostdir,
         '-d', distdir(''),
         '-m', buildroot(),
-        '-t',
+        '-t', '-s', '-S',
     ]
     if props.getProperty('cross') == 'True':
         command += ['-a', props.getProperty('target')]
@@ -229,36 +235,26 @@ def build_packages(props):
     return cmds
 
 
-@util.renderer
-def make_prune_cmd(props):
-    return ['bash', '-c',
-            util.Interpolate(f"""
-export XBPS_TARGET_ARCH="%(prop:target)s"
-for repo in / /debug /nonfree /bootstrap; do
-    xbps-rindex -r "{hostdir}/binpkgs/$repo"
-done
-if [ "$XBPS_TARGET_ARCH" = i686 ]; then
-    for repo in /multilib /multilib/nonfree /multilib/bootstrap; do
-        XBPS_TARGET_ARCH=x86_64 xbps-rindex -r "{hostdir}/binpkgs/$repo"
-    done
-fi
-""")]
-
 @util.renderer
 def make_rsync_cmd(props):
-    return ['bash', '-c',
+    return ['bash', '-e', '-c',
             util.Interpolate("""
-rsync -vurk --delete-after --delay-updates \
---filter='+ */' --filter='+ %(prop:target)s-repodata' --filter='+ *.%(prop:target)s.xbps' \
---filter='+ otime' --filter='- .*' --filter='- *' \
+echo "=> Syncing packages to the shadow repository..."
+rsync -vurk --delete-after \
+--filter='+ */' --filter='+ *.%(prop:target)s.xbps' \
+--filter='- .*' --filter='- *' \
 --password-file=/secrets/rsync/password /hostdir/binpkgs/ \
 {{ range nomadService 1 $allocID "build-rsyncd" -}}
-rsync://buildsync-%(prop:worker)s@{{ .Address }}:{{ .Port }}/%(prop:worker)s
-{{ end -}}""")]
+rsync://buildsync@{{ .Address }}:{{ .Port }}/incoming-%(prop:target)s
+{{ end -}}
+echo "=> Cleaning built packages..."
+find /hostdir/binpkgs -name '*.xbps' -o -name '*-repodata' -print -delete
+""")]
 
 
 factory.addStep(GitWithDiff(
-    repourl='https://github.com/void-linux/void-packages.git',
+    # XXX
+    repourl='https://github.com/classabbyamp/void-packages.git',
     mode='incremental',
     workdir=distdir(''),
     progress=True,
@@ -271,7 +267,8 @@ factory.addStep(GitWithDiff(
 ))
 
 factory.addStep(steps.Git(
-    repourl='https://github.com/void-linux/xbps-bulk.git',
+    # XXX
+    repourl='https://github.com/classabbyamp/xbps-bulk.git',
     mode='incremental',
     workdir=bulkdir(''),
     progress=True,
@@ -350,18 +347,6 @@ factory.addStep(steps.ShellSequence(
     timeout=14400,
 ))
 
-factory.addStep(steps.ShellCommand(
-    command=make_prune_cmd,
-    name='prune_packages',
-    description='removing obsolete packages',
-    descriptionDone='removed obsolete packages',
-    haltOnFailure=True,
-    logEnviron=False,
-    usePTY=True,
-    workdir='.',
-    timeout=14400,
-))
-
 factory.addStep(steps.ShellCommand(
     command=make_rsync_cmd,
     name='sync_packages',
@@ -371,8 +356,6 @@ factory.addStep(steps.ShellCommand(
     logEnviron=False,
     usePTY=True,
     workdir='.',
-    doStepIf=do_sync(),
-    hideStepIf=hide_skipped,
     timeout=14400,
     decodeRC={
         0: SUCCESS,
@@ -393,8 +376,7 @@ for b in builders:
         'target': targetarch,
         'cross': str(hostarch != targetarch),
         'worker': b['worker'],
-        'sync': str(b['sync']),
-        'bootstrap_args': b.get('bootstrap_args', '-N'),
+        'bootstrap_args': b.get('bootstrap_args', ''),
     }
 
     c['builders'].append(util.BuilderConfig(