fix: Add keepalive.

.github/buildfarm/authorized_keys

@@ -0,0 +1 @@
+ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC3qOXW5twS+gnni5Y4kCU3BBPjTShWsCTW5mQPRNA9GEYlDnBz3Wk0w4hfyI+NcUoSuOxxqrz6NGowfC/F4fYUe53R2/r1hWNuHloArbs9/8yROAaAlqhqmQDVsJxtbZdhh6sgwxcSe65OTYYhsZ8xzD34LBhuyLNaVEZy4WYUyjKjmh0YYSjiO6VFmzdvP3FSAqdFjG/pRs0igum9D8FJhl+FOrZ68fiFd70pyhl/yUsBntvtDbKJ73GKvSV58GytkLaxpDy4KLOVtnNYeqGFlFQ4eBrd+mXeyc2WvwYrn5V7HB2+Q2RTTvpEYazCjAolZ/E0wyqS2fSKFAGcKcDcoWordJ4VXwjiR9k28RPWSk9AcYJU1jI6v0KABOGTR7lNKmQr7fkD0KyZbYIB5EAcOsQHGCCDibFnxglIPFJt/YBZLLTm+ssTQ4ENkVrlS8dmC2drLMN3hsjBagoYD0e7hRARiv7/snREoNrQmKWz21Ft2Y37P7hWKOwQQ/NfZCs= iphy@code.tox.chat

.github/buildfarm/generate-wg

@@ -0,0 +1,15 @@
+#!/bin/sh
+
+echo '# Peers for the server' >wg0-1.conf
+
+readarray -t NODES <<<"$(seq 2 9)"
+
+for NODE in "${NODES[@]}"; do
+  PRIVATE_KEY="$(wg genkey)"
+  sed -e "s!@NODE@!$NODE!g;s!@PRIVATE_KEY@!$PRIVATE_KEY!g" wg0.conf.template >"wg0-$NODE.conf"
+  sed -e "s!@NODE@!$NODE!g" worker.yml.template >"worker-$NODE.yml"
+  echo >>wg0-1.conf
+  echo '[Peer]' >>wg0-1.conf
+  echo "PublicKey = $(echo "$PRIVATE_KEY" | wg pubkey)" >>wg0-1.conf
+  echo "AllowedIPs = 10.100.0.$NODE/32" >>wg0-1.conf
+done

.github/buildfarm/server-2.yml

@@ -0,0 +1,12 @@
+backplane:
+  redisUri: "redis://10.100.0.2:6379"
+  queues:
+    - name: "cpu"
+      properties:
+        - name: "min-cores"
+          value: "*"
+        - name: "max-cores"
+          value: "*"
+server:
+  publicName: "10.100.0.2:8980"
+prometheusPort: 9091

.github/buildfarm/wg0-1.conf

@@ -0,0 +1,33 @@
+# Peers for the server
+
+[Peer]
+PublicKey = 2UeU4A9G489ts7e0gXucEIXIU+IrtMCJnbnlONu+PlQ=
+AllowedIPs = 10.100.0.2/32
+
+[Peer]
+PublicKey = RCPvqEb+R+pKmZVOqJTHiLHIsb1KKYRNENZ5gtciQAo=
+AllowedIPs = 10.100.0.3/32
+
+[Peer]
+PublicKey = i2dITECpbIX3E7hAMIFTd0fce3uc15Zi4MzeggKF3zQ=
+AllowedIPs = 10.100.0.4/32
+
+[Peer]
+PublicKey = pSpYbq1/gPPqf65nTHgL3/w1xhSOd8sjSV84zYiRCEQ=
+AllowedIPs = 10.100.0.5/32
+
+[Peer]
+PublicKey = eLgqPRirKd5Z7Az1Xe9v5xV1AUq1JxB0jQXBwPOywCA=
+AllowedIPs = 10.100.0.6/32
+
+[Peer]
+PublicKey = z3cUh059Lo9JVf3Ea8r8x88JThVdcnmco63Iz/1dZ1Y=
+AllowedIPs = 10.100.0.7/32
+
+[Peer]
+PublicKey = UON0ft/NzF/0T5aWsZVqO45a4KKmP3pSUoXSkj4Y9kQ=
+AllowedIPs = 10.100.0.8/32
+
+[Peer]
+PublicKey = 594I6K0nCuDEeV3XFubePmR6jT3HVkRIQu4sTrXNZRE=
+AllowedIPs = 10.100.0.9/32

.github/buildfarm/wg0-2.conf

@@ -0,0 +1,9 @@
+[Interface]
+PrivateKey = WE+bkC80LF5s5+BBZ5TtWMKyBXAiSEdgVRd8ytWvkU0=
+Address = 10.100.0.2/24
+
+[Peer]
+PublicKey = Tb1SOeotVZa6FbEmRq1x4Wo62PxjrjLeHbRYp4kTH2A=
+Endpoint = code.tox.chat:51820
+AllowedIPs = 10.100.0.0/24
+PersistentKeepalive = 25

.github/buildfarm/wg0-3.conf

@@ -0,0 +1,9 @@
+[Interface]
+PrivateKey = mDRI1EKXXY8siirWEx+ifXV0zHM3s3yOeS6WBp2VfGE=
+Address = 10.100.0.3/24
+
+[Peer]
+PublicKey = Tb1SOeotVZa6FbEmRq1x4Wo62PxjrjLeHbRYp4kTH2A=
+Endpoint = code.tox.chat:51820
+AllowedIPs = 10.100.0.0/24
+PersistentKeepalive = 25

.github/buildfarm/wg0-4.conf

@@ -0,0 +1,9 @@
+[Interface]
+PrivateKey = 4P1Ct0HsoLvSz5KXVe8Jt/mNkdvf26H0R51xMzDiFFs=
+Address = 10.100.0.4/24
+
+[Peer]
+PublicKey = Tb1SOeotVZa6FbEmRq1x4Wo62PxjrjLeHbRYp4kTH2A=
+Endpoint = code.tox.chat:51820
+AllowedIPs = 10.100.0.0/24
+PersistentKeepalive = 25

.github/buildfarm/wg0-5.conf

@@ -0,0 +1,9 @@
+[Interface]
+PrivateKey = WN5cSY9S9Sh8jlLbR1BDlF/rcRQfRZIX5xIoXZRZJUg=
+Address = 10.100.0.5/24
+
+[Peer]
+PublicKey = Tb1SOeotVZa6FbEmRq1x4Wo62PxjrjLeHbRYp4kTH2A=
+Endpoint = code.tox.chat:51820
+AllowedIPs = 10.100.0.0/24
+PersistentKeepalive = 25

.github/buildfarm/wg0-6.conf

@@ -0,0 +1,9 @@
+[Interface]
+PrivateKey = sPpPQmp3eZqWv0DyWkSDsJoz5Ofxp7eb2SIT8TZwt3Q=
+Address = 10.100.0.6/24
+
+[Peer]
+PublicKey = Tb1SOeotVZa6FbEmRq1x4Wo62PxjrjLeHbRYp4kTH2A=
+Endpoint = code.tox.chat:51820
+AllowedIPs = 10.100.0.0/24
+PersistentKeepalive = 25

.github/buildfarm/wg0-7.conf

@@ -0,0 +1,9 @@
+[Interface]
+PrivateKey = CFyxB7DmHXM1VaF2v8SbLTyRH1eGrfGIPd74Rq3EQFc=
+Address = 10.100.0.7/24
+
+[Peer]
+PublicKey = Tb1SOeotVZa6FbEmRq1x4Wo62PxjrjLeHbRYp4kTH2A=
+Endpoint = code.tox.chat:51820
+AllowedIPs = 10.100.0.0/24
+PersistentKeepalive = 25

.github/buildfarm/wg0-8.conf

@@ -0,0 +1,9 @@
+[Interface]
+PrivateKey = OM17rUuFLFGxhMI0gIW59JCDyITC34eQRPKEx/cpMmQ=
+Address = 10.100.0.8/24
+
+[Peer]
+PublicKey = Tb1SOeotVZa6FbEmRq1x4Wo62PxjrjLeHbRYp4kTH2A=
+Endpoint = code.tox.chat:51820
+AllowedIPs = 10.100.0.0/24
+PersistentKeepalive = 25

.github/buildfarm/wg0-9.conf

@@ -0,0 +1,9 @@
+[Interface]
+PrivateKey = yJA/1szzSxqxwSxT3si/1v3Gx0oTDlOS10HZgTwfZFc=
+Address = 10.100.0.9/24
+
+[Peer]
+PublicKey = Tb1SOeotVZa6FbEmRq1x4Wo62PxjrjLeHbRYp4kTH2A=
+Endpoint = code.tox.chat:51820
+AllowedIPs = 10.100.0.0/24
+PersistentKeepalive = 25

.github/buildfarm/wg0.conf.template

@@ -0,0 +1,9 @@
+[Interface]
+PrivateKey = @PRIVATE_KEY@
+Address = 10.100.0.@NODE@/24
+
+[Peer]
+PublicKey = Tb1SOeotVZa6FbEmRq1x4Wo62PxjrjLeHbRYp4kTH2A=
+Endpoint = code.tox.chat:51820
+AllowedIPs = 10.100.0.0/24
+PersistentKeepalive = 25

.github/buildfarm/worker-2.yml

@@ -0,0 +1,11 @@
+backplane:
+  redisUri: "redis://10.100.0.2:6379"
+  queues:
+    - name: "cpu"
+      properties:
+        - name: "min-cores"
+          value: "*"
+        - name: "max-cores"
+          value: "*"
+worker:
+  publicName: "10.100.0.2:8981"

.github/buildfarm/worker-3.yml

@@ -0,0 +1,11 @@
+backplane:
+  redisUri: "redis://10.100.0.2:6379"
+  queues:
+    - name: "cpu"
+      properties:
+        - name: "min-cores"
+          value: "*"
+        - name: "max-cores"
+          value: "*"
+worker:
+  publicName: "10.100.0.3:8981"

.github/buildfarm/worker-4.yml

@@ -0,0 +1,11 @@
+backplane:
+  redisUri: "redis://10.100.0.2:6379"
+  queues:
+    - name: "cpu"
+      properties:
+        - name: "min-cores"
+          value: "*"
+        - name: "max-cores"
+          value: "*"
+worker:
+  publicName: "10.100.0.4:8981"

.github/buildfarm/worker-5.yml

@@ -0,0 +1,11 @@
+backplane:
+  redisUri: "redis://10.100.0.2:6379"
+  queues:
+    - name: "cpu"
+      properties:
+        - name: "min-cores"
+          value: "*"
+        - name: "max-cores"
+          value: "*"
+worker:
+  publicName: "10.100.0.5:8981"

.github/buildfarm/worker-6.yml

@@ -0,0 +1,11 @@
+backplane:
+  redisUri: "redis://10.100.0.2:6379"
+  queues:
+    - name: "cpu"
+      properties:
+        - name: "min-cores"
+          value: "*"
+        - name: "max-cores"
+          value: "*"
+worker:
+  publicName: "10.100.0.6:8981"

.github/buildfarm/worker-7.yml

@@ -0,0 +1,11 @@
+backplane:
+  redisUri: "redis://10.100.0.2:6379"
+  queues:
+    - name: "cpu"
+      properties:
+        - name: "min-cores"
+          value: "*"
+        - name: "max-cores"
+          value: "*"
+worker:
+  publicName: "10.100.0.7:8981"

.github/buildfarm/worker-8.yml

@@ -0,0 +1,11 @@
+backplane:
+  redisUri: "redis://10.100.0.2:6379"
+  queues:
+    - name: "cpu"
+      properties:
+        - name: "min-cores"
+          value: "*"
+        - name: "max-cores"
+          value: "*"
+worker:
+  publicName: "10.100.0.8:8981"

.github/buildfarm/worker-9.yml

@@ -0,0 +1,11 @@
+backplane:
+  redisUri: "redis://10.100.0.2:6379"
+  queues:
+    - name: "cpu"
+      properties:
+        - name: "min-cores"
+          value: "*"
+        - name: "max-cores"
+          value: "*"
+worker:
+  publicName: "10.100.0.9:8981"

.github/buildfarm/worker.yml.template

@@ -0,0 +1,11 @@
+backplane:
+  redisUri: "redis://10.100.0.2:6379"
+  queues:
+    - name: "cpu"
+      properties:
+        - name: "min-cores"
+          value: "*"
+        - name: "max-cores"
+          value: "*"
+worker:
+  publicName: "10.100.0.@NODE@:8981"

.github/workflows/ci.yml

@@ -12,15 +12,62 @@ concurrency:
 
 jobs:
   wireguard-vpn:
+    strategy:
+      matrix:
+        node: [2, 3, 4, 5, 6, 7, 8, 9]
+#       node: [2, 3, 4]
     runs-on: ubuntu-latest
     steps:
-      - run: |
-          sudo apt install wireguard
-          echo "UGdw/CjBpnKF/xJ3/orlUty29kYi1NMnXdXVYExFFH8=" > privatekey
-          sudo ip link add dev wg0 type wireguard
-          sudo ip address add dev wg0 10.100.0.2 peer 10.100.0.1
-          sudo wg set wg0 listen-port 48123 private-key privatekey peer Tb1SOeotVZa6FbEmRq1x4Wo62PxjrjLeHbRYp4kTH2A= allowed-ips 10.100.0.0/24 endpoint code.tox.chat:51820
-          sudo ip link set up dev wg0
-          #ping 10.100.0.1
-          sudo ifconfig wg0
+      - uses: actions/checkout@v3
+      - name: Setup wireguard
+        run: |
+          sudo apt-get -y --no-install-recommends install redis-tools wireguard
+          sudo cp .github/buildfarm/wg0-${{ matrix.node }}.conf /etc/wireguard/wg0.conf
+          sudo systemctl enable wg-quick@wg0.service
+          sudo systemctl start wg-quick@wg0.service || (sudo systemctl status wg-quick@wg0.service && false)
+          sudo systemctl start ssh
+          mkdir "$HOME/.ssh"
+          cp .github/buildfarm/authorized_keys "$HOME/.ssh/"
+      - name: Start redis
+        if: matrix.node == 2
+        run: |
+          docker pull "redis:alpine"
+          docker run --detach --rm -i --network host "redis:alpine"
+      - name: Wait for VPN to be up
+        run: |
+          ifconfig wg0
           sudo wg show
+          # VPN server and redis/buildfarm server need to be up.
+          for node in `seq 1 9`; do
+            for i in `seq 0 9`; do
+              if ping -c1 "10.100.0.$node"; then break; fi
+              sleep 1
+            done
+          done
+          # Error if VPN server is still down after the above timeout.
+          ping -c1 10.100.0.1
+          sudo wg show
+          ip route
+          # Wait for redis connectivity.
+          for i in `seq 0 9`; do
+            if redis-cli -h 10.100.0.2 -p 6379 ping; then break; fi
+            sleep 1
+          done
+          redis-cli -h 10.100.0.2 -p 6379 ping
+      - name: Pull latest worker image
+        run: docker pull "toxchat/buildfarm-worker"
+      - name: Test connectivity to other nodes
+        run: |
+          ping -c1 10.100.0.1  # server
+          ping -c1 10.100.0.2  # either self or some other node
+          ping -c1 10.100.0.3  # either some other node or self
+      - name: Start worker
+        if: matrix.node != 2
+        run: |
+          docker run          --privileged --rm -i --network host -v "$PWD/.github/buildfarm/worker-${{ matrix.node }}.yml:/app/build_buildfarm/examples/config.minimal.yml" "toxchat/buildfarm-worker"
+      - name: Start server and worker
+        if: matrix.node == 2
+        run: |
+          docker pull "toxchat/buildfarm-server"
+          docker run --detach --privileged --rm -i --network host -v "$PWD/.github/buildfarm/worker-${{ matrix.node }}.yml:/app/build_buildfarm/examples/config.minimal.yml" "toxchat/buildfarm-worker"
+          docker run --rm -i --network host -v "$PWD/.github/buildfarm/server-${{ matrix.node }}.yml:/app/build_buildfarm/examples/config.minimal.yml" "toxchat/buildfarm-server"