[RELOPS-1610] linux bootstrap: use openvox

data/os/Debian.yaml

@@ -1 +1,2 @@
----
+--- {}
+# A hash/dictionary is expected. This prevents a puppet load failure warning.

provisioners/linux/.gitignore

@@ -1,3 +1,3 @@
-vault.yml
-vault.yaml
-ronin_settings
+vault.yml*
+vault.yaml*
+ronin_settings*

provisioners/linux/README.md

@@ -0,0 +1,103 @@
+# Linux Provisioners
+
+Scripts for bootstrapping and maintaining Linux (Ubuntu) hosts managed by ronin_puppet.
+
+## Scripts
+
+### `bootstrap_linux.sh`
+
+Bootstraps a fresh Linux host from post-install to a fully converged Puppet run. Installs
+[OpenVox](https://voxpupuli.org/openvox/) (an open-source Puppet agent), configures NTP,
+and runs Puppet until it succeeds. Supports Ubuntu 18.04 and 24.04.
+
+**Prerequisites:**
+- `/root/vault.yaml` — hiera secrets file
+- `/etc/puppet_role` — puppet role for this host (e.g. `gecko_t_linux_talos`)
+- `wget` installed
+
+**Usage:**
+
+```bash
+# Interactive mode
+sudo ./bootstrap_linux.sh
+
+# Non-interactive (logs to file)
+sudo ./bootstrap_linux.sh -l /var/log/bootstrap.log
+
+# Use a specific repo/branch during development
+PUPPET_REPO="https://github.com/youruser/ronin_puppet.git" \
+PUPPET_BRANCH="your-branch" \
+sudo ./bootstrap_linux.sh
+```
+
+### `deliver_linux.sh`
+
+Delivers bootstrap prerequisites to a remote host over SSH and prints the commands needed
+to kick off the bootstrap. Auto-detects whether to connect as `relops` or `root`.
+
+**Usage:**
+
+```bash
+./deliver_linux.sh <hostname> <puppet-role>
+
+# Example
+./deliver_linux.sh devicepool-0.relops.mozops.net gecko_t_linux_talos
+```
+
+The script copies `bootstrap_linux.sh`, `vault.yaml`, and (optionally) a `ronin_settings`
+file to the remote host and sets the role file. It then prints the SSH commands to run the
+bootstrap.
+
+### `update_linux.sh`
+
+Updates `vault.yaml` (hiera secrets) on an already-bootstrapped host. Can optionally
+update the puppet role with `force`.
+
+**Usage:**
+
+```bash
+./update_linux.sh <hostname> [role] [force]
+
+# Update secrets only
+./update_linux.sh myhost.example.com
+
+# Update secrets and change role (requires 'force')
+./update_linux.sh myhost.example.com gecko_t_linux_talos force
+```
+
+### `bootstrap_bitbar_devicepool.sh`
+
+Bootstrap script specific to `bitbar_devicepool` hosts. Installs Puppet 7, r10k, and runs
+a masterless Puppet apply against a git-cloned copy of the repo.
+
+## Configuration
+
+### `vault.yaml`
+
+Hiera secrets file. Must be present at `/root/vault.yaml` on the target host before
+bootstrap runs. A `vault.yaml.bak` and `vault.yaml.bak2` are kept as backups.
+
+### `ronin_settings` / `ronin_settings.dis`
+
+Optional settings file that overrides `PUPPET_REPO`, `PUPPET_BRANCH`, and other variables.
+Rename to `ronin_settings` (drop the `.dis` extension) to activate it. When present,
+`deliver_linux.sh` will copy it to `/etc/puppet/ronin_settings` on the remote host.
+
+## Testing
+
+Bootstrap scripts can be tested locally using Docker via the script in `test/`.
+
+```bash
+# Test against Ubuntu 24.04
+./test/test_bootstrap.sh 24.04
+
+# Test against Ubuntu 18.04
+./test/test_bootstrap.sh 18.04
+```
+
+The test script spins up an Ubuntu container, copies `bootstrap_linux.sh` into it, and
+verifies that `openvox-agent` installs successfully. The script uses `SKIP_NTP=true` since
+Docker containers don't have an init system — NTP setup is expected to fail, but
+`openvox-agent` installation is validated.
+
+Requires Docker to be installed and running.

provisioners/linux/bootstrap_linux.sh

@@ -182,49 +182,76 @@ if [ ! -f /root/vault.yaml ]; then
     exit 1
 fi
 
-# if on ubuntu 22.04, install puppet 8, else install puppet 7
+# determine ubuntu version so we can set NTP method appropriately
 if [ -f /etc/os-release ]; then
     . /etc/os-release
 else
     echo "This script requires /etc/os-release to determine the OS version."
     exit 1
 fi
 
-# if on ubuntu 24.04, install puppet 8, else install puppet 7
+TEMP_DEB_NAME="bootstrap_temp.deb"
+# noble, bionic, etc (VERSION_CODENAME) and 18.04, 24.04, etc (VERSION_ID)
+# are sourced from /etc/os-release above
+
+# puppet vars
+#
+# PKG_TO_INSTALL="puppet-agent"
+# INSTALL_URL_BASE="https://apt.puppetlabs.com"
+# INSTALL_URL_DEB="puppet8-release-noble.deb"
+# INSTALL_URL_DEB="puppet7-release-bionic.deb"
+
+# openvox vars
+#
+PKG_TO_INSTALL="openvox-agent"
+INSTALL_URL_BASE="https://apt.voxpupuli.org"
+INSTALL_URL_DEB="openvox8-release-ubuntu${VERSION_ID}.deb"
+
+# we install openvox 8 on both, but NTP setup differs
 if [ "$VERSION_ID" = "24.04" ]; then
-    echo "Installing Puppet 8..."
-    wget https://apt.puppetlabs.com/puppet8-release-noble.deb -O /tmp/puppet.deb
+    echo "Installing Openvox 8..."
+
+    wget "${INSTALL_URL_BASE}/${INSTALL_URL_DEB}" -O /tmp/${TEMP_DEB_NAME}
     # install puppet release deb for the version we've selected
-    dpkg -i /tmp/puppet.deb
+    dpkg -i /tmp/${TEMP_DEB_NAME}
     # update apt and install puppet-agent and ntp
     apt-get update
     # shellcheck disable=SC2090
     DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::='--force-confnew' remove -y puppet
     # shellcheck disable=SC2090
-    DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::='--force-confnew' install -y puppet-agent
-    # 24.04 uses timesyncd (on by default), see `systemctl status systemd-timesyncd`
-    # place our config and restart the service
-    mkdir -p /etc/systemd/timesyncd.conf.d
-    echo -e "[Time]\nNTP=ntp.build.mozilla.org" >/etc/systemd/timesyncd.conf.d/mozilla.conf
-    systemctl restart systemd-timesyncd
+    DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::='--force-confnew' install -y "${PKG_TO_INSTALL}"
+
+    if [ "${SKIP_NTP:-false}" != "true" ]; then
+        # get clock synced. if clock is way off, run-puppet.sh will never finish
+        #   it's git clone because the SSL cert will appear invalid.
+        #
+        # 24.04 uses timesyncd (on by default), see `systemctl status systemd-timesyncd`
+        # place our config and restart the service
+        mkdir -p /etc/systemd/timesyncd.conf.d
+        echo -e "[Time]\nNTP=ntp.build.mozilla.org" >/etc/systemd/timesyncd.conf.d/mozilla.conf
+        systemctl restart systemd-timesyncd
+    fi
 elif [ "$VERSION_ID" = "18.04" ]; then
-    echo "Installing Puppet 7..."
-    wget https://apt.puppetlabs.com/puppet7-release-bionic.deb -O /tmp/puppet.deb
+    echo "Installing Openvox 8..."
+
+    wget "${INSTALL_URL_BASE}/${INSTALL_URL_DEB}" -O /tmp/${TEMP_DEB_NAME}
     # install puppet release deb for the version we've selected
-    dpkg -i /tmp/puppet.deb
+    dpkg -i /tmp/${TEMP_DEB_NAME}
     # update apt and install puppet-agent and ntp
     apt-get update
     # shellcheck disable=SC2090
     DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::='--force-confnew' remove -y puppet
     # shellcheck disable=SC2090
-    DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::='--force-confnew' install -y puppet-agent ntp
-
-    # get clock synced. if clock is way off, run-puppet.sh will never finish
-    # it's git clone because the SSL cert will appear invalid.
-    /etc/init.d/ntp stop
-    echo "server ntp.build.mozilla.org iburst" >/etc/ntp.conf # place barebones config
-    ntpd -q -g                                                # runs once and force allows huge skews
-    /etc/init.d/ntp start
+    DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::='--force-confnew' install -y "${PKG_TO_INSTALL}" ntp
+
+    if [ "${SKIP_NTP:-false}" != "true" ]; then
+        # get clock synced. if clock is way off, run-puppet.sh will never finish
+        #   it's git clone because the SSL cert will appear invalid.
+        /etc/init.d/ntp stop
+        echo "server ntp.build.mozilla.org iburst" >/etc/ntp.conf # place barebones config
+        ntpd -q -g                                                 # runs once and force allows huge skews
+        /etc/init.d/ntp start
+    fi
 else
     echo "Unsupported Ubuntu version: $VERSION_ID. This script only supports Ubuntu 18.04 and 24.04."
     exit 1

provisioners/linux/deliver_linux.sh

@@ -15,12 +15,20 @@ set -e
 #
 
 # Detect which SSH user works (try relops first, then root)
+# Retries for up to ~10 minutes to handle hosts where auth isn't ready immediately
 detect_ssh_user() {
   local host="$1"
-  for user in "relops" "root" ; do
-    if ssh -o ConnectTimeout=5 -o BatchMode=yes -q "$user"@"$host" exit 2>/dev/null; then
-      echo "$user"
-      return 0
+  local max_attempts=60
+  for attempt in $(seq 1 "$max_attempts"); do
+    for user in "relops" "root"; do
+      if ssh -o ConnectTimeout=5 -o BatchMode=yes -q "$user"@"$host" exit 2>/dev/null; then
+        echo "$user"
+        return 0
+      fi
+    done
+    if [ "$attempt" -lt "$max_attempts" ]; then
+      printf '.' >&2
+      sleep 10
     fi
   done
   echo ""
@@ -74,8 +82,17 @@ fi
 
 # cleanup ssh key, it will be new after kickstarting
 ssh-keygen -R "${THE_HOST}" || true
-# readd to avoid prompts
-ssh-keyscan -H "${THE_HOST}" >> ~/.ssh/known_hosts
+# readd to avoid prompts; retry until SSH daemon is ready
+echo "Waiting for SSH on ${THE_HOST}..."
+for _i in $(seq 1 60); do
+  result=$(ssh-keyscan -H "${THE_HOST}" 2>/dev/null)
+  if [ -n "$result" ]; then
+    echo "$result" >> ~/.ssh/known_hosts
+    break
+  fi
+  printf '.'
+  sleep 10
+done
 
 # detect which SSH user works
 echo "Detecting SSH user for ${THE_HOST}..."

provisioners/linux/test/test_bootstrap.sh

@@ -0,0 +1,68 @@
+#!/bin/bash
+# Tests that bootstrap_linux.sh successfully installs openvox-agent on a given
+# Ubuntu version. The script is expected to exit early at the NTP/systemd step
+# since Docker containers don't have an init system — that's fine, openvox is
+# installed before that point.
+#
+# Usage: ./test_bootstrap.sh <ubuntu-version>
+#        ./test_bootstrap.sh 18.04
+#        ./test_bootstrap.sh 24.04
+
+set -e
+
+UBUNTU_VERSION="${1:-}"
+
+if [ -z "$UBUNTU_VERSION" ]; then
+    echo "Usage: $0 <ubuntu-version>"
+    echo "       $0 18.04"
+    echo "       $0 24.04"
+    exit 1
+fi
+
+case "$UBUNTU_VERSION" in
+    18.04|24.04) ;;
+    *)
+        echo "Unsupported version: $UBUNTU_VERSION (supported: 18.04, 24.04)"
+        exit 1
+        ;;
+esac
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+BOOTSTRAP_SCRIPT="${SCRIPT_DIR}/../bootstrap_linux.sh"
+
+if [ ! -f "$BOOTSTRAP_SCRIPT" ]; then
+    echo "Bootstrap script not found: $BOOTSTRAP_SCRIPT"
+    exit 1
+fi
+
+CONTAINER_ID=""
+cleanup() {
+    if [ -n "$CONTAINER_ID" ]; then
+        echo "==> Cleaning up container"
+        docker rm -f "$CONTAINER_ID" >/dev/null 2>&1 || true
+    fi
+}
+trap cleanup EXIT
+
+echo "==> Starting ubuntu:${UBUNTU_VERSION} container"
+CONTAINER_ID=$(docker run -d "ubuntu:${UBUNTU_VERSION}" sleep infinity)
+
+echo "==> Installing prerequisite: wget"
+docker exec "$CONTAINER_ID" bash -c "apt-get update -qq && apt-get install -y -qq wget"
+
+echo "==> Copying bootstrap_linux.sh"
+docker cp "$BOOTSTRAP_SCRIPT" "$CONTAINER_ID:/root/bootstrap_linux.sh"
+
+echo "==> Creating stub /root/vault.yaml"
+docker exec "$CONTAINER_ID" bash -c "echo '---' > /root/vault.yaml"
+
+echo "==> Running bootstrap_linux.sh"
+docker exec -e SKIP_NTP=true "$CONTAINER_ID" bash /root/bootstrap_linux.sh || true
+
+echo "==> Checking openvox-agent installation"
+if docker exec "$CONTAINER_ID" dpkg-query -W -f='${Status}' openvox-agent 2>/dev/null | grep -q "install ok installed"; then
+    echo "PASS: openvox-agent is installed on Ubuntu ${UBUNTU_VERSION}"
+else
+    echo "FAIL: openvox-agent is not installed on Ubuntu ${UBUNTU_VERSION}"
+    exit 1
+fi