From 829919b95032c4a1bbd5a7b5d5560ffa6931873e Mon Sep 17 00:00:00 2001 From: Andrew Erickson Date: Fri, 20 Feb 2026 15:52:09 -0800 Subject: [PATCH 01/11] todos --- provisioners/linux/bootstrap_linux.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/provisioners/linux/bootstrap_linux.sh b/provisioners/linux/bootstrap_linux.sh index cf83aefba5..0b1440d54a 100644 --- a/provisioners/linux/bootstrap_linux.sh +++ b/provisioners/linux/bootstrap_linux.sh @@ -193,6 +193,7 @@ fi # if on ubuntu 24.04, install puppet 8, else install puppet 7 if [ "$VERSION_ID" = "24.04" ]; then echo "Installing Puppet 8..." + # TODO: use openvox wget https://apt.puppetlabs.com/puppet8-release-noble.deb -O /tmp/puppet.deb # install puppet release deb for the version we've selected dpkg -i /tmp/puppet.deb @@ -209,6 +210,7 @@ if [ "$VERSION_ID" = "24.04" ]; then systemctl restart systemd-timesyncd elif [ "$VERSION_ID" = "18.04" ]; then echo "Installing Puppet 7..." + # TODO: use openvox and puppet 8 wget https://apt.puppetlabs.com/puppet7-release-bionic.deb -O /tmp/puppet.deb # install puppet release deb for the version we've selected dpkg -i /tmp/puppet.deb From 80a481c9388889a7860bb603fcc3fdd799d01382 Mon Sep 17 00:00:00 2001 From: Andrew Erickson Date: Fri, 20 Feb 2026 15:56:09 -0800 Subject: [PATCH 02/11] comments, formatting --- provisioners/linux/bootstrap_linux.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/provisioners/linux/bootstrap_linux.sh b/provisioners/linux/bootstrap_linux.sh index 0b1440d54a..6b5b318d28 100644 --- a/provisioners/linux/bootstrap_linux.sh +++ b/provisioners/linux/bootstrap_linux.sh @@ -203,6 +203,10 @@ if [ "$VERSION_ID" = "24.04" ]; then DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::='--force-confnew' remove -y puppet # shellcheck disable=SC2090 DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::='--force-confnew' install -y puppet-agent + + # get clock synced. if clock is way off, run-puppet.sh will never finish + # it's git clone because the SSL cert will appear invalid. + # # 24.04 uses timesyncd (on by default), see `systemctl status systemd-timesyncd` # place our config and restart the service mkdir -p /etc/systemd/timesyncd.conf.d @@ -222,7 +226,7 @@ elif [ "$VERSION_ID" = "18.04" ]; then DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::='--force-confnew' install -y puppet-agent ntp # get clock synced. if clock is way off, run-puppet.sh will never finish - # it's git clone because the SSL cert will appear invalid. + # it's git clone because the SSL cert will appear invalid. /etc/init.d/ntp stop echo "server ntp.build.mozilla.org iburst" >/etc/ntp.conf # place barebones config ntpd -q -g # runs once and force allows huge skews From 5b754ed1731de135c5176f327640dc78cefa9ee2 Mon Sep 17 00:00:00 2001 From: Andrew Erickson Date: Fri, 20 Feb 2026 16:02:30 -0800 Subject: [PATCH 03/11] pulling vars out --- provisioners/linux/bootstrap_linux.sh | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/provisioners/linux/bootstrap_linux.sh b/provisioners/linux/bootstrap_linux.sh index 6b5b318d28..efc61dd9a2 100644 --- a/provisioners/linux/bootstrap_linux.sh +++ b/provisioners/linux/bootstrap_linux.sh @@ -190,19 +190,23 @@ else exit 1 fi +# PKG_TO_INSTALL="puppet-agent" +PKG_TO_INSTALL="openvox-agent" +TEMP_DEB_NAME="bootstrap_temp.deb" + # if on ubuntu 24.04, install puppet 8, else install puppet 7 if [ "$VERSION_ID" = "24.04" ]; then echo "Installing Puppet 8..." # TODO: use openvox - wget https://apt.puppetlabs.com/puppet8-release-noble.deb -O /tmp/puppet.deb + wget https://apt.puppetlabs.com/puppet8-release-noble.deb -O /tmp/${TEMP_DEB_NAME} # install puppet release deb for the version we've selected - dpkg -i /tmp/puppet.deb + dpkg -i /tmp/${TEMP_DEB_NAME} # update apt and install puppet-agent and ntp apt-get update # shellcheck disable=SC2090 DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::='--force-confnew' remove -y puppet # shellcheck disable=SC2090 - DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::='--force-confnew' install -y puppet-agent + DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::='--force-confnew' install -y ${PKG_TO_INSTALL} # get clock synced. if clock is way off, run-puppet.sh will never finish # it's git clone because the SSL cert will appear invalid. @@ -215,15 +219,15 @@ if [ "$VERSION_ID" = "24.04" ]; then elif [ "$VERSION_ID" = "18.04" ]; then echo "Installing Puppet 7..." # TODO: use openvox and puppet 8 - wget https://apt.puppetlabs.com/puppet7-release-bionic.deb -O /tmp/puppet.deb + wget https://apt.puppetlabs.com/puppet7-release-bionic.deb -O /tmp/${TEMP_DEB_NAME} # install puppet release deb for the version we've selected - dpkg -i /tmp/puppet.deb + dpkg -i /tmp/${TEMP_DEB_NAME} # update apt and install puppet-agent and ntp apt-get update # shellcheck disable=SC2090 DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::='--force-confnew' remove -y puppet # shellcheck disable=SC2090 - DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::='--force-confnew' install -y puppet-agent ntp + DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::='--force-confnew' install -y ${PKG_TO_INSTALL} ntp # get clock synced. if clock is way off, run-puppet.sh will never finish # it's git clone because the SSL cert will appear invalid. From f1b061fa3a85d942acaf8490294aa01eb71d7405 Mon Sep 17 00:00:00 2001 From: Andrew Erickson Date: Fri, 20 Feb 2026 16:16:00 -0800 Subject: [PATCH 04/11] refactoring --- provisioners/linux/bootstrap_linux.sh | 30 ++++++++++++++++++++------- 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/provisioners/linux/bootstrap_linux.sh b/provisioners/linux/bootstrap_linux.sh index efc61dd9a2..c46fe4eda8 100644 --- a/provisioners/linux/bootstrap_linux.sh +++ b/provisioners/linux/bootstrap_linux.sh @@ -190,15 +190,30 @@ else exit 1 fi +TEMP_DEB_NAME="bootstrap_temp.deb" +# noble, bionic, etc (CODENAME in /etc/lsb-release) on ubuntu +# VERSION_CODENAME=$(lsb_release -cs 2>/dev/null) +# 18.04, 24.04, etc +VERSION_ID=$(lsb_release -rs 2>/dev/null) + +# puppet vars +# # PKG_TO_INSTALL="puppet-agent" +# INSTALL_URL_BASE="https://apt.puppetlabs.com" +# INSTALL_URL_DEB="puppet8-release-noble.deb" +# INSTALL_URL_DEB="puppet7-release-bionic.deb" + +# openvox vars +# PKG_TO_INSTALL="openvox-agent" -TEMP_DEB_NAME="bootstrap_temp.deb" +INSTALL_URL_BASE="https://apt.voxpupuli.org" +INSTALL_URL_DEB="openvox8-release-ubuntu${VERSION_ID}.deb" -# if on ubuntu 24.04, install puppet 8, else install puppet 7 +# we install openvox 8 on both, but NTP setup differs if [ "$VERSION_ID" = "24.04" ]; then - echo "Installing Puppet 8..." - # TODO: use openvox - wget https://apt.puppetlabs.com/puppet8-release-noble.deb -O /tmp/${TEMP_DEB_NAME} + echo "Installing Openvox 8..." + + wget "${INSTALL_URL_BASE}/${INSTALL_URL_DEB}" -O /tmp/${TEMP_DEB_NAME} # install puppet release deb for the version we've selected dpkg -i /tmp/${TEMP_DEB_NAME} # update apt and install puppet-agent and ntp @@ -217,9 +232,10 @@ if [ "$VERSION_ID" = "24.04" ]; then echo -e "[Time]\nNTP=ntp.build.mozilla.org" >/etc/systemd/timesyncd.conf.d/mozilla.conf systemctl restart systemd-timesyncd elif [ "$VERSION_ID" = "18.04" ]; then - echo "Installing Puppet 7..." + echo "Installing Openvox 8..." + # TODO: use openvox and puppet 8 - wget https://apt.puppetlabs.com/puppet7-release-bionic.deb -O /tmp/${TEMP_DEB_NAME} + wget "${INSTALL_URL_BASE}/${INSTALL_URL_DEB}" -O /tmp/${TEMP_DEB_NAME} # install puppet release deb for the version we've selected dpkg -i /tmp/${TEMP_DEB_NAME} # update apt and install puppet-agent and ntp From 438b0a9ea81da37a4c15fc385b42377fdb8390e3 Mon Sep 17 00:00:00 2001 From: Andrew Erickson Date: Fri, 20 Feb 2026 16:25:28 -0800 Subject: [PATCH 05/11] cleanup and simplification --- provisioners/linux/bootstrap_linux.sh | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/provisioners/linux/bootstrap_linux.sh b/provisioners/linux/bootstrap_linux.sh index c46fe4eda8..7b837ce9e1 100644 --- a/provisioners/linux/bootstrap_linux.sh +++ b/provisioners/linux/bootstrap_linux.sh @@ -182,7 +182,7 @@ if [ ! -f /root/vault.yaml ]; then exit 1 fi -# if on ubuntu 22.04, install puppet 8, else install puppet 7 +# determine ubuntu version so we can set NTP method appropriately if [ -f /etc/os-release ]; then . /etc/os-release else @@ -191,10 +191,8 @@ else fi TEMP_DEB_NAME="bootstrap_temp.deb" -# noble, bionic, etc (CODENAME in /etc/lsb-release) on ubuntu -# VERSION_CODENAME=$(lsb_release -cs 2>/dev/null) -# 18.04, 24.04, etc -VERSION_ID=$(lsb_release -rs 2>/dev/null) +# noble, bionic, etc (VERSION_CODENAME) and 18.04, 24.04, etc (VERSION_ID) +# are sourced from /etc/os-release above # puppet vars # @@ -221,7 +219,7 @@ if [ "$VERSION_ID" = "24.04" ]; then # shellcheck disable=SC2090 DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::='--force-confnew' remove -y puppet # shellcheck disable=SC2090 - DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::='--force-confnew' install -y ${PKG_TO_INSTALL} + DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::='--force-confnew' install -y "${PKG_TO_INSTALL}" # get clock synced. if clock is way off, run-puppet.sh will never finish # it's git clone because the SSL cert will appear invalid. @@ -234,7 +232,6 @@ if [ "$VERSION_ID" = "24.04" ]; then elif [ "$VERSION_ID" = "18.04" ]; then echo "Installing Openvox 8..." - # TODO: use openvox and puppet 8 wget "${INSTALL_URL_BASE}/${INSTALL_URL_DEB}" -O /tmp/${TEMP_DEB_NAME} # install puppet release deb for the version we've selected dpkg -i /tmp/${TEMP_DEB_NAME} @@ -243,7 +240,7 @@ elif [ "$VERSION_ID" = "18.04" ]; then # shellcheck disable=SC2090 DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::='--force-confnew' remove -y puppet # shellcheck disable=SC2090 - DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::='--force-confnew' install -y ${PKG_TO_INSTALL} ntp + DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::='--force-confnew' install -y "${PKG_TO_INSTALL}" ntp # get clock synced. if clock is way off, run-puppet.sh will never finish # it's git clone because the SSL cert will appear invalid. From fed44a46ff215799e3735d767bb17ba2027898cf Mon Sep 17 00:00:00 2001 From: Andrew Erickson Date: Fri, 20 Feb 2026 16:44:41 -0800 Subject: [PATCH 06/11] add code to skip ntp, make test use flag --- provisioners/linux/bootstrap_linux.sh | 32 ++++++----- provisioners/linux/test/test_bootstrap.sh | 68 +++++++++++++++++++++++ 2 files changed, 86 insertions(+), 14 deletions(-) create mode 100755 provisioners/linux/test/test_bootstrap.sh diff --git a/provisioners/linux/bootstrap_linux.sh b/provisioners/linux/bootstrap_linux.sh index 7b837ce9e1..0f58e3394c 100644 --- a/provisioners/linux/bootstrap_linux.sh +++ b/provisioners/linux/bootstrap_linux.sh @@ -221,14 +221,16 @@ if [ "$VERSION_ID" = "24.04" ]; then # shellcheck disable=SC2090 DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::='--force-confnew' install -y "${PKG_TO_INSTALL}" - # get clock synced. if clock is way off, run-puppet.sh will never finish - # it's git clone because the SSL cert will appear invalid. - # - # 24.04 uses timesyncd (on by default), see `systemctl status systemd-timesyncd` - # place our config and restart the service - mkdir -p /etc/systemd/timesyncd.conf.d - echo -e "[Time]\nNTP=ntp.build.mozilla.org" >/etc/systemd/timesyncd.conf.d/mozilla.conf - systemctl restart systemd-timesyncd + if [ "${SKIP_NTP:-false}" != "true" ]; then + # get clock synced. if clock is way off, run-puppet.sh will never finish + # it's git clone because the SSL cert will appear invalid. + # + # 24.04 uses timesyncd (on by default), see `systemctl status systemd-timesyncd` + # place our config and restart the service + mkdir -p /etc/systemd/timesyncd.conf.d + echo -e "[Time]\nNTP=ntp.build.mozilla.org" >/etc/systemd/timesyncd.conf.d/mozilla.conf + systemctl restart systemd-timesyncd + fi elif [ "$VERSION_ID" = "18.04" ]; then echo "Installing Openvox 8..." @@ -242,12 +244,14 @@ elif [ "$VERSION_ID" = "18.04" ]; then # shellcheck disable=SC2090 DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::='--force-confnew' install -y "${PKG_TO_INSTALL}" ntp - # get clock synced. if clock is way off, run-puppet.sh will never finish - # it's git clone because the SSL cert will appear invalid. - /etc/init.d/ntp stop - echo "server ntp.build.mozilla.org iburst" >/etc/ntp.conf # place barebones config - ntpd -q -g # runs once and force allows huge skews - /etc/init.d/ntp start + if [ "${SKIP_NTP:-false}" != "true" ]; then + # get clock synced. if clock is way off, run-puppet.sh will never finish + # it's git clone because the SSL cert will appear invalid. + /etc/init.d/ntp stop + echo "server ntp.build.mozilla.org iburst" >/etc/ntp.conf # place barebones config + ntpd -q -g # runs once and force allows huge skews + /etc/init.d/ntp start + fi else echo "Unsupported Ubuntu version: $VERSION_ID. This script only supports Ubuntu 18.04 and 24.04." exit 1 diff --git a/provisioners/linux/test/test_bootstrap.sh b/provisioners/linux/test/test_bootstrap.sh new file mode 100755 index 0000000000..73d6d1c19d --- /dev/null +++ b/provisioners/linux/test/test_bootstrap.sh @@ -0,0 +1,68 @@ +#!/bin/bash +# Tests that bootstrap_linux.sh successfully installs openvox-agent on a given +# Ubuntu version. The script is expected to exit early at the NTP/systemd step +# since Docker containers don't have an init system — that's fine, openvox is +# installed before that point. +# +# Usage: ./test_bootstrap.sh +# ./test_bootstrap.sh 18.04 +# ./test_bootstrap.sh 24.04 + +set -e + +UBUNTU_VERSION="${1:-}" + +if [ -z "$UBUNTU_VERSION" ]; then + echo "Usage: $0 " + echo " $0 18.04" + echo " $0 24.04" + exit 1 +fi + +case "$UBUNTU_VERSION" in + 18.04|24.04) ;; + *) + echo "Unsupported version: $UBUNTU_VERSION (supported: 18.04, 24.04)" + exit 1 + ;; +esac + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +BOOTSTRAP_SCRIPT="${SCRIPT_DIR}/../bootstrap_linux.sh" + +if [ ! -f "$BOOTSTRAP_SCRIPT" ]; then + echo "Bootstrap script not found: $BOOTSTRAP_SCRIPT" + exit 1 +fi + +CONTAINER_ID="" +cleanup() { + if [ -n "$CONTAINER_ID" ]; then + echo "==> Cleaning up container" + docker rm -f "$CONTAINER_ID" >/dev/null 2>&1 || true + fi +} +trap cleanup EXIT + +echo "==> Starting ubuntu:${UBUNTU_VERSION} container" +CONTAINER_ID=$(docker run -d "ubuntu:${UBUNTU_VERSION}" sleep infinity) + +echo "==> Installing prerequisite: wget" +docker exec "$CONTAINER_ID" bash -c "apt-get update -qq && apt-get install -y -qq wget" + +echo "==> Copying bootstrap_linux.sh" +docker cp "$BOOTSTRAP_SCRIPT" "$CONTAINER_ID:/root/bootstrap_linux.sh" + +echo "==> Creating stub /root/vault.yaml" +docker exec "$CONTAINER_ID" bash -c "echo '---' > /root/vault.yaml" + +echo "==> Running bootstrap_linux.sh" +docker exec -e SKIP_NTP=true "$CONTAINER_ID" bash /root/bootstrap_linux.sh || true + +echo "==> Checking openvox-agent installation" +if docker exec "$CONTAINER_ID" dpkg-query -W -f='${Status}' openvox-agent 2>/dev/null | grep -q "install ok installed"; then + echo "PASS: openvox-agent is installed on Ubuntu ${UBUNTU_VERSION}" +else + echo "FAIL: openvox-agent is not installed on Ubuntu ${UBUNTU_VERSION}" + exit 1 +fi From 4b24aefb08c73cdb04e88734f09ee1604f58da21 Mon Sep 17 00:00:00 2001 From: Andrew Erickson Date: Mon, 23 Feb 2026 11:54:19 -0800 Subject: [PATCH 07/11] update gitignore patterns --- provisioners/linux/.gitignore | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/provisioners/linux/.gitignore b/provisioners/linux/.gitignore index b8d8d80f4d..8ea5eada2d 100644 --- a/provisioners/linux/.gitignore +++ b/provisioners/linux/.gitignore @@ -1,3 +1,3 @@ -vault.yml -vault.yaml -ronin_settings +vault.yml* +vault.yaml* +ronin_settings* From 3fddcf90bd9e6c248409adcbc657add74f653269 Mon Sep 17 00:00:00 2001 From: Andrew Erickson Date: Mon, 23 Feb 2026 11:54:33 -0800 Subject: [PATCH 08/11] create a readme for provisioners/linux --- provisioners/linux/README.md | 103 +++++++++++++++++++++++++++++++++++ 1 file changed, 103 insertions(+) create mode 100644 provisioners/linux/README.md diff --git a/provisioners/linux/README.md b/provisioners/linux/README.md new file mode 100644 index 0000000000..d4cb60a8c1 --- /dev/null +++ b/provisioners/linux/README.md @@ -0,0 +1,103 @@ +# Linux Provisioners + +Scripts for bootstrapping and maintaining Linux (Ubuntu) hosts managed by ronin_puppet. + +## Scripts + +### `bootstrap_linux.sh` + +Bootstraps a fresh Linux host from post-install to a fully converged Puppet run. Installs +[OpenVox](https://voxpupuli.org/openvox/) (an open-source Puppet agent), configures NTP, +and runs Puppet until it succeeds. Supports Ubuntu 18.04 and 24.04. + +**Prerequisites:** +- `/root/vault.yaml` — hiera secrets file +- `/etc/puppet_role` — puppet role for this host (e.g. `gecko_t_linux_talos`) +- `wget` installed + +**Usage:** + +```bash +# Interactive mode +sudo ./bootstrap_linux.sh + +# Non-interactive (logs to file) +sudo ./bootstrap_linux.sh -l /var/log/bootstrap.log + +# Use a specific repo/branch during development +PUPPET_REPO="https://github.com/youruser/ronin_puppet.git" \ +PUPPET_BRANCH="your-branch" \ +sudo ./bootstrap_linux.sh +``` + +### `deliver_linux.sh` + +Delivers bootstrap prerequisites to a remote host over SSH and prints the commands needed +to kick off the bootstrap. Auto-detects whether to connect as `relops` or `root`. + +**Usage:** + +```bash +./deliver_linux.sh + +# Example +./deliver_linux.sh devicepool-0.relops.mozops.net gecko_t_linux_talos +``` + +The script copies `bootstrap_linux.sh`, `vault.yaml`, and (optionally) a `ronin_settings` +file to the remote host and sets the role file. It then prints the SSH commands to run the +bootstrap. + +### `update_linux.sh` + +Updates `vault.yaml` (hiera secrets) on an already-bootstrapped host. Can optionally +update the puppet role with `force`. + +**Usage:** + +```bash +./update_linux.sh [role] [force] + +# Update secrets only +./update_linux.sh myhost.example.com + +# Update secrets and change role (requires 'force') +./update_linux.sh myhost.example.com gecko_t_linux_talos force +``` + +### `bootstrap_bitbar_devicepool.sh` + +Bootstrap script specific to `bitbar_devicepool` hosts. Installs Puppet 7, r10k, and runs +a masterless Puppet apply against a git-cloned copy of the repo. + +## Configuration + +### `vault.yaml` + +Hiera secrets file. Must be present at `/root/vault.yaml` on the target host before +bootstrap runs. A `vault.yaml.bak` and `vault.yaml.bak2` are kept as backups. + +### `ronin_settings` / `ronin_settings.dis` + +Optional settings file that overrides `PUPPET_REPO`, `PUPPET_BRANCH`, and other variables. +Rename to `ronin_settings` (drop the `.dis` extension) to activate it. When present, +`deliver_linux.sh` will copy it to `/etc/puppet/ronin_settings` on the remote host. + +## Testing + +Bootstrap scripts can be tested locally using Docker via the script in `test/`. + +```bash +# Test against Ubuntu 24.04 +./test/test_bootstrap.sh 24.04 + +# Test against Ubuntu 18.04 +./test/test_bootstrap.sh 18.04 +``` + +The test script spins up an Ubuntu container, copies `bootstrap_linux.sh` into it, and +verifies that `openvox-agent` installs successfully. The script uses `SKIP_NTP=true` since +Docker containers don't have an init system — NTP setup is expected to fail, but +`openvox-agent` installation is validated. + +Requires Docker to be installed and running. From 3863b8fbc40c36e68edcb4debc3102b29083a215 Mon Sep 17 00:00:00 2001 From: Andrew Erickson Date: Wed, 25 Feb 2026 14:57:56 -0800 Subject: [PATCH 09/11] avoid hiera warning with debian.yaml --- data/os/Debian.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/data/os/Debian.yaml b/data/os/Debian.yaml index ed97d539c0..0358884c7a 100644 --- a/data/os/Debian.yaml +++ b/data/os/Debian.yaml @@ -1 +1,2 @@ ---- +--- {} +# A hash/dictionary is expected. This prevents a puppet load failure warning. From eb504e50524b124c6ba32856a858b32aea23fce8 Mon Sep 17 00:00:00 2001 From: Andrew Erickson Date: Thu, 26 Feb 2026 17:26:49 -0800 Subject: [PATCH 10/11] deliver_linux.sh: retry ssh-keyscan and auth to handle slow host startup --- provisioners/linux/deliver_linux.sh | 29 +++++++++++++++++++++++------ 1 file changed, 23 insertions(+), 6 deletions(-) diff --git a/provisioners/linux/deliver_linux.sh b/provisioners/linux/deliver_linux.sh index ae6d53ac00..320f28b751 100755 --- a/provisioners/linux/deliver_linux.sh +++ b/provisioners/linux/deliver_linux.sh @@ -15,12 +15,20 @@ set -e # # Detect which SSH user works (try relops first, then root) +# Retries for up to ~2 minutes to handle hosts where auth isn't ready immediately detect_ssh_user() { local host="$1" - for user in "relops" "root"; do - if ssh -q "$user"@"$host" exit 2>/dev/null; then - echo "$user" - return 0 + local max_attempts=12 + for attempt in $(seq 1 "$max_attempts"); do + for user in "relops" "root"; do + if ssh -o ConnectTimeout=5 -o BatchMode=yes -q "$user"@"$host" exit 2>/dev/null; then + echo "$user" + return 0 + fi + done + if [ "$attempt" -lt "$max_attempts" ]; then + echo " SSH auth not ready yet, retrying in 10s (attempt ${attempt}/${max_attempts})..." >&2 + sleep 10 fi done echo "" @@ -74,8 +82,17 @@ fi # cleanup ssh key, it will be new after kickstarting ssh-keygen -R "${THE_HOST}" || true -# readd to avoid prompts -ssh-keyscan -H "${THE_HOST}" >> ~/.ssh/known_hosts +# readd to avoid prompts; retry until SSH daemon is ready +echo "Waiting for SSH on ${THE_HOST}..." +for i in $(seq 1 30); do + result=$(ssh-keyscan -H "${THE_HOST}" 2>/dev/null) + if [ -n "$result" ]; then + echo "$result" >> ~/.ssh/known_hosts + break + fi + echo " attempt ${i}/30, retrying in 10s..." + sleep 10 +done # detect which SSH user works echo "Detecting SSH user for ${THE_HOST}..." From 1f7604fe10af603e44a2ff6a0e0a18271e265f7c Mon Sep 17 00:00:00 2001 From: Andrew Erickson Date: Thu, 26 Feb 2026 17:49:10 -0800 Subject: [PATCH 11/11] devlier: retry more, simplify output when spinning --- provisioners/linux/deliver_linux.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/provisioners/linux/deliver_linux.sh b/provisioners/linux/deliver_linux.sh index 320f28b751..1d779a4943 100755 --- a/provisioners/linux/deliver_linux.sh +++ b/provisioners/linux/deliver_linux.sh @@ -15,10 +15,10 @@ set -e # # Detect which SSH user works (try relops first, then root) -# Retries for up to ~2 minutes to handle hosts where auth isn't ready immediately +# Retries for up to ~10 minutes to handle hosts where auth isn't ready immediately detect_ssh_user() { local host="$1" - local max_attempts=12 + local max_attempts=60 for attempt in $(seq 1 "$max_attempts"); do for user in "relops" "root"; do if ssh -o ConnectTimeout=5 -o BatchMode=yes -q "$user"@"$host" exit 2>/dev/null; then @@ -27,7 +27,7 @@ detect_ssh_user() { fi done if [ "$attempt" -lt "$max_attempts" ]; then - echo " SSH auth not ready yet, retrying in 10s (attempt ${attempt}/${max_attempts})..." >&2 + printf '.' >&2 sleep 10 fi done @@ -84,13 +84,13 @@ fi ssh-keygen -R "${THE_HOST}" || true # readd to avoid prompts; retry until SSH daemon is ready echo "Waiting for SSH on ${THE_HOST}..." -for i in $(seq 1 30); do +for _i in $(seq 1 60); do result=$(ssh-keyscan -H "${THE_HOST}" 2>/dev/null) if [ -n "$result" ]; then echo "$result" >> ~/.ssh/known_hosts break fi - echo " attempt ${i}/30, retrying in 10s..." + printf '.' sleep 10 done