diff --git a/data/os/Debian.yaml b/data/os/Debian.yaml index ed97d539c..0358884c7 100644 --- a/data/os/Debian.yaml +++ b/data/os/Debian.yaml @@ -1 +1,2 @@ ---- +--- {} +# A hash/dictionary is expected. This prevents a puppet load failure warning. diff --git a/provisioners/linux/.gitignore b/provisioners/linux/.gitignore index b8d8d80f4..8ea5eada2 100644 --- a/provisioners/linux/.gitignore +++ b/provisioners/linux/.gitignore @@ -1,3 +1,3 @@ -vault.yml -vault.yaml -ronin_settings +vault.yml* +vault.yaml* +ronin_settings* diff --git a/provisioners/linux/README.md b/provisioners/linux/README.md new file mode 100644 index 000000000..d4cb60a8c --- /dev/null +++ b/provisioners/linux/README.md @@ -0,0 +1,103 @@ +# Linux Provisioners + +Scripts for bootstrapping and maintaining Linux (Ubuntu) hosts managed by ronin_puppet. + +## Scripts + +### `bootstrap_linux.sh` + +Bootstraps a fresh Linux host from post-install to a fully converged Puppet run. Installs +[OpenVox](https://voxpupuli.org/openvox/) (an open-source Puppet agent), configures NTP, +and runs Puppet until it succeeds. Supports Ubuntu 18.04 and 24.04. + +**Prerequisites:** +- `/root/vault.yaml` — hiera secrets file +- `/etc/puppet_role` — puppet role for this host (e.g. `gecko_t_linux_talos`) +- `wget` installed + +**Usage:** + +```bash +# Interactive mode +sudo ./bootstrap_linux.sh + +# Non-interactive (logs to file) +sudo ./bootstrap_linux.sh -l /var/log/bootstrap.log + +# Use a specific repo/branch during development +PUPPET_REPO="https://github.com/youruser/ronin_puppet.git" \ +PUPPET_BRANCH="your-branch" \ +sudo ./bootstrap_linux.sh +``` + +### `deliver_linux.sh` + +Delivers bootstrap prerequisites to a remote host over SSH and prints the commands needed +to kick off the bootstrap. Auto-detects whether to connect as `relops` or `root`. + +**Usage:** + +```bash +./deliver_linux.sh + +# Example +./deliver_linux.sh devicepool-0.relops.mozops.net gecko_t_linux_talos +``` + +The script copies `bootstrap_linux.sh`, `vault.yaml`, and (optionally) a `ronin_settings` +file to the remote host and sets the role file. It then prints the SSH commands to run the +bootstrap. + +### `update_linux.sh` + +Updates `vault.yaml` (hiera secrets) on an already-bootstrapped host. Can optionally +update the puppet role with `force`. + +**Usage:** + +```bash +./update_linux.sh [role] [force] + +# Update secrets only +./update_linux.sh myhost.example.com + +# Update secrets and change role (requires 'force') +./update_linux.sh myhost.example.com gecko_t_linux_talos force +``` + +### `bootstrap_bitbar_devicepool.sh` + +Bootstrap script specific to `bitbar_devicepool` hosts. Installs Puppet 7, r10k, and runs +a masterless Puppet apply against a git-cloned copy of the repo. + +## Configuration + +### `vault.yaml` + +Hiera secrets file. Must be present at `/root/vault.yaml` on the target host before +bootstrap runs. A `vault.yaml.bak` and `vault.yaml.bak2` are kept as backups. + +### `ronin_settings` / `ronin_settings.dis` + +Optional settings file that overrides `PUPPET_REPO`, `PUPPET_BRANCH`, and other variables. +Rename to `ronin_settings` (drop the `.dis` extension) to activate it. When present, +`deliver_linux.sh` will copy it to `/etc/puppet/ronin_settings` on the remote host. + +## Testing + +Bootstrap scripts can be tested locally using Docker via the script in `test/`. + +```bash +# Test against Ubuntu 24.04 +./test/test_bootstrap.sh 24.04 + +# Test against Ubuntu 18.04 +./test/test_bootstrap.sh 18.04 +``` + +The test script spins up an Ubuntu container, copies `bootstrap_linux.sh` into it, and +verifies that `openvox-agent` installs successfully. The script uses `SKIP_NTP=true` since +Docker containers don't have an init system — NTP setup is expected to fail, but +`openvox-agent` installation is validated. + +Requires Docker to be installed and running. diff --git a/provisioners/linux/bootstrap_linux.sh b/provisioners/linux/bootstrap_linux.sh index cf83aefba..0f58e3394 100644 --- a/provisioners/linux/bootstrap_linux.sh +++ b/provisioners/linux/bootstrap_linux.sh @@ -182,7 +182,7 @@ if [ ! -f /root/vault.yaml ]; then exit 1 fi -# if on ubuntu 22.04, install puppet 8, else install puppet 7 +# determine ubuntu version so we can set NTP method appropriately if [ -f /etc/os-release ]; then . /etc/os-release else @@ -190,41 +190,68 @@ else exit 1 fi -# if on ubuntu 24.04, install puppet 8, else install puppet 7 +TEMP_DEB_NAME="bootstrap_temp.deb" +# noble, bionic, etc (VERSION_CODENAME) and 18.04, 24.04, etc (VERSION_ID) +# are sourced from /etc/os-release above + +# puppet vars +# +# PKG_TO_INSTALL="puppet-agent" +# INSTALL_URL_BASE="https://apt.puppetlabs.com" +# INSTALL_URL_DEB="puppet8-release-noble.deb" +# INSTALL_URL_DEB="puppet7-release-bionic.deb" + +# openvox vars +# +PKG_TO_INSTALL="openvox-agent" +INSTALL_URL_BASE="https://apt.voxpupuli.org" +INSTALL_URL_DEB="openvox8-release-ubuntu${VERSION_ID}.deb" + +# we install openvox 8 on both, but NTP setup differs if [ "$VERSION_ID" = "24.04" ]; then - echo "Installing Puppet 8..." - wget https://apt.puppetlabs.com/puppet8-release-noble.deb -O /tmp/puppet.deb + echo "Installing Openvox 8..." + + wget "${INSTALL_URL_BASE}/${INSTALL_URL_DEB}" -O /tmp/${TEMP_DEB_NAME} # install puppet release deb for the version we've selected - dpkg -i /tmp/puppet.deb + dpkg -i /tmp/${TEMP_DEB_NAME} # update apt and install puppet-agent and ntp apt-get update # shellcheck disable=SC2090 DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::='--force-confnew' remove -y puppet # shellcheck disable=SC2090 - DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::='--force-confnew' install -y puppet-agent - # 24.04 uses timesyncd (on by default), see `systemctl status systemd-timesyncd` - # place our config and restart the service - mkdir -p /etc/systemd/timesyncd.conf.d - echo -e "[Time]\nNTP=ntp.build.mozilla.org" >/etc/systemd/timesyncd.conf.d/mozilla.conf - systemctl restart systemd-timesyncd + DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::='--force-confnew' install -y "${PKG_TO_INSTALL}" + + if [ "${SKIP_NTP:-false}" != "true" ]; then + # get clock synced. if clock is way off, run-puppet.sh will never finish + # it's git clone because the SSL cert will appear invalid. + # + # 24.04 uses timesyncd (on by default), see `systemctl status systemd-timesyncd` + # place our config and restart the service + mkdir -p /etc/systemd/timesyncd.conf.d + echo -e "[Time]\nNTP=ntp.build.mozilla.org" >/etc/systemd/timesyncd.conf.d/mozilla.conf + systemctl restart systemd-timesyncd + fi elif [ "$VERSION_ID" = "18.04" ]; then - echo "Installing Puppet 7..." - wget https://apt.puppetlabs.com/puppet7-release-bionic.deb -O /tmp/puppet.deb + echo "Installing Openvox 8..." + + wget "${INSTALL_URL_BASE}/${INSTALL_URL_DEB}" -O /tmp/${TEMP_DEB_NAME} # install puppet release deb for the version we've selected - dpkg -i /tmp/puppet.deb + dpkg -i /tmp/${TEMP_DEB_NAME} # update apt and install puppet-agent and ntp apt-get update # shellcheck disable=SC2090 DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::='--force-confnew' remove -y puppet # shellcheck disable=SC2090 - DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::='--force-confnew' install -y puppet-agent ntp - - # get clock synced. if clock is way off, run-puppet.sh will never finish - # it's git clone because the SSL cert will appear invalid. - /etc/init.d/ntp stop - echo "server ntp.build.mozilla.org iburst" >/etc/ntp.conf # place barebones config - ntpd -q -g # runs once and force allows huge skews - /etc/init.d/ntp start + DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::='--force-confnew' install -y "${PKG_TO_INSTALL}" ntp + + if [ "${SKIP_NTP:-false}" != "true" ]; then + # get clock synced. if clock is way off, run-puppet.sh will never finish + # it's git clone because the SSL cert will appear invalid. + /etc/init.d/ntp stop + echo "server ntp.build.mozilla.org iburst" >/etc/ntp.conf # place barebones config + ntpd -q -g # runs once and force allows huge skews + /etc/init.d/ntp start + fi else echo "Unsupported Ubuntu version: $VERSION_ID. This script only supports Ubuntu 18.04 and 24.04." exit 1 diff --git a/provisioners/linux/deliver_linux.sh b/provisioners/linux/deliver_linux.sh index d6d6009e3..9522b3da8 100755 --- a/provisioners/linux/deliver_linux.sh +++ b/provisioners/linux/deliver_linux.sh @@ -15,12 +15,20 @@ set -e # # Detect which SSH user works (try relops first, then root) +# Retries for up to ~10 minutes to handle hosts where auth isn't ready immediately detect_ssh_user() { local host="$1" - for user in "relops" "root" ; do - if ssh -o ConnectTimeout=5 -o BatchMode=yes -q "$user"@"$host" exit 2>/dev/null; then - echo "$user" - return 0 + local max_attempts=60 + for attempt in $(seq 1 "$max_attempts"); do + for user in "relops" "root"; do + if ssh -o ConnectTimeout=5 -o BatchMode=yes -q "$user"@"$host" exit 2>/dev/null; then + echo "$user" + return 0 + fi + done + if [ "$attempt" -lt "$max_attempts" ]; then + printf '.' >&2 + sleep 10 fi done echo "" @@ -74,8 +82,17 @@ fi # cleanup ssh key, it will be new after kickstarting ssh-keygen -R "${THE_HOST}" || true -# readd to avoid prompts -ssh-keyscan -H "${THE_HOST}" >> ~/.ssh/known_hosts +# readd to avoid prompts; retry until SSH daemon is ready +echo "Waiting for SSH on ${THE_HOST}..." +for _i in $(seq 1 60); do + result=$(ssh-keyscan -H "${THE_HOST}" 2>/dev/null) + if [ -n "$result" ]; then + echo "$result" >> ~/.ssh/known_hosts + break + fi + printf '.' + sleep 10 +done # detect which SSH user works echo "Detecting SSH user for ${THE_HOST}..." diff --git a/provisioners/linux/test/test_bootstrap.sh b/provisioners/linux/test/test_bootstrap.sh new file mode 100755 index 000000000..73d6d1c19 --- /dev/null +++ b/provisioners/linux/test/test_bootstrap.sh @@ -0,0 +1,68 @@ +#!/bin/bash +# Tests that bootstrap_linux.sh successfully installs openvox-agent on a given +# Ubuntu version. The script is expected to exit early at the NTP/systemd step +# since Docker containers don't have an init system — that's fine, openvox is +# installed before that point. +# +# Usage: ./test_bootstrap.sh +# ./test_bootstrap.sh 18.04 +# ./test_bootstrap.sh 24.04 + +set -e + +UBUNTU_VERSION="${1:-}" + +if [ -z "$UBUNTU_VERSION" ]; then + echo "Usage: $0 " + echo " $0 18.04" + echo " $0 24.04" + exit 1 +fi + +case "$UBUNTU_VERSION" in + 18.04|24.04) ;; + *) + echo "Unsupported version: $UBUNTU_VERSION (supported: 18.04, 24.04)" + exit 1 + ;; +esac + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +BOOTSTRAP_SCRIPT="${SCRIPT_DIR}/../bootstrap_linux.sh" + +if [ ! -f "$BOOTSTRAP_SCRIPT" ]; then + echo "Bootstrap script not found: $BOOTSTRAP_SCRIPT" + exit 1 +fi + +CONTAINER_ID="" +cleanup() { + if [ -n "$CONTAINER_ID" ]; then + echo "==> Cleaning up container" + docker rm -f "$CONTAINER_ID" >/dev/null 2>&1 || true + fi +} +trap cleanup EXIT + +echo "==> Starting ubuntu:${UBUNTU_VERSION} container" +CONTAINER_ID=$(docker run -d "ubuntu:${UBUNTU_VERSION}" sleep infinity) + +echo "==> Installing prerequisite: wget" +docker exec "$CONTAINER_ID" bash -c "apt-get update -qq && apt-get install -y -qq wget" + +echo "==> Copying bootstrap_linux.sh" +docker cp "$BOOTSTRAP_SCRIPT" "$CONTAINER_ID:/root/bootstrap_linux.sh" + +echo "==> Creating stub /root/vault.yaml" +docker exec "$CONTAINER_ID" bash -c "echo '---' > /root/vault.yaml" + +echo "==> Running bootstrap_linux.sh" +docker exec -e SKIP_NTP=true "$CONTAINER_ID" bash /root/bootstrap_linux.sh || true + +echo "==> Checking openvox-agent installation" +if docker exec "$CONTAINER_ID" dpkg-query -W -f='${Status}' openvox-agent 2>/dev/null | grep -q "install ok installed"; then + echo "PASS: openvox-agent is installed on Ubuntu ${UBUNTU_VERSION}" +else + echo "FAIL: openvox-agent is not installed on Ubuntu ${UBUNTU_VERSION}" + exit 1 +fi