infra: add optional data volume bind-mount phase

infra/README.md

@@ -51,6 +51,7 @@ Logs go to `/var/log/bli-bootstrap.log`. Phases are idempotent: re-running recon
 These happen outside the box and stay manual:
 
 - **Provision the Hetzner instance** with `hcloud server create --type cax21 --image ubuntu-24.04 --location hel1 ...`
+- **(Optional) Attach a data Volume.** The CAX21's 80GB root disk fills up once `cache.db` and its daily backups grow. Create a Hetzner Cloud Volume in the same location (50GB+ recommended), attach it to this server with auto-mount enabled (ext4), and set `LYRICS_API_VOLUME_ID` in `secrets.env`. Phase 11 picks it up, migrates any existing `/var/lib/lyrics-api` contents onto it, and adds a fstab bind mount so DB and backup writes land on the volume.
 - **DNS records in Cloudflare** for the five hostnames in `secrets.env` (primary, staging, logs, metrics, keep) plus the preview wildcard, all proxied (orange cloud)
 - **Beszel hub** running somewhere reachable, with an agent slot for this host. The hub UI hands you the KEY/TOKEN pair for `secrets.env`.
 - **keep first-run setup**. After phase 05 puts keep up at `https://$KEEP_DOMAIN`, browse to it and complete `/setup`: pick a master password (save it to a password manager), scan TOTP, save the 8 recovery codes offline. Then create project `lyrics-api`, env `prod`, bulk-import the env via the .env paste UI.

infra/phases/11-data-volume.sh

@@ -0,0 +1,82 @@
+#!/bin/bash
+set -euo pipefail
+
+# Optional: attach a Hetzner Cloud Volume as the data dir for lyrics-api.
+# Set LYRICS_API_VOLUME_ID in secrets.env (just the numeric ID from the Hetzner
+# console). The volume must already be created, attached to this server, and
+# auto-mounted by Hetzner at /mnt/HC_Volume_<id>. This phase bind-mounts a
+# subdir of the volume over /var/lib/lyrics-api so all DB + backup writes land
+# on the volume instead of the root disk. Safe to omit entirely if you don't
+# need extra storage.
+
+if [ -z "${LYRICS_API_VOLUME_ID:-}" ]; then
+    echo "OK: LYRICS_API_VOLUME_ID unset, skipping data-volume phase"
+    exit 0
+fi
+
+VOL_ID="$LYRICS_API_VOLUME_ID"
+VOL_MOUNT="/mnt/HC_Volume_${VOL_ID}"
+VOL_SUBDIR="${VOL_MOUNT}/lyrics-api"
+APP_DIR="/var/lib/lyrics-api"
+FSTAB_LINE="${VOL_SUBDIR} ${APP_DIR} none bind,nofail,x-systemd.requires-mounts-for=${VOL_MOUNT} 0 0"
+DROPIN=/etc/systemd/system/lyrics-api.service.d/data-volume.conf
+
+# Volume must be attached and mounted before we can bind onto it
+if ! mountpoint -q "$VOL_MOUNT"; then
+    echo "ERROR: Hetzner volume $VOL_ID is not mounted at $VOL_MOUNT" >&2
+    echo "       Attach it to this server in the Hetzner console with auto-mount enabled," >&2
+    echo "       confirm it appears in 'df -h', then re-run this phase." >&2
+    exit 1
+fi
+
+install -d -o deploy -g deploy -m 755 "$VOL_SUBDIR"
+
+# If already bind-mounted to our source, nothing to migrate
+if mountpoint -q "$APP_DIR" && findmnt -no SOURCE "$APP_DIR" | grep -qE "(^|\W)${VOL_SUBDIR}(\W|$)"; then
+    echo "OK: $APP_DIR already bind-mounted from $VOL_SUBDIR"
+else
+    # Migrate existing data if APP_DIR is a regular dir with content
+    if [ -d "$APP_DIR" ] && ! mountpoint -q "$APP_DIR" && [ -n "$(ls -A "$APP_DIR" 2>/dev/null)" ]; then
+        echo "Migrating existing data from $APP_DIR to $VOL_SUBDIR"
+        WAS_RUNNING=0
+        if systemctl is-active --quiet lyrics-api; then
+            WAS_RUNNING=1
+            systemctl stop lyrics-api
+            # Arm a restart trap before any operation that could fail mid-migration
+            # (rsync, mv, mount, etc.). The trap fires on every script exit path,
+            # so the service always comes back up, even if a later step blows up.
+            # Idempotent: systemctl start is a no-op if the service is already
+            # running by the time the trap fires on a successful run.
+            trap 'systemctl start lyrics-api' EXIT
+        fi
+        rsync -aHAX --info=progress2 "${APP_DIR}/" "${VOL_SUBDIR}/"
+        BACKUP_NAME="${APP_DIR}.pre-volume.$(date -u +%Y%m%d-%H%M%S)"
+        mv "$APP_DIR" "$BACKUP_NAME"
+        echo "Original preserved at $BACKUP_NAME (delete once you've verified the migration)"
+    fi
+
+    install -d -o deploy -g deploy -m 755 "$APP_DIR"
+
+    if ! grep -qE "^\S+\s+${APP_DIR}\s+none\s+bind" /etc/fstab; then
+        echo "$FSTAB_LINE" >> /etc/fstab
+    fi
+
+    mount "$APP_DIR"
+fi
+
+# Refuse to start the service if the bind mount is missing, so a failed mount
+# can never silently let lyrics-api write a fresh empty cache.db on root.
+install -d -m 755 /etc/systemd/system/lyrics-api.service.d
+cat > "$DROPIN" <<CONF
+[Unit]
+RequiresMountsFor=${APP_DIR}
+CONF
+systemctl daemon-reload
+
+# Bring the service back if we stopped it for migration
+if [ "${WAS_RUNNING:-0}" = "1" ]; then
+    systemctl start lyrics-api
+fi
+
+findmnt "$APP_DIR"
+echo "OK: data volume $VOL_ID bind-mounted at $APP_DIR"

infra/secrets.env.example

@@ -29,5 +29,13 @@ METRICS_DOMAIN=
 KEEP_DOMAIN=
 PREVIEW_WILDCARD=
 
+# === Optional: Hetzner Cloud Volume for lyrics-api data ===
+# If set, phase 11 bind-mounts the volume's lyrics-api/ subdir over
+# /var/lib/lyrics-api so cache.db + backups live on the volume instead of the
+# root disk. Just the numeric volume ID from the Hetzner console (the volume
+# must already be attached to this server with auto-mount enabled). Leave
+# empty to keep everything on the root disk.
+LYRICS_API_VOLUME_ID=
+
 # === Pinned versions (bump deliberately, not silently) ===
 LOGDY_VERSION=0.17.0