diff --git a/infra/README.md b/infra/README.md index 02a7852..1560927 100644 --- a/infra/README.md +++ b/infra/README.md @@ -51,6 +51,7 @@ Logs go to `/var/log/bli-bootstrap.log`. Phases are idempotent: re-running recon These happen outside the box and stay manual: - **Provision the Hetzner instance** with `hcloud server create --type cax21 --image ubuntu-24.04 --location hel1 ...` +- **(Optional) Attach a data Volume.** The CAX21's 80GB root disk fills up once `cache.db` and its daily backups grow. Create a Hetzner Cloud Volume in the same location (50GB+ recommended), attach it to this server with auto-mount enabled (ext4), and set `LYRICS_API_VOLUME_ID` in `secrets.env`. Phase 11 picks it up, migrates any existing `/var/lib/lyrics-api` contents onto it, and adds a fstab bind mount so DB and backup writes land on the volume. - **DNS records in Cloudflare** for the five hostnames in `secrets.env` (primary, staging, logs, metrics, keep) plus the preview wildcard, all proxied (orange cloud) - **Beszel hub** running somewhere reachable, with an agent slot for this host. The hub UI hands you the KEY/TOKEN pair for `secrets.env`. - **keep first-run setup**. After phase 05 puts keep up at `https://$KEEP_DOMAIN`, browse to it and complete `/setup`: pick a master password (save it to a password manager), scan TOTP, save the 8 recovery codes offline. Then create project `lyrics-api`, env `prod`, bulk-import the env via the .env paste UI. diff --git a/infra/phases/11-data-volume.sh b/infra/phases/11-data-volume.sh new file mode 100755 index 0000000..b7ace88 --- /dev/null +++ b/infra/phases/11-data-volume.sh @@ -0,0 +1,82 @@ +#!/bin/bash +set -euo pipefail + +# Optional: attach a Hetzner Cloud Volume as the data dir for lyrics-api. +# Set LYRICS_API_VOLUME_ID in secrets.env (just the numeric ID from the Hetzner +# console). The volume must already be created, attached to this server, and +# auto-mounted by Hetzner at /mnt/HC_Volume_. This phase bind-mounts a +# subdir of the volume over /var/lib/lyrics-api so all DB + backup writes land +# on the volume instead of the root disk. Safe to omit entirely if you don't +# need extra storage. + +if [ -z "${LYRICS_API_VOLUME_ID:-}" ]; then + echo "OK: LYRICS_API_VOLUME_ID unset, skipping data-volume phase" + exit 0 +fi + +VOL_ID="$LYRICS_API_VOLUME_ID" +VOL_MOUNT="/mnt/HC_Volume_${VOL_ID}" +VOL_SUBDIR="${VOL_MOUNT}/lyrics-api" +APP_DIR="/var/lib/lyrics-api" +FSTAB_LINE="${VOL_SUBDIR} ${APP_DIR} none bind,nofail,x-systemd.requires-mounts-for=${VOL_MOUNT} 0 0" +DROPIN=/etc/systemd/system/lyrics-api.service.d/data-volume.conf + +# Volume must be attached and mounted before we can bind onto it +if ! mountpoint -q "$VOL_MOUNT"; then + echo "ERROR: Hetzner volume $VOL_ID is not mounted at $VOL_MOUNT" >&2 + echo " Attach it to this server in the Hetzner console with auto-mount enabled," >&2 + echo " confirm it appears in 'df -h', then re-run this phase." >&2 + exit 1 +fi + +install -d -o deploy -g deploy -m 755 "$VOL_SUBDIR" + +# If already bind-mounted to our source, nothing to migrate +if mountpoint -q "$APP_DIR" && findmnt -no SOURCE "$APP_DIR" | grep -qE "(^|\W)${VOL_SUBDIR}(\W|$)"; then + echo "OK: $APP_DIR already bind-mounted from $VOL_SUBDIR" +else + # Migrate existing data if APP_DIR is a regular dir with content + if [ -d "$APP_DIR" ] && ! mountpoint -q "$APP_DIR" && [ -n "$(ls -A "$APP_DIR" 2>/dev/null)" ]; then + echo "Migrating existing data from $APP_DIR to $VOL_SUBDIR" + WAS_RUNNING=0 + if systemctl is-active --quiet lyrics-api; then + WAS_RUNNING=1 + systemctl stop lyrics-api + # Arm a restart trap before any operation that could fail mid-migration + # (rsync, mv, mount, etc.). The trap fires on every script exit path, + # so the service always comes back up, even if a later step blows up. + # Idempotent: systemctl start is a no-op if the service is already + # running by the time the trap fires on a successful run. + trap 'systemctl start lyrics-api' EXIT + fi + rsync -aHAX --info=progress2 "${APP_DIR}/" "${VOL_SUBDIR}/" + BACKUP_NAME="${APP_DIR}.pre-volume.$(date -u +%Y%m%d-%H%M%S)" + mv "$APP_DIR" "$BACKUP_NAME" + echo "Original preserved at $BACKUP_NAME (delete once you've verified the migration)" + fi + + install -d -o deploy -g deploy -m 755 "$APP_DIR" + + if ! grep -qE "^\S+\s+${APP_DIR}\s+none\s+bind" /etc/fstab; then + echo "$FSTAB_LINE" >> /etc/fstab + fi + + mount "$APP_DIR" +fi + +# Refuse to start the service if the bind mount is missing, so a failed mount +# can never silently let lyrics-api write a fresh empty cache.db on root. +install -d -m 755 /etc/systemd/system/lyrics-api.service.d +cat > "$DROPIN" <