feat: add dkg result injection script

scripts/external-keypers-dkg-injection-guide.md

@@ -0,0 +1,59 @@
+# External Keypers: How to use DKG injection script
+
+This guide describes the process for how external keypers can use DKG injection script in the **shutter-api-1002** deployment.
+
+## Purpose
+
+To restore key material generated during previous deployment, necessary to fulfill pending decryption tasks.
+
+---
+
+**Initial Keypers**: Keypers who were active during **eon 11**. Timestamp range: Mar-24-2025 01:03:45 PM UTC (1742821425) - Dec-01-2025 11:25:35 AM UTC (1764588335).
+
+---
+
+## Prerequisites
+
+- Fully synced keyper running the shutter-api-1002 deployment version
+- The same signing keys used for initial keypers deployment
+- Backup from the initial keypers
+
+---
+
+## Process Steps
+
+### 1. Run Keypers with Same Signing Keys
+
+In the **shutter-api-1002** deployment, run the keypers with the **same signing keys** that were used previously for the initial keypers deployment and wait for them to sync with the network.
+
+Sync can be confirmed by this log line.
+```
+synced registry contract end-block=20044460 num-discarded-events=0 num-inserted-events=0 start-block=20044460
+```
+The **end-block** should be (or greater than) the current head of the chain in the explorer.
+
+### 2. Ensure the backup is copied to the same instance
+
+Copy the backup to the same instance where the keyper is running.
+
+### 3. Run DKG Injection Script
+
+After a keyperset transition is done, run the DKG injection script with the backup path:
+
+```bash
+curl -fsSL https://raw.githubusercontent.com/shutter-network/shutter-keyper-deployment/feat/dkg-result-injection/scripts/inject_dkg_result.sh | bash -s -- <path_to_backup>
+```
+
+Replace `<path_to_backup>` with the actual path to your backup.
+
+Check if there is no error in running the script.
+
+---
+
+## Summary Checklist
+
+| Step | Action |
+|------|--------|
+| 1 | Run keypers in shutter-api-1002 with same signing keys as initial keypers and wait for keypers to sync  |
+| 2 | Ensure the backup is copied to the same instance |
+| 3 | Run DKG injection script with backup path |

scripts/inject_dkg_result.sh

@@ -0,0 +1,291 @@
+#!/usr/bin/env bash
+
+# This script overrides a selected DKG result in the keyper database
+# with the corresponding data from a backup. The following tables are
+# affected:
+# - dkg_result (columns: success, error, pure_result)
+# - keyper_set (columns: keypers, threshold)
+# - tendermint_batch_config (columns: keypers, threshold)
+#
+# The existing tables are backed up in the same database (with suffix
+# "_backup") before applying the changes in case they need to be
+# restored.
+#
+# The rows to update are identified by EON and KEYPER_CONFIG_INDEX
+# variables defined below.
+#
+# Usage: ./inject_dkg_result.sh <path-to-backup.tar|path-to-backup.tar.xz>
+#
+# Ensure the node is sufficiently synced before running. If the keyper
+# service is running, it will be stopped during the operation and
+# restarted afterwards. The database service will be started if not
+# already running, and stopped again afterwards if it was not running.
+
+set -euo pipefail
+
+EON="11"
+KEYPER_CONFIG_INDEX="11"
+MIN_TENDERMINT_CURRENT_BLOCK="0"
+
+BACKUP_CONTAINER="backup-db"
+BACKUP_IMAGE="postgres"
+BACKUP_DB="postgres"
+BACKUP_USER="postgres"
+BACKUP_PASSWORD="postgres"
+KEYPER_DB="keyper"
+BACKUP_TABLE_SUFFIX="_backup"
+
+TMP_DIR="$(mktemp -d 2>/dev/null || mktemp -d -t inject-dkg-result)"
+DUMP_FILE="${TMP_DIR}/keyper.dump"
+TABLES=(
+  "dkg_result:eon:${EON}:success, error, pure_result"
+  "tendermint_batch_config:keyper_config_index:${KEYPER_CONFIG_INDEX}:keypers, threshold"
+  "keyper_set:keyper_config_index:${KEYPER_CONFIG_INDEX}:keypers, threshold"
+)
+
+log() {
+  echo "==> $1"
+}
+
+usage() {
+  echo "Usage: $(basename "$0") <path-to-backup.tar|path-to-backup.tar.xz>" >&2
+  exit 1
+}
+
+if [[ "$#" -ne 1 ]]; then
+  usage
+fi
+
+if ! command -v tar >/dev/null 2>&1; then
+  echo "ERROR: required command 'tar' not found in PATH" >&2
+  exit 1
+fi
+
+BACKUP_TARBALL_PATH="$1"
+
+if [[ ! -f "$BACKUP_TARBALL_PATH" ]]; then
+  echo "ERROR: tarball not found: $BACKUP_TARBALL_PATH" >&2
+  exit 1
+fi
+
+if docker ps -a --format '{{.Names}}' | grep -q "^${BACKUP_CONTAINER}\$"; then
+  echo "ERROR: container '${BACKUP_CONTAINER}' already exists. Aborting." >&2
+  exit 1
+fi
+
+DB_WAS_RUNNING=0
+KEYPER_WAS_RUNNING=0
+
+if [[ -n "$(docker compose ps --status=running -q db 2>/dev/null || true)" ]]; then
+  DB_WAS_RUNNING=1
+fi
+
+if [[ -n "$(docker compose ps --status=running -q keyper 2>/dev/null || true)" ]]; then
+  KEYPER_WAS_RUNNING=1
+fi
+
+cleanup() {
+  rv=$?
+  if [[ "$rv" -ne 0 ]]; then
+    echo "Aborting due to error (exit code $rv)" >&2
+  fi
+
+  log "Stopping backup container"
+  docker stop "$BACKUP_CONTAINER" >/dev/null 2>&1 || true
+
+  if [[ "$KEYPER_WAS_RUNNING" -eq 1 ]]; then
+    log "Restarting keyper service (was running before)"
+    docker compose start keyper >/dev/null 2>&1 || true
+  else
+    log "Leaving keyper service stopped (was not running before)"
+  fi
+
+  if [[ "$DB_WAS_RUNNING" -eq 0 ]]; then
+    log "Stopping db service (was not running before)"
+    docker compose stop db >/dev/null 2>&1 || true
+  else
+    log "Keeping db service running (was running before)"
+  fi
+
+  if [[ -d "$TMP_DIR" ]]; then
+    log "Removing temporary directory ${TMP_DIR}"
+    rm -rf "$TMP_DIR"
+  fi
+
+  exit "$rv"
+}
+trap cleanup EXIT
+
+if [[ "$DB_WAS_RUNNING" -eq 0 ]]; then
+  log "Starting db service (was not running)"
+  docker compose start db >/dev/null
+fi
+
+log "Checking shuttermint sync block number >= ${MIN_TENDERMINT_CURRENT_BLOCK}"
+CURRENT_BLOCK=$(docker compose exec -T db sh -lc \
+  "psql -t -A -U postgres -d ${KEYPER_DB} -c \"SELECT current_block FROM tendermint_sync_meta ORDER BY current_block DESC LIMIT 1\"" \
+  2>/dev/null | tr -d '[:space:]')
+
+if [[ -z "$CURRENT_BLOCK" ]]; then
+  echo "ERROR: failed to read shuttermint sync block number" >&2
+  exit 1
+fi
+
+if ! [[ "$CURRENT_BLOCK" =~ ^[0-9]+$ ]]; then
+  echo "ERROR: shuttermint sync block number is not an integer: $CURRENT_BLOCK" >&2
+  exit 1
+fi
+
+if (( CURRENT_BLOCK < MIN_TENDERMINT_CURRENT_BLOCK )); then
+  echo "ERROR: shuttermint sync block number ($CURRENT_BLOCK) is below MIN_TENDERMINT_CURRENT_BLOCK ($MIN_TENDERMINT_CURRENT_BLOCK); aborting. Please wait until the node is sufficiently synced and try again." >&2
+  exit 1
+fi
+
+log "Stopping keyper service"
+docker compose stop keyper >/dev/null 2>&1 || true
+
+log "Extracting keyper DB from backup"
+TAR_WARNING_FLAGS=()
+if tar --help 2>/dev/null | grep -q -- '--warning'; then
+  TAR_WARNING_FLAGS+=(--warning=no-unknown-keyword)
+fi
+
+TAR_COMPRESS_FLAGS=()
+if [[ "$BACKUP_TARBALL_PATH" == *.tar.xz ]]; then
+  TAR_COMPRESS_FLAGS=(-J)
+fi
+
+TAR_LIST_OUTPUT=""
+if ! TAR_LIST_OUTPUT=$(tar "${TAR_WARNING_FLAGS[@]}" "${TAR_COMPRESS_FLAGS[@]}" -tf "$BACKUP_TARBALL_PATH" 2>/dev/null); then
+  if [[ "${#TAR_COMPRESS_FLAGS[@]}" -eq 0 ]]; then
+    TAR_COMPRESS_FLAGS=(-J)
+    TAR_LIST_OUTPUT=$(tar "${TAR_WARNING_FLAGS[@]}" "${TAR_COMPRESS_FLAGS[@]}" -tf "$BACKUP_TARBALL_PATH" 2>/dev/null) || true
+  fi
+fi
+
+DUMP_TAR_MEMBER=""
+while IFS= read -r entry; do
+  [[ -z "$entry" ]] && continue
+  normalized_entry="${entry#./}"
+  if [[ "$normalized_entry" == "keyper.dump" || "$normalized_entry" == */keyper.dump ]]; then
+    DUMP_TAR_MEMBER="$entry"
+    break
+  fi
+done <<< "$TAR_LIST_OUTPUT"
+
+if [[ -z "$DUMP_TAR_MEMBER" ]]; then
+  echo "ERROR: could not find keyper.dump inside ${BACKUP_TARBALL_PATH}" >&2
+  exit 1
+fi
+
+tar "${TAR_WARNING_FLAGS[@]}" "${TAR_COMPRESS_FLAGS[@]}" -xOf "$BACKUP_TARBALL_PATH" "$DUMP_TAR_MEMBER" >"$DUMP_FILE"
+
+if [[ ! -s "$DUMP_FILE" ]]; then
+  echo "ERROR: failed to extract ${DUMP_TAR_MEMBER} from ${BACKUP_TARBALL_PATH}" >&2
+  exit 1
+fi
+
+log "Starting backup container"
+docker run -d --rm \
+  --name "$BACKUP_CONTAINER" \
+  -e POSTGRES_USER="$BACKUP_USER" \
+  -e POSTGRES_PASSWORD="$BACKUP_PASSWORD" \
+  -e POSTGRES_DB="$BACKUP_DB" \
+  -v "$DUMP_FILE:/backup/dump.sql:ro" \
+  "$BACKUP_IMAGE" >/dev/null
+
+log "Waiting for backup DB to become ready"
+for i in {1..30}; do
+  if docker exec "$BACKUP_CONTAINER" pg_isready -U "$BACKUP_USER" -d "$BACKUP_DB" >/dev/null 2>&1; then
+    break
+  fi
+  sleep 1
+done
+if ! docker exec "$BACKUP_CONTAINER" pg_isready -U "$BACKUP_USER" -d "$BACKUP_DB" >/dev/null 2>&1; then
+  echo "ERROR: backup DB did not become ready after 30 seconds" >&2
+  exit 1
+fi
+
+log "Restoring dump into backup DB"
+docker exec "$BACKUP_CONTAINER" bash -lc \
+  "pg_restore -v -C -U '$BACKUP_USER' -d '$BACKUP_DB' /backup/dump.sql" >/dev/null 2>&1
+
+for entry in "${TABLES[@]}"; do
+  IFS=: read -r TABLE KEY_COLUMN KEY_VALUE SELECT_COLUMNS <<<"$entry"
+  BACKUP_CSV_FILE="${TMP_DIR}/${TABLE}_backup_${KEY_COLUMN}_${KEY_VALUE}.csv"
+  LIVE_CSV_FILE="${TMP_DIR}/${TABLE}_live_${KEY_COLUMN}_${KEY_VALUE}.csv"
+  SELECT_COLUMN_LIST=()
+
+  for col in ${SELECT_COLUMNS//,/ }; do
+    [[ -z "$col" ]] && continue
+    if [[ "$col" == "$KEY_COLUMN" ]]; then
+      echo "ERROR: column list for ${TABLE} must not include key column ${KEY_COLUMN}" >&2
+      exit 1
+    fi
+    SELECT_COLUMN_LIST+=("$col")
+  done
+
+  if [[ "${#SELECT_COLUMN_LIST[@]}" -eq 0 ]]; then
+    echo "ERROR: no non-key columns specified for update in ${TABLE}" >&2
+    exit 1
+  fi
+
+  SELECT_COLUMN_LIST_WITH_KEY=("$KEY_COLUMN" "${SELECT_COLUMN_LIST[@]}")
+  SELECT_COLUMNS_WITH_KEY=$(IFS=', '; echo "${SELECT_COLUMN_LIST_WITH_KEY[*]}")
+
+  log "Extracting ${TABLE} row ${KEY_COLUMN}=${KEY_VALUE} from backup DB"
+  docker exec "$BACKUP_CONTAINER" bash -lc \
+    "psql -v ON_ERROR_STOP=1 -U '$BACKUP_USER' -d '$KEYPER_DB' -c \"COPY (SELECT ${SELECT_COLUMNS_WITH_KEY} FROM ${TABLE} WHERE ${KEY_COLUMN} = '${KEY_VALUE}' LIMIT 1) TO STDOUT WITH CSV\"" \
+    >"$BACKUP_CSV_FILE" 2>/dev/null
+
+  if [[ ! -s "$BACKUP_CSV_FILE" ]]; then
+    echo "ERROR: no data extracted from backup DB (no row with ${KEY_COLUMN}=${KEY_VALUE} in ${TABLE})" >&2
+    exit 1
+  fi
+
+  log "Extracting ${TABLE} row ${KEY_COLUMN}=${KEY_VALUE} from live DB"
+  docker compose exec -T db sh -lc \
+    "psql -v ON_ERROR_STOP=1 -U postgres -d ${KEYPER_DB} -c \"COPY (SELECT ${SELECT_COLUMNS_WITH_KEY} FROM ${TABLE} WHERE ${KEY_COLUMN} = '${KEY_VALUE}' LIMIT 1) TO STDOUT WITH CSV\"" \
+    >"$LIVE_CSV_FILE" 2>/dev/null || true
+
+  if [[ ! -s "$LIVE_CSV_FILE" ]]; then
+    echo "ERROR: no data extracted from live DB (no row with ${KEY_COLUMN}=${KEY_VALUE} in ${TABLE})" >&2
+    exit 1
+  fi
+
+  if [[ -s "$LIVE_CSV_FILE" && -s "$BACKUP_CSV_FILE" && "$(cat "$LIVE_CSV_FILE")" == "$(cat "$BACKUP_CSV_FILE")" ]]; then
+    log "Live row for ${TABLE} already matches backup, nothing to do"
+    continue
+  fi
+
+  BACKUP_TABLE_NAME="${TABLE}${BACKUP_TABLE_SUFFIX}"
+
+  log "Backing up table ${TABLE} to ${BACKUP_TABLE_NAME} in live DB"
+  {
+    echo "CREATE TABLE IF NOT EXISTS ${BACKUP_TABLE_NAME} (LIKE ${TABLE} INCLUDING ALL);"
+    echo "TRUNCATE ${BACKUP_TABLE_NAME};"
+    echo "INSERT INTO ${BACKUP_TABLE_NAME} SELECT * FROM ${TABLE};"
+  } | docker compose exec -T db psql -U postgres -d "${KEYPER_DB}" >/dev/null 2>&1
+
+  UPDATE_SET=""
+  for col in "${SELECT_COLUMN_LIST[@]}"; do
+    if [[ -z "$UPDATE_SET" ]]; then
+      UPDATE_SET="${col} = u.${col}"
+    else
+      UPDATE_SET="${UPDATE_SET}, ${col} = u.${col}"
+    fi
+  done
+
+  log "Restoring ${TABLE} row ${KEY_COLUMN}=${KEY_VALUE}"
+  {
+    echo "BEGIN;"
+    echo "CREATE TEMP TABLE tmp_update AS SELECT ${SELECT_COLUMNS_WITH_KEY} FROM ${TABLE} WHERE 1=0;"
+    echo "COPY tmp_update FROM STDIN WITH CSV;"
+    cat "$BACKUP_CSV_FILE"
+    echo '\.'
+    echo "UPDATE ${TABLE} AS t SET ${UPDATE_SET} FROM tmp_update u WHERE t.${KEY_COLUMN} = u.${KEY_COLUMN};"
+    echo "COMMIT;"
+  } | docker compose exec -T db psql -U postgres -d "${KEYPER_DB}" >/dev/null 2>&1
+done
+
+log "Done"