Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 23 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -342,6 +342,29 @@ Operational visibility is a first-class concern:
- ticket, certificate, and reload lifecycle management
- autoscaling and cluster integration hooks

### Production Debug Process

For live KingRT video-chat investigation, use the read-only production debug
process:

```bash
demo/video-chat/scripts/prod-debug.sh
```

It reads `demo/video-chat/.env.local` for the existing deploy host and domain
settings, or `VIDEOCHAT_PROD_DEBUG_ENV_FILE` when debugging from a separate
worktree. It then reports public runtime health, configured domains,
asset/version reachability, API/WS/SFU probes, TURN domain configuration, Call
App/Mothernode reachability, read-only Call App marketplace counts, compose
status, and recent remote container logs.
Output is passed through secret redaction for token-, password-, auth-, cookie-,
session-, key-, and secret-like values.

This process is read-only by contract: it does not deploy, restart, write
database data, change DNS, or use admin actions. Set
`VIDEOCHAT_PROD_DEBUG_SKIP_SSH=1` to run only public network probes when SSH is
not needed.

## Public Programming Model

The core programming model is:
Expand Down
2 changes: 2 additions & 0 deletions demo/video-chat/frontend-vue/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@
"dev": "vite --config ./vite.config.js",
"dev:call-stabilization-harness": "vite --config ./vite.config.js --host 127.0.0.1 --port 8765",
"build": "vite build",
"prod:debug": "../scripts/prod-debug.sh",
"test:contract:prod-debug": "node tests/contract/prod-debug-process-contract.mjs",
"test:contract:wlvc": "node tests/contract/wlvc-wire-contract.mjs && node tests/contract/wlvc-binding-recovery-contract.mjs && node tests/contract/wlvc-codec-port-contract.mjs && node tests/contract/wlvc-runtime-regression-contract.mjs && node tests/contract/wlvc-hybrid-fallback-contract.mjs && node tests/contract/sfu-wlvc-abi-gating-contract.mjs",
"test:contract:sfu": "node tests/contract/more-payload-intake-contract.mjs && node tests/contract/sfu-origin-room-binding-contract.mjs && node tests/contract/sfu-motion-backpressure-contract.mjs && node tests/contract/sfu-video-recovery-timing-contract.mjs && node tests/contract/sfu-throughput-path-contract.mjs && node tests/contract/sfu-real-media-plane-architecture-contract.mjs && node tests/contract/sfu-control-data-plane-split-contract.mjs && node tests/contract/sfu-signalling-unit-policy-contract.mjs && node tests/contract/sfu-media-recovery-control-contract.mjs && node tests/contract/sfu-gossip-planning-cfh-contract.mjs && node tests/contract/sfu-publisher-path-trace-contract.mjs && node tests/contract/sfu-capture-pipeline-capabilities-contract.mjs && node tests/contract/sfu-capture-worker-boundary-contract.mjs && node tests/contract/sfu-video-frame-primary-path-contract.mjs && node tests/contract/sfu-video-frame-rgba-copy-contract.mjs && node tests/contract/sfu-protected-browser-encoder-contract.mjs && node tests/contract/sfu-zero-copy-readback-gate-contract.mjs && node tests/contract/sfu-offscreen-canvas-fallback-contract.mjs && node tests/contract/sfu-dom-canvas-last-resort-contract.mjs && node tests/contract/sfu-transport-metrics-contract.mjs && node tests/contract/sfu-end-to-end-observability-contract.mjs && node tests/contract/sfu-diagnostic-surface-contract.mjs && node tests/contract/sfu-profile-budget-contract.mjs && node tests/contract/sfu-source-budget-profile-coupling-contract.mjs && node tests/contract/sfu-capture-constraints-contract.mjs && node tests/contract/mobile-call-media-devices-contract.mjs && node tests/contract/sfu-source-readback-contract.mjs && node tests/contract/sfu-auto-readback-downgrade-contract.mjs && node tests/contract/sfu-auto-readback-recovery-contract.mjs && node tests/contract/sfu-wlvc-rate-control-contract.mjs && node tests/contract/sfu-high-motion-payload-contract.mjs && node tests/contract/sfu-high-motion-readback-budget-contract.mjs && node tests/contract/sfu-portrait-aspect-preservation-contract.mjs && node tests/contract/sfu-client-side-framing-crop-contract.mjs && node tests/contract/sfu-fullscreen-render-scheduler-contract.mjs && node tests/contract/sfu-receiver-jitter-buffer-contract.mjs && node tests/contract/sfu-adaptive-quality-layers-contract.mjs && node tests/contract/sfu-dual-video-layer-routing-contract.mjs && node tests/contract/sfu-background-tab-policy-contract.mjs && node tests/contract/sfu-keyframe-cache-pacing-contract.mjs && node tests/contract/sfu-security-throughput-budget-contract.mjs && node tests/contract/sfu-profile-switch-actuator-contract.mjs && node tests/contract/sfu-publisher-backpressure-controller-contract.mjs && node tests/contract/sfu-browser-ws-send-drain-contract.mjs && node tests/contract/sfu-binary-envelope-copy-audit-contract.mjs && node tests/contract/sfu-king-binary-decode-fanout-contract.mjs && node tests/contract/sfu-no-frame-persistence-regression-contract.mjs && node tests/contract/sfu-relay-broker-io-budget-contract.mjs && node tests/contract/sfu-king-receive-loop-fairness-contract.mjs && node tests/contract/sfu-slow-subscriber-isolation-contract.mjs && node tests/contract/sfu-replay-pacing-slow-subscriber-contract.mjs && node tests/contract/sfu-receiver-feedback-loop-contract.mjs && node tests/contract/sfu-production-socket-proxy-budget-contract.mjs && node tests/contract/sfu-online-acceptance-no-critical-pressure-contract.mjs",
"test:contract:gossip": "node tests/contract/gossip-controller-decentralized-routing-contract.mjs && node tests/contract/gossip-harness-faults-contract.mjs && node tests/contract/gossip-local-5-peer-network-harness-contract.mjs && node tests/contract/gossip-telemetry-contract.mjs && node tests/contract/gossip-rollout-gate-contract.mjs && node tests/contract/gossip-sfu-baseline-rollout-gate-contract.mjs && node tests/contract/gossip-primary-health-gate-contract.mjs && node tests/contract/gossip-native-recovery-contract.mjs && node tests/contract/gossip-server-no-media-fanout-contract.mjs && node tests/contract/gossip-topology-hint-contract.mjs && node tests/contract/gossip-room-state-topology-contract.mjs && ../backend-king-php/tests/realtime-gossipmesh-room-state-topology-contract.sh && node tests/contract/gossip-dedicated-neighbor-lifecycle-contract.mjs && node tests/contract/gossip-authoritative-topology-repair-contract.mjs && ../backend-king-php/tests/realtime-gossipmesh-runtime-contract.sh && node tests/contract/gossip-data-lane-feature-flag-contract.mjs && node tests/contract/gossip-media-carrier-mode-contract.mjs && node tests/contract/gossip-production-deploy-profile-contract.mjs && node tests/contract/gossip-publisher-pipeline-decoupling-contract.mjs && node tests/contract/gossip-primary-fallback-backtrace-contract.mjs && node tests/contract/gossip-media-carrier-integration-smoke-contract.mjs && node tests/contract/gossip-native-webrtc-binding-contract.mjs && node tests/contract/gossip-live-receive-decode-route-contract.mjs && node tests/contract/gossip-outbound-live-publication-contract.mjs && node tests/contract/gossip-server-topology-ingestion-contract.mjs && node tests/contract/gossip-stale-target-pruning-contract.mjs && node tests/contract/gossip-neighbor-health-repair-contract.mjs && node tests/contract/gossip-neighbor-health-topology-repair-contract.mjs && node tests/contract/gossip-native-binary-data-plane-contract.mjs && node tests/contract/kingrt-three-user-regression-harness-contract.mjs && node tests/contract/gossip-overview-map-analysis-contract.mjs && node tests/contract/gossip-docs-process-contract.mjs",
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,65 @@
import assert from 'node:assert/strict';
import { readFile } from 'node:fs/promises';
import path from 'node:path';

const root = path.resolve(new URL('../..', import.meta.url).pathname);
const repoRoot = path.resolve(root, '../../..');

async function read(relativePath) {
return readFile(path.join(repoRoot, relativePath), 'utf8');
}

const [script, packageJsonRaw, readme] = await Promise.all([
read('demo/video-chat/scripts/prod-debug.sh'),
read('demo/video-chat/frontend-vue/package.json'),
read('README.md'),
]);

const packageJson = JSON.parse(packageJsonRaw);

assert.match(script, /^#!/, 'prod-debug.sh must be directly executable as a documented process');
assert.match(script, /VIDEOCHAT_PROD_DEBUG_ENV_FILE/, 'prod-debug must support an explicit env-file override for worktree debugging');
assert.match(script, /VIDEOCHAT_PROD_DEBUG_SKIP_SSH/, 'prod-debug must allow SSH diagnostics to be skipped');
assert.match(script, /redact_stream[\s\S]*(TOKEN|SECRET|PASSWORD|AUTH|COOKIE|SESSION)/, 'prod-debug must redact secret-like output');
assert.match(script, /curl_code[\s\S]*\/health[\s\S]*\/api\/version/s, 'prod-debug must inspect public API health and version');
assert.match(script, /websocket_handshake_probe[\s\S]*DEPLOY_WS_DOMAIN[\s\S]*DEPLOY_SFU_DOMAIN/s, 'prod-debug must inspect WS and SFU reachability');
assert.match(script, /call_app[\s\S]*MOTHERNODE/s, 'prod-debug must include Call App and Mothernode domain visibility');
assert.match(script, /SQLITE3_OPEN_READONLY/, 'marketplace and Call App database checks must use a read-only SQLite open');
assert.match(script, /docker compose[\s\S]* ps[\s\S]*logs --no-color --tail/s, 'remote container diagnostics must be limited to status and recent logs');

const activeScript = script
.split('\n')
.filter((line) => {
const trimmed = line.trim();
return trimmed !== '' && !trimmed.startsWith('#') && !trimmed.startsWith('echo ') && !trimmed.startsWith('cat <<');
})
.join('\n');

for (const forbidden of [
/\bcurl\b[^\n]*(?:-X|--request)\s*(?:POST|PUT|PATCH|DELETE)\b/i,
/\bdocker\s+compose\b[^\n]*\b(?:up|down|start|stop|restart|kill|pull|build|rm|run)\b/i,
/\bdocker\b[^\n]*\b(?:run|start|stop|restart|kill|rm|rmi|pull|build|push)\b/i,
/\b(?:rsync|scp|certbot|hcloud|ssh-keygen)\b/i,
/\b(?:INSERT|UPDATE|DELETE|CREATE|DROP|ALTER|REPLACE|VACUUM|PRAGMA)\b/i,
]) {
assert.doesNotMatch(activeScript, forbidden, `prod-debug.sh must remain read-only: ${forbidden}`);
}

assert.equal(
packageJson.scripts?.['prod:debug'],
'../scripts/prod-debug.sh',
'package.json must expose the production debug command',
);
assert.equal(
packageJson.scripts?.['test:contract:prod-debug'],
'node tests/contract/prod-debug-process-contract.mjs',
'package.json must expose the prod-debug contract',
);

assert.match(
readme,
/Production Debug Process[\s\S]*demo\/video-chat\/scripts\/prod-debug\.sh[\s\S]*read-only[\s\S]*does not deploy,\s+restart,\s+write\s+database\s+data,\s+change\s+DNS,\s+or\s+use\s+admin\s+actions/i,
'README.md must document the read-only production debug process and safety boundary',
);

console.log('[prod-debug-process-contract] PASS');
243 changes: 243 additions & 0 deletions demo/video-chat/scripts/prod-debug.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,243 @@
#!/usr/bin/env bash
set -euo pipefail

SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
VIDEOCHAT_DIR="$(cd -- "${SCRIPT_DIR}/.." && pwd)"
LOCAL_ENV_FILE="${VIDEOCHAT_PROD_DEBUG_ENV_FILE:-${VIDEOCHAT_DIR}/.env.local}"
TIMEOUT="${VIDEOCHAT_PROD_DEBUG_TIMEOUT:-12}"
LOG_TAIL="${VIDEOCHAT_PROD_DEBUG_LOG_TAIL:-120}"

log() {
printf '[videochat-prod-debug] %s\n' "$*"
}

warn() {
printf '[videochat-prod-debug] WARN: %s\n' "$*" >&2
}

fail() {
printf '[videochat-prod-debug] ERROR: %s\n' "$*" >&2
exit 1
}

usage() {
cat <<'USAGE'
Usage:
demo/video-chat/scripts/prod-debug.sh

Optional environment:
VIDEOCHAT_PROD_DEBUG_TIMEOUT Public probe timeout in seconds, default: 12.
VIDEOCHAT_PROD_DEBUG_LOG_TAIL Recent remote log lines per service, default: 120.
VIDEOCHAT_PROD_DEBUG_SKIP_SSH Skip remote SSH diagnostics, default: 0.
VIDEOCHAT_PROD_DEBUG_ENV_FILE Env file to read, default: demo/video-chat/.env.local.

The command reads the configured env file for deploy host/domain settings.
USAGE
}

require_cmd() {
command -v "$1" >/dev/null 2>&1 || fail "missing required command: $1"
}

shell_quote() {
printf '%q' "$1"
}

load_local_env() {
[[ -f "${LOCAL_ENV_FILE}" ]] || return 0
set -a
# shellcheck source=/dev/null
source "${LOCAL_ENV_FILE}"
set +a
}

redact_stream() {
sed -E \
-e 's/([A-Za-z0-9_]*(TOKEN|SECRET|PASSWORD|PASS|KEY|AUTH|COOKIE|SESSION)[A-Za-z0-9_]*=)[^[:space:]]+/\1[REDACTED]/Ig' \
-e 's/(Authorization:[[:space:]]*Bearer[[:space:]]+)[A-Za-z0-9._~+\/=-]+/\1[REDACTED]/Ig' \
-e 's/((token|secret|password|pass|key|auth|cookie|session)["'\'']?[[:space:]]*[:=][[:space:]]*["'\'']?)[^"'\''[:space:],}]+/\1[REDACTED]/Ig'
}

curl_code() {
local label="$1" url="$2" output code
output="$(mktemp)"
code="$(curl -sS --max-time "${TIMEOUT}" -o "${output}" -w '%{http_code}' "${url}" || true)"
printf '[videochat-prod-debug] %-32s HTTP %s %s\n' "${label}" "${code:-000}" "${url}"
if [[ -s "${output}" && "${label}" == "api health" ]]; then
redact_stream <"${output}" | head -c 1200
printf '\n'
fi
rm -f "${output}"
}

curl_head_code() {
local label="$1" url="$2" code
code="$(curl -sS -I --max-time "${TIMEOUT}" -o /dev/null -w '%{http_code}' "${url}" || true)"
printf '[videochat-prod-debug] %-32s HTTP %s %s\n' "${label}" "${code:-000}" "${url}"
}

websocket_handshake_probe() {
local label="$1" url="$2" headers body code header_code
headers="$(mktemp)"
body="$(mktemp)"
code="$(
curl -sS --http1.1 --max-time "${TIMEOUT}" \
-D "${headers}" \
-o "${body}" \
-w '%{http_code}' \
-H 'Connection: Upgrade' \
-H 'Upgrade: websocket' \
-H 'Sec-WebSocket-Key: dGhlIHNhbXBsZSBub25jZQ==' \
-H 'Sec-WebSocket-Version: 13' \
"${url}" || true
)"
if [[ "${code}" == "000" ]]; then
header_code="$(awk '/^HTTP\// {code=$2} END {print code}' "${headers}" | tr -d '\r')"
[[ -n "${header_code}" ]] && code="${header_code}"
fi
printf '[videochat-prod-debug] %-32s HTTP %s %s\n' "${label}" "${code:-000}" "${url}"
if [[ -s "${headers}" ]]; then
awk 'BEGIN{IGNORECASE=1} /^(HTTP\/|upgrade:|connection:|sec-websocket-accept:)/ {print}' "${headers}" | redact_stream
fi
rm -f "${headers}" "${body}"
}

remote_read_only_diagnostics() {
local deploy_path_q log_tail_q locale_q
deploy_path_q="$(shell_quote "${DEPLOY_PATH}")"
log_tail_q="$(shell_quote "${LOG_TAIL}")"
locale_q="$(shell_quote "${DEPLOY_REMOTE_LOCALE}")"
ssh "${SSH_ARGS[@]}" "${SSH_DEST}" "LC_ALL=${locale_q} LANG=${locale_q} LANGUAGE= DEPLOY_PATH=${deploy_path_q} LOG_TAIL=${log_tail_q} bash -s" <<'REMOTE' | redact_stream
set -euo pipefail
VIDEOCHAT_DIR="${DEPLOY_PATH}/demo/video-chat"

echo "== remote identity =="
hostname || true
date -u '+%Y-%m-%dT%H:%M:%SZ' || true

if [ ! -d "${VIDEOCHAT_DIR}" ]; then
echo "remote video-chat directory missing: ${VIDEOCHAT_DIR}"
exit 0
fi

cd "${VIDEOCHAT_DIR}"
if [ -f docker-compose.deploy.local.yml ]; then
COMPOSE=(docker compose --env-file .env --env-file .env.local -f docker-compose.v1.yml -f docker-compose.deploy.local.yml --profile edge --profile turn)
elif [ -f docker-compose.v1.yml ]; then
COMPOSE=(docker compose --env-file .env -f docker-compose.v1.yml --profile edge --profile turn)
else
echo "compose file missing"
exit 0
fi

echo "== compose ps =="
"${COMPOSE[@]}" ps || true

echo "== compose service names =="
"${COMPOSE[@]}" config --services || true

echo "== call app marketplace read-only db summary =="
"${COMPOSE[@]}" exec -T videochat-backend-v1 php -r '
function emit_count(SQLite3 $db, string $label, string $sql): void {
$result = @$db->querySingle($sql);
if ($result === false || $result === null) {
echo $label . ": unavailable\n";
return;
}
echo $label . ": " . $result . "\n";
}
if (!class_exists("SQLite3")) {
echo "sqlite3 extension unavailable\n";
exit(0);
}
$path = getenv("VIDEOCHAT_KING_DB_PATH") ?: "/data/video-chat.sqlite";
try {
$db = new SQLite3($path, SQLITE3_OPEN_READONLY);
} catch (Throwable $exception) {
echo "database unavailable for readonly open\n";
exit(0);
}
emit_count($db, "catalog_entries", "SELECT count(*) FROM call_app_catalog_entries");
emit_count($db, "healthy_catalog_entries", "SELECT count(*) FROM call_app_catalog_entries WHERE health_status = '\''healthy'\''");
emit_count($db, "enabled_installations", "SELECT count(*) FROM organization_call_app_installations WHERE status = '\''enabled'\''");
emit_count($db, "active_entitlements", "SELECT count(*) FROM organization_call_app_entitlements WHERE status = '\''active'\''");
' || true

for service in videochat-edge-v1 videochat-backend-v1 videochat-backend-ws-v1 videochat-backend-sfu-v1 videochat-frontend-v1 videochat-turn-v1; do
if "${COMPOSE[@]}" ps -q "${service}" 2>/dev/null | grep -q .; then
echo "== recent logs: ${service} =="
"${COMPOSE[@]}" logs --no-color --tail "${LOG_TAIL}" "${service}" || true
fi
done
REMOTE
}

case "${1:-}" in
help|-h|--help)
usage
exit 0
;;
"")
;;
*)
usage >&2
fail "unknown argument: $1"
;;
esac

load_local_env
require_cmd curl

DEPLOY_DOMAIN="${VIDEOCHAT_DEPLOY_DOMAIN:-${VIDEOCHAT_V1_PUBLIC_HOST:-}}"
[[ -n "${DEPLOY_DOMAIN}" ]] || fail "VIDEOCHAT_DEPLOY_DOMAIN or VIDEOCHAT_V1_PUBLIC_HOST is required"
DEPLOY_HOST="${VIDEOCHAT_DEPLOY_HOST:-}"
DEPLOY_USER="${VIDEOCHAT_DEPLOY_USER:-root}"
DEPLOY_SSH_PORT="${VIDEOCHAT_DEPLOY_SSH_PORT:-22}"
DEPLOY_PATH="${VIDEOCHAT_DEPLOY_PATH:-/opt/king-videochat}"
DEPLOY_REMOTE_LOCALE="${VIDEOCHAT_DEPLOY_REMOTE_LOCALE:-C.UTF-8}"
DEPLOY_API_DOMAIN="${VIDEOCHAT_DEPLOY_API_DOMAIN:-api.${DEPLOY_DOMAIN}}"
DEPLOY_WS_DOMAIN="${VIDEOCHAT_DEPLOY_WS_DOMAIN:-ws.${DEPLOY_DOMAIN}}"
DEPLOY_SFU_DOMAIN="${VIDEOCHAT_DEPLOY_SFU_DOMAIN:-sfu.${DEPLOY_DOMAIN}}"
DEPLOY_TURN_DOMAIN="${VIDEOCHAT_DEPLOY_TURN_DOMAIN:-turn.${DEPLOY_DOMAIN}}"
DEPLOY_CDN_DOMAIN="${VIDEOCHAT_DEPLOY_CDN_DOMAIN:-cdn.${DEPLOY_DOMAIN}}"
DEPLOY_CALL_APP_DOMAIN="${VIDEOCHAT_DEPLOY_CALL_APP_DOMAIN:-apps.${DEPLOY_DOMAIN}}"
DEPLOY_MOTHERNODE_DOMAIN="${VIDEOCHAT_DEPLOY_MOTHERNODE_DOMAIN:-mother.${DEPLOY_DOMAIN}}"

log "read-only production debug for ${DEPLOY_DOMAIN}"
log "domains: api=${DEPLOY_API_DOMAIN} ws=${DEPLOY_WS_DOMAIN} sfu=${DEPLOY_SFU_DOMAIN} turn=${DEPLOY_TURN_DOMAIN} cdn=${DEPLOY_CDN_DOMAIN} call_app=${DEPLOY_CALL_APP_DOMAIN} mothernode=${DEPLOY_MOTHERNODE_DOMAIN}"

log "public HTTP health and asset probes"
curl_head_code "frontend" "https://${DEPLOY_DOMAIN}/"
curl_code "api health" "https://${DEPLOY_API_DOMAIN}/health"
curl_code "api version" "https://${DEPLOY_API_DOMAIN}/api/version"
curl_head_code "cdn root" "https://${DEPLOY_CDN_DOMAIN}/"
curl_head_code "call app whiteboard" "https://${DEPLOY_CALL_APP_DOMAIN}/call-app/whiteboard/public/index.html"
curl_head_code "mothernode host" "https://${DEPLOY_MOTHERNODE_DOMAIN}/"

log "public websocket reachability probes"
websocket_handshake_probe "lobby websocket host" "https://${DEPLOY_WS_DOMAIN}/ws?room=prod-debug"
websocket_handshake_probe "lobby websocket api fallback" "https://${DEPLOY_API_DOMAIN}/ws?room=prod-debug"
websocket_handshake_probe "sfu websocket host" "https://${DEPLOY_SFU_DOMAIN}/sfu?room_id=prod-debug"
websocket_handshake_probe "sfu websocket api fallback" "https://${DEPLOY_API_DOMAIN}/sfu?room_id=prod-debug"

case "${VIDEOCHAT_PROD_DEBUG_SKIP_SSH:-0}" in
1|true|TRUE|yes|YES)
log "remote SSH diagnostics skipped"
exit 0
;;
esac

if [[ -z "${DEPLOY_HOST}" ]]; then
warn "VIDEOCHAT_DEPLOY_HOST is not set; skipping remote compose/log diagnostics"
exit 0
fi

require_cmd ssh
SSH_DEST="${DEPLOY_USER}@${DEPLOY_HOST}"
SSH_ARGS=(-p "${DEPLOY_SSH_PORT}" -o BatchMode=yes -o StrictHostKeyChecking=accept-new)
if [[ -n "${VIDEOCHAT_DEPLOY_SSH_KEY:-}" ]]; then
SSH_ARGS+=(-i "${VIDEOCHAT_DEPLOY_SSH_KEY}")
fi

log "remote read-only diagnostics from ${SSH_DEST}"
remote_read_only_diagnostics
Loading