feat: improve admin queue diagnostics and recovery ui

src/components/docs/versions/build-status-dashboard.tsx

@@ -6,13 +6,31 @@
   selectedRepoVersion: string | undefined;
 }
 
+const minutesSinceTimestamp = (
+  timestamp?: {
+    seconds?: number;
+  } | null,
+): number | null => {
+  if (!timestamp?.seconds) return null;
+  const diffMs = Date.now() - timestamp.seconds * 1000;
+  return diffMs < 0 ? 0 : Math.floor(diffMs / 60000);
+};
+
+const formatAgeMinutes = (minutes: number | null): string => {
+  if (minutes === null) return 'n/a';
+  if (minutes < 60) return `${minutes}m`;
+  const hours = Math.floor(minutes / 60);
+  const remainingMinutes = minutes % 60;
+  return remainingMinutes === 0 ? `${hours}h` : `${hours}h ${remainingMinutes}m`;
+};
+
 const BuildStatusDashboard = ({ selectedRepoVersion }: Props) => {
   const firestore = useFirestore();
   const ciBuilds = firestore
     .collection('ciBuilds')
     .where('buildInfo.repoVersion', '==', selectedRepoVersion || '__none__');
 
   const { status, data } = useFirestoreCollectionData<{ [key: string]: any }>(ciBuilds);
 
   if (!selectedRepoVersion) return null;
 
@@ -31,6 +49,12 @@
   const maxedOut = builds.filter(
     (b) => b.status === 'failed' && (b.meta?.failureCount || 0) >= 15,
   ).length;
+  const startedAges = builds
+    .filter((b) => b.status === 'started')
+    .map((b) => minutesSinceTimestamp(b.meta?.lastBuildStart))
+    .filter((minutes): minutes is number => minutes !== null);
+  const staleStarted = startedAges.filter((minutes) => minutes >= 45).length;
+  const oldestStarted = startedAges.length > 0 ? Math.max(...startedAges) : null;
   const total = builds.length;
 
   const statStyle = (color: string): React.CSSProperties => ({
@@ -74,6 +98,14 @@
             Stuck (15+): <strong>{maxedOut}</strong>
           </span>
         )}
+        {staleStarted > 0 && (
+          <span style={statStyle('#92400e')}>
+            Started 45m+: <strong>{staleStarted}</strong>
+          </span>
+        )}
+        <span style={statStyle('#475569')}>
+          Oldest started: <strong>{formatAgeMinutes(oldestStarted)}</strong>
+        </span>
         <span
           style={{
             marginLeft: 'auto',

src/components/docs/versions/image-job-admin-actions.tsx

@@ -0,0 +1,153 @@
+import React, { useState } from 'react';
+import { HiOutlineRefresh } from 'react-icons/hi';
+import { MdRestartAlt } from 'react-icons/md';
+import { useFirestore, useFirestoreCollectionData, useUser } from 'reactfire';
+import { SimpleAuthCheck } from '@site/src/components/auth/safe-auth-check';
+import config from '@site/src/core/config';
+import { useNotification } from '@site/src/core/hooks/use-notification';
+import Spinner from '@site/src/components/molecules/spinner';
+import Tooltip from '@site/src/components/molecules/tooltip/tooltip';
+
+interface Props {
+  ciJobId: string;
+  status: string;
+}
+
+type BuildRecord = {
+  buildId: string;
+  relatedJobId: string;
+  status: string;
+  meta?: {
+    failureCount?: number;
+  };
+};
+
+const buttonStyle: React.CSSProperties = {
+  padding: 0,
+  border: 0,
+  outline: 0,
+  cursor: 'pointer',
+  background: 'transparent',
+  display: 'inline-flex',
+  alignItems: 'center',
+};
+
+const ImageJobAdminActions = ({ ciJobId, status }: Props) => {
+  const firestore = useFirestore();
+  const { data: user } = useUser();
+  const notify = useNotification();
+  const [runningAction, setRunningAction] = useState<'reset' | 'retry' | null>(null);
+
+  const ciBuilds = firestore.collection('ciBuilds').where('relatedJobId', '==', ciJobId);
+  const { status: buildStatus, data = [] } = useFirestoreCollectionData<BuildRecord>(ciBuilds);
+
+  if (status !== 'failed' || buildStatus === 'loading') return null;
+
+  const failedBuilds = data.filter((build) => build.status === 'failed');
+  if (failedBuilds.length === 0) return null;
+
+  const maxedOutBuilds = failedBuilds.filter((build) => (build.meta?.failureCount || 0) >= 15);
+
+  const callEndpoint = async (endpoint: string, payload: object) => {
+    if (!user) {
+      throw new Error('User not authenticated');
+    }
+
+    const token = await user.getIdToken();
+    const response = await fetch(`${config.backendUrl}/${endpoint}`, {
+      headers: {
+        Authorization: `Bearer ${token}`,
+        'Content-Type': 'application/json',
+      },
+      mode: 'cors',
+      method: 'POST',
+      body: JSON.stringify(payload),
+    });
+
+    const body = await response.json();
+    if (!response.ok) {
+      const detail = body.error ? `${body.message}: ${body.error}` : body.message;
+      throw new Error(detail || `Request failed (${response.status})`);
+    }
+    return body;
+  };
+
+  const runAction = async (
+    action: 'reset' | 'retry',
+    builds: BuildRecord[],
+    endpoint: 'resetFailedBuilds' | 'retryBuild',
+  ) => {
+    if (builds.length === 0) return;
+
+    setRunningAction(action);
+    try {
+      let succeeded = 0;
+      let failed = 0;
+
+      for (const build of builds) {
+        try {
+          const payload =
+            endpoint === 'retryBuild'
+              ? { buildId: build.buildId, relatedJobId: build.relatedJobId }
+              : { buildId: build.buildId };
+          // Sequential calls avoid hammering the backend for a single image row action.
+          await callEndpoint(endpoint, payload);
+          succeeded += 1;
+        } catch {
+          failed += 1;
+        }
+      }
+
+      if (failed > 0) {
+        notify.error(
+          `${action === 'retry' ? 'Retried' : 'Reset'} ${succeeded}/${builds.length} builds. ${failed} failed.`,
+        );
+      } else {
+        notify.success(
+          `${action === 'retry' ? 'Retried' : 'Reset'} ${succeeded} build${succeeded === 1 ? '' : 's'} for ${ciJobId}.`,
+        );
+      }
+    } finally {
+      setRunningAction(null);
+    }
+  };
+
+  return (
+    <SimpleAuthCheck fallback={<span />} requiredClaims={{ admin: true }}>
+      <span style={{ display: 'inline-flex', alignItems: 'center', gap: 6, marginLeft: 8 }}>
+        {maxedOutBuilds.length > 0 && (
+          <Tooltip
+            content={`Reset ${maxedOutBuilds.length} maxed-out failed build${maxedOutBuilds.length === 1 ? '' : 's'} so Ingeminator can retry them.`}
+          >
+            <button
+              type="button"
+              onClick={() => {
+                void runAction('reset', maxedOutBuilds, 'resetFailedBuilds');
+              }}
+              style={buttonStyle}
+              disabled={runningAction !== null}
+            >
+              <MdRestartAlt color={runningAction === 'reset' ? 'orange' : '#b45309'} />
+            </button>
+          </Tooltip>
+        )}
+        <Tooltip
+          content={`Retry ${failedBuilds.length} failed build${failedBuilds.length === 1 ? '' : 's'} for this image.`}
+        >
+          <button
+            type="button"
+            onClick={() => {
+              void runAction('retry', failedBuilds, 'retryBuild');
+            }}
+            style={buttonStyle}
+            disabled={runningAction !== null}
+          >
+            {runningAction === 'retry' ? <Spinner type="spin" /> : <HiOutlineRefresh color="red" />}
+          </button>
+        </Tooltip>
+      </span>
+    </SimpleAuthCheck>
+  );
+};
+
+export default ImageJobAdminActions;

src/components/docs/versions/image-versions.tsx

@@ -9,13 +9,13 @@
 import styles from './unity-version.module.scss';
 
 interface Props {
   versions: { [key: string]: any }[];
 }
 
 export type StatusFilter = 'all' | 'started' | 'failed' | 'published' | 'stuck';
 
 const ImageVersions = ({ versions }: Props) => {
   const [selectedVersion, setSelectedVersion] = useState<any>(versions[0].NO_ID_FIELD);
   const [searchQuery, setSearchQuery] = useState('');
   const [statusFilter, setStatusFilter] = useState<StatusFilter>('all');
 
@@ -40,29 +40,62 @@
     minWidth: 220,
   };
 
+  const headerCardStyle: React.CSSProperties = {
+    display: 'grid',
+    gap: 12,
+    padding: '14px 16px',
+    borderRadius: 10,
+    border: '1px solid #33333322',
+    background: '#fafafa08',
+    marginBottom: 12,
+  };
+
+  const toolbarStyle: React.CSSProperties = {
+    display: 'flex',
+    flexWrap: 'wrap',
+    gap: 10,
+    alignItems: 'center',
+    justifyContent: 'space-between',
+  };
+
   return (
     <main className={styles.versionsPanel}>
       <h1>Supported Editor Versions</h1>
 
-      <div>
-        <span>Docker repo version: </span>
+      <div style={headerCardStyle}>
+        <div style={toolbarStyle}>
+          <div style={{ display: 'flex', alignItems: 'center', gap: 8, flexWrap: 'wrap' }}>
+            <strong>Docker repo version</strong>
+            <select
+              value={selectedVersion}
+              onChange={(event) => setSelectedVersion(event.target.value)}
+              style={selectStyle}
+            >
+              {versions.map((version) => {
+                const { NO_ID_FIELD: id } = version;
 
-        <select onChange={(event) => setSelectedVersion(event.target.value)}>
-          {versions.map((version) => {
-            const { NO_ID_FIELD: id } = version;
+                return (
+                  <option key={id} value={id}>
+                    {id}
+                  </option>
+                );
+              })}
+            </select>
+          </div>
 
-            return (
-              <option key={id} value={id}>
-                {id}
-              </option>
-            );
-          })}
-        </select>
-        <span style={{ float: 'right', display: 'flex', alignItems: 'center', gap: 8 }}>
-          <ResetAllFailedBuildsButton />
-          <CleanUpStuckBuildsButton />
-          <SignInSignOutButton />
-        </span>
+          <div style={{ display: 'flex', alignItems: 'center', gap: 8, flexWrap: 'wrap' }}>
+            <SimpleAuthCheck fallback={<span />} requiredClaims={{ admin: true }}>
+              <ResetAllFailedBuildsButton />
+              <CleanUpStuckBuildsButton />
+            </SimpleAuthCheck>
+            <SignInSignOutButton />
+          </div>
+        </div>
+
+        <p style={{ margin: 0, opacity: 0.7, fontSize: '0.85em' }}>
+          Admin controls for queue recovery, retry management, and selected-repo diagnostics live
+          below. Use the dashboard first, then the queue panel for root-cause detail.
+        </p>
       </div>
 
       <BuildStatusDashboard selectedRepoVersion={selectedVersion} />