Add async queue-based verify to eliminate CloudFront timeout

- New SQLite-backed job queue (queue_manager.py) shared between API and
  worker containers via a named Docker volume
- New worker container (worker.py) processes one Ollama inference at a
  time, retrying up to 3 times on failure — eliminates all GPU-level
  concurrency issues
- POST /verify/async + GET /verify/status/{job_id} API endpoints replace
  the blocking /verify call for UI and test_verifier.py remote mode
- UI POST /ui/verify now enqueues and redirects to a polling spinner page
  (verify_pending.html) instead of blocking on Ollama
- test_verifier.py --remote-host updated to use async endpoint with
  poll_for_result() helper — fixes the sequential timeout failures
- Dockerfile gains a worker stage; docker-compose.dev.yml and production
  docker-compose.yml (instance_init.sh) gain the worker service
- deploy.yml builds and pushes ttb-verifier-worker:latest alongside app

Author: Project Team

.github/workflows/deploy.yml

@@ -7,6 +7,7 @@ on:
     paths:
       - 'app/**'
       - 'Dockerfile'
+      - 'docker-compose.dev.yml'
       - 'scripts/deploy.sh'
       - '.github/workflows/deploy.yml'
 
@@ -79,6 +80,18 @@ jobs:
           cache-from: type=gha
           cache-to: type=gha,mode=max
 
+      - name: Build and push worker Docker image
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          target: worker
+          push: true
+          tags: |
+            ${{ env.REGISTRY }}/${{ steps.image.outputs.name }}-worker:latest
+            ${{ env.REGISTRY }}/${{ steps.image.outputs.name }}-worker:${{ github.ref_name }}-${{ github.sha }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+
   deploy:
     needs: build-and-push
     runs-on: ubuntu-latest

Dockerfile

@@ -31,7 +31,7 @@ RUN pytest tests/ \
     --cov-fail-under=50 \
     -v
 
-# Stage 4: Production image
+# Stage 4: Production image (FastAPI app — uvicorn 4 workers)
 FROM base AS production
 
 WORKDIR /app
@@ -48,8 +48,8 @@ COPY app/templates ./templates
 # Create samples directory (optional - for golden samples)
 RUN mkdir -p ./samples
 
-# Create jobs directory for async batch processing
-RUN mkdir -p /app/tmp/jobs
+# Create jobs directory for async batch processing and queue DB
+RUN mkdir -p /app/tmp/jobs /app/tmp/async
 
 # Set environment
 ENV PATH=/root/.local/bin:$PATH \
@@ -65,3 +65,29 @@ EXPOSE 8000
 
 # Run FastAPI with uvicorn (4 workers for concurrent request handling)
 CMD ["uvicorn", "api:app", "--host", "0.0.0.0", "--port", "8000", "--workers", "4"]
+
+# Stage 5: Worker image (single-process queue consumer)
+FROM base AS worker
+
+WORKDIR /app
+
+# Copy Python packages from builder
+COPY --from=builder /root/.local /root/.local
+
+# Copy application code (worker only needs queue_manager, worker, label_validator, ocr_backends)
+COPY app/*.py ./
+
+# Create shared volume directories
+RUN mkdir -p /app/tmp/jobs /app/tmp/async
+
+# Set environment
+ENV PATH=/root/.local/bin:$PATH \
+    PYTHONUNBUFFERED=1 \
+    PYTHONDONTWRITEBYTECODE=1
+
+# No health check port — worker has no HTTP server.
+# Docker will mark it healthy if the process is running.
+HEALTHCHECK --interval=30s --timeout=5s --start-period=15s --retries=3 \
+    CMD python -c "from queue_manager import QueueManager; QueueManager().queue_depth()" || exit 1
+
+CMD ["python", "worker.py"]

app/api.py

@@ -30,6 +30,7 @@
 from auth import get_current_user
 from middleware import HostCheckMiddleware
 from job_manager import JobManager, JobStatus
+from queue_manager import QueueManager
 
 
 # ============================================================================
@@ -48,6 +49,12 @@
 # Initialize job manager for async batch processing
 job_manager = JobManager()
 
+# Initialize queue manager for async single-image processing
+verify_queue = QueueManager(
+    db_path=Path(settings.queue_db_path),
+    max_attempts=settings.queue_max_attempts,
+)
+
 
 # ============================================================================
 # Pydantic Models
@@ -154,6 +161,24 @@ class ErrorResponse(BaseModel):
     correlation_id: str
 
 
+class AsyncVerifySubmitResponse(BaseModel):
+    """Response from async single-image verify submission."""
+    job_id: str
+    status: str   # always 'pending' on submit
+    message: str
+
+
+class AsyncVerifyStatusResponse(BaseModel):
+    """Response from async single-image verify status poll."""
+    job_id: str
+    status: str   # pending | processing | completed | failed | cancelled
+    attempts: int
+    max_attempts: int
+    result: Optional[VerifyResponse] = None
+    error: Optional[str] = None
+    queue_depth: Optional[int] = None  # jobs ahead in queue (only when pending)
+
+
 # ============================================================================
 # FastAPI Application
 # ============================================================================
@@ -910,6 +935,131 @@ async def delete_batch_job(
     return {"message": f"Job {job_id} deleted successfully"}
 
 
+# ============================================================================
+# Async Single-Image Verify Endpoints (queue-based, CloudFront-safe)
+# ============================================================================
+
+@app.post("/verify/async", response_model=AsyncVerifySubmitResponse)
+async def submit_async_verify(
+    image: UploadFile = File(..., description="Label image file (max 10MB)"),
+    ground_truth: Optional[str] = Form(None, description="Ground truth JSON string"),
+    username: str = Depends(get_current_user)
+) -> AsyncVerifySubmitResponse:
+    """
+    Submit a single label image for asynchronous verification via the worker queue.
+
+    Returns immediately with a ``job_id``.  Poll ``GET /verify/status/{job_id}``
+    every 2–3 seconds until ``status`` is ``completed`` or ``failed``.
+
+    This endpoint is designed to work within CloudFront's 60-second origin
+    read timeout: the HTTP response is sent instantly, and Ollama processing
+    (~10s) happens in the separate worker container.
+
+    **Request:**
+    - ``image``: Label image (JPEG or PNG, max 10MB)
+    - ``ground_truth``: Optional JSON with expected values
+
+    **Response:**
+    - ``job_id``: Use to poll ``GET /verify/status/{job_id}``
+    - ``status``: ``pending``
+
+    **Example:**
+    ```bash
+    JOB=$(curl -s -X POST https://example.com/verify/async \\
+      -F "image=@label.jpg" | jq -r .job_id)
+    # Poll until done:
+    curl https://example.com/verify/status/$JOB
+    ```
+    """
+    correlation_id = get_correlation_id()
+    logger.info(f"[{correlation_id}] POST /verify/async - {image.filename}")
+
+    # Validate image
+    validate_image_file(image, correlation_id)
+
+    # Parse optional ground truth
+    ground_truth_data = parse_ground_truth(ground_truth, correlation_id)
+
+    # Persist image to shared volume so the worker container can read it.
+    # Each job gets its own subdirectory to avoid filename collisions.
+    job_dir = Path(settings.queue_db_path).parent / "async" / str(uuid.uuid4())
+    job_dir.mkdir(parents=True, exist_ok=True)
+
+    # Sanitise filename (keep extension only)
+    suffix = Path(image.filename).suffix.lower() if image.filename else ".jpg"
+    image_dest = job_dir / f"image{suffix}"
+    await save_upload_file(image, image_dest)
+
+    job_id = verify_queue.enqueue(
+        image_path=str(image_dest),
+        ground_truth=ground_truth_data,
+    )
+
+    logger.info(f"[{correlation_id}] Enqueued async verify job {job_id}")
+
+    return AsyncVerifySubmitResponse(
+        job_id=job_id,
+        status="pending",
+        message=f"Job submitted. Poll GET /verify/status/{job_id} for results.",
+    )
+
+
+@app.get("/verify/status/{job_id}", response_model=AsyncVerifyStatusResponse)
+async def get_async_verify_status(
+    job_id: str,
+    username: str = Depends(get_current_user)
+) -> AsyncVerifyStatusResponse:
+    """
+    Poll the status of a queued single-image verify job.
+
+    Call this endpoint every 2–3 seconds after submitting via
+    ``POST /verify/async``.
+
+    **Status values:**
+    - ``pending``    — job is waiting in the queue
+    - ``processing`` — worker is currently running Ollama inference
+    - ``completed``  — result is ready (see ``result`` field)
+    - ``failed``     — all retry attempts exhausted (see ``error`` field)
+    - ``cancelled``  — job was cancelled
+
+    **Example:**
+    ```bash
+    curl https://example.com/verify/status/abc123
+    ```
+    """
+    correlation_id = get_correlation_id()
+    job = verify_queue.get(job_id)
+
+    if job is None:
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND,
+            detail=f"Verify job {job_id} not found",
+        )
+
+    result_obj: Optional[VerifyResponse] = None
+    if job["status"] == "completed" and job.get("result"):
+        try:
+            result_obj = VerifyResponse(**job["result"])
+        except Exception as exc:
+            logger.error(
+                f"[{correlation_id}] Failed to deserialise result for job {job_id}: {exc}"
+            )
+
+    queue_depth = None
+    if job["status"] == "pending":
+        queue_depth = verify_queue.queue_depth()
+
+    return AsyncVerifyStatusResponse(
+        job_id=job_id,
+        status=job["status"],
+        attempts=job["attempts"],
+        max_attempts=job["max_attempts"],
+        result=result_obj,
+        error=job.get("error"),
+        queue_depth=queue_depth,
+    )
+
+
 # ============================================================================
 # Exception Handlers
 # ============================================================================

app/config.py

@@ -53,6 +53,20 @@ class Settings(BaseSettings):
         default=3600,
         description="Interval between job cleanup runs in seconds (default: 1 hour)"
     )
+
+    # Async single-image queue configuration
+    queue_db_path: str = Field(
+        default="/app/tmp/queue.db",
+        description="Path to the SQLite queue database (shared volume)"
+    )
+    queue_max_attempts: int = Field(
+        default=3,
+        description="Maximum processing attempts per queued verify job before permanent failure"
+    )
+    worker_ollama_timeout_seconds: int = Field(
+        default=12,
+        description="Per-attempt Ollama timeout used by the worker process (seconds)"
+    )
 
     # CORS Configuration
     cors_origins: str = Field(