api_server.py

from fastapi import FastAPI, HTTPException, File, Form, UploadFile, Depends, Header, status, Request
from fastapi.security import APIKeyHeader
from fastapi.responses import HTMLResponse, FileResponse, StreamingResponse
from pydantic import ValidationError
from typing import Optional
from contextlib import asynccontextmanager
import logging
import json
import asyncio
import os
import hashlib
import shutil
import aiofiles
import time
import glob
import docker
import zipfile
import io
import httpx
import uuid
import subprocess
from logging.handlers import TimedRotatingFileHandler

from models import SubmissionRequest, LimitObject, Job
from queue_manager import job_queue
from worker import worker_manager
from llm_judge import get_llm_judge

# 設定日誌
log_dir = "logs"
os.makedirs(log_dir, exist_ok=True)
log_file = os.path.join(log_dir, "sandbox.log")
AUTH_LOG_FILE = os.path.join(log_dir, "auth_failures.jsonl")

# 設定 Checkers 目錄
CHECKERS_DIR = "checkers"
os.makedirs(CHECKERS_DIR, exist_ok=True)

handler = TimedRotatingFileHandler(
    log_file, 
    when="midnight", 
    interval=1, 
    backupCount=30, 
    encoding='utf-8'
)
handler.suffix = "%Y-%m-%d"

logging.basicConfig(
    level=logging.INFO, 
    format="%(asctime)s - %(levelname)s - %(message)s",
    handlers=[
        handler,
        logging.StreamHandler()
    ]
)
logger = logging.getLogger(__name__)

@asynccontextmanager
async def lifespan(app: FastAPI):
    # Startup
    # 檢查必要的環境變數
    api_key = os.getenv("SANDBOX_API_KEY", "")
    if not api_key or api_key == "default-insecure-key":
        logger.warning("=" * 60)
        logger.warning("WARNING: SANDBOX_API_KEY 未設定或使用不安全的預設值！")
        logger.warning("請在 .env 檔案中設定強密碼: SANDBOX_API_KEY=your-strong-key")
        logger.warning("生成強密碼: openssl rand -hex 32")
        logger.warning("=" * 60)
    
    # 預先拉取常用 Sidecar 映像檔（背景執行，不阻塞啟動）
    from worker import SidecarManager
    asyncio.create_task(SidecarManager.preload_common_images())
    
    await worker_manager.start()
    yield
    # Shutdown
    await worker_manager.stop()

app = FastAPI(
    title="Sandbox Isolate API",
    version="2.0.0",
    description="非同步 Isolate 沙盒評測服務",
    lifespan=lifespan
)

# ===== Security =====
API_KEY_NAME = "X-API-KEY"
API_KEY = os.getenv("SANDBOX_API_KEY", "")
if not API_KEY:
    API_KEY = "default-insecure-key"  # Fallback for development only
    
api_key_header = APIKeyHeader(name=API_KEY_NAME, auto_error=False)

async def verify_api_key(request: Request, api_key: str = Depends(api_key_header)):
    """驗證 API Key"""
    if not API_KEY or API_KEY == "default-insecure-key":
        raise HTTPException(
            status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
            detail="Server not configured: SANDBOX_API_KEY not set"
        )
    if api_key != API_KEY:
        # Log failure
        try:
            entry = {
                "timestamp": time.time(),
                "ip": request.client.host if request.client else "unknown",
                "method": request.method,
                "path": request.url.path,
                "provided_key": (api_key[:3] + "***") if api_key else "None"
            }
            async with aiofiles.open(AUTH_LOG_FILE, "a") as f:
                await f.write(json.dumps(entry) + "\n")
        except Exception as e:
            logger.error(f"Failed to log auth failure: {e}")

        raise HTTPException(
            status_code=status.HTTP_403_FORBIDDEN,
            detail="Could not validate credentials"
        )
    return api_key

async def fetch_submission_code(submission_id: str):
    """從後端取得學生程式碼與語言"""
    backend_url = os.getenv("BACKEND_API_URL", "").strip()
    
    # 如果沒有設定後端 URL，拋出異常
    if not backend_url:
        raise HTTPException(status_code=503, detail="BACKEND_API_URL not configured")
    
    url = f"{backend_url}/submission/{submission_id}/code/"

    try:
        async with httpx.AsyncClient() as client:
            resp = await client.get(url, timeout=30.0)

        if resp.status_code in (403, 404):
            raise HTTPException(status_code=resp.status_code, detail=resp.text or "Upstream returned error")
        if resp.status_code != 200:
            raise HTTPException(status_code=502, detail=f"Failed to fetch submission code (status {resp.status_code})")

        code = None
        language = None

        content_type = resp.headers.get("content-type", "").lower()
        if "application/json" in content_type:
            try:
                payload = resp.json()
                data_obj = payload.get("data") or {}

                # 嘗試各種鍵名，優先 data 區塊
                code = (
                    data_obj.get("source_code")
                    or data_obj.get("sourceCode")
                    or payload.get("code")
                    or payload.get("submission_code")
                )

                language = (
                    data_obj.get("language_type")
                    or data_obj.get("languageType")
                    or data_obj.get("language")
                    or payload.get("language")
                    or payload.get("lang")
                )
            except Exception as e:
                logger.warning(f"Submission code JSON parse failed: {e}")

        if code is None:
            code = resp.text

        if not code:
            raise HTTPException(status_code=502, detail="Submission code is empty from backend")
        if not language:
            raise HTTPException(status_code=502, detail="Submission language missing from backend response")

        # 語言標準化：支援數字代碼與字串代碼
        lang_map_numeric = {
            0: "c",
            1: "cpp",
            2: "python",
            3: "java",
            4: "javascript",
        }

        normalized_language = None
        if isinstance(language, int):
            normalized_language = lang_map_numeric.get(language)
        else:
            normalized_language = str(language).lower()

        if normalized_language is None:
            raise HTTPException(status_code=400, detail=f"Unsupported language from backend: {language}")

        return code, normalized_language
    except HTTPException:
        raise
    except Exception as e:
        logger.error(f"Error fetching submission code: {e}")
        raise HTTPException(status_code=502, detail=f"Failed to fetch submission code: {str(e)}")

# ===== API Endpoints =====

@app.get("/api/v1/checkers", dependencies=[Depends(verify_api_key)])
async def list_checkers():
    """
    列出所有已上傳的 Checker
    """
    checkers = []
    try:
        for f in os.listdir(CHECKERS_DIR):
            path = os.path.join(CHECKERS_DIR, f)
            if os.path.isfile(path):
                stat = os.stat(path)
                # 判斷類型
                checker_type = "unknown"
                if f.endswith('.py'):
                    checker_type = "python"
                elif os.access(path, os.X_OK):
                    checker_type = "binary"
                
                checkers.append({
                    "name": f,
                    "size": stat.st_size,
                    "modified": stat.st_mtime,
                    "type": checker_type,
                    "executable": os.access(path, os.X_OK)
                })
        return {"success": True, "checkers": checkers, "count": len(checkers)}
    except Exception as e:
        raise HTTPException(status_code=500, detail=f"Failed to list checkers: {str(e)}")

@app.get("/api/v1/checkers/{name}", dependencies=[Depends(verify_api_key)])
async def get_checker_info(name: str):
    """
    查詢單個 Checker 詳情
    """
    path = os.path.join(CHECKERS_DIR, name)
    if not os.path.exists(path):
        raise HTTPException(status_code=404, detail=f"Checker '{name}' not found")
    
    stat = os.stat(path)
    checker_type = "unknown"
    if name.endswith('.py'):
        checker_type = "python"
    elif os.access(path, os.X_OK):
        checker_type = "binary"
    
    return {
        "success": True,
        "name": name,
        "path": os.path.abspath(path),
        "size": stat.st_size,
        "modified": stat.st_mtime,
        "type": checker_type,
        "executable": os.access(path, os.X_OK)
    }

@app.get("/api/v1/checkers/{name}/download", dependencies=[Depends(verify_api_key)])
async def download_checker(name: str):
    """
    下載指定的 Checker 檔案
    """
    path = os.path.join(CHECKERS_DIR, name)
    if not os.path.exists(path):
        raise HTTPException(status_code=404, detail=f"Checker '{name}' not found")
    
    # 判斷 media_type
    if name.endswith('.py'):
        media_type = "text/x-python"
    else:
        media_type = "application/octet-stream"
    
    return FileResponse(
        path=path,
        filename=name,
        media_type=media_type
    )

@app.delete("/api/v1/checkers/{name}", dependencies=[Depends(verify_api_key)])
async def delete_checker(name: str):
    """
    刪除指定的 Checker
    """
    path = os.path.join(CHECKERS_DIR, name)
    if not os.path.exists(path):
        raise HTTPException(status_code=404, detail=f"Checker '{name}' not found")
    
    try:
        os.remove(path)
        logger.info(f"Checker '{name}' deleted")
        return {"success": True, "message": f"Checker '{name}' deleted successfully"}
    except Exception as e:
        raise HTTPException(status_code=500, detail=f"Failed to delete checker: {str(e)}")

@app.post("/api/v1/checkers", dependencies=[Depends(verify_api_key)])
async def upload_checker(
    name: str = Form(..., description="Checker 名稱 (e.g., my_checker)"),
    file: UploadFile = File(..., description="Checker 執行檔 (Binary or Script)")
):
    """
    上傳 Custom Checker
    """
    file_path = os.path.join(CHECKERS_DIR, name)
    
    try:
        async with aiofiles.open(file_path, 'wb') as out_file:
            content = await file.read()
            await out_file.write(content)
        
        # Make executable
        os.chmod(file_path, 0o755)
        
        return {"success": True, "message": f"Checker '{name}' uploaded successfully", "path": file_path}
    except Exception as e:
        raise HTTPException(status_code=500, detail=f"Failed to upload checker: {str(e)}")

@app.post("/api/v1/checkers/compile", dependencies=[Depends(verify_api_key)])
async def upload_and_compile_checker(
    name: str = Form(..., description="編譯後的 Checker 名稱"),
    source: UploadFile = File(..., description="Checker 源碼 (.cpp/.c)")
):
    """
    上傳 Checker 源碼並編譯
    
    支援 C/C++ 源碼上傳，自動編譯為可執行檔
    """
    content = await source.read()
    ext = os.path.splitext(source.filename)[1].lower()
    
    if ext not in ['.c', '.cpp']:
        raise HTTPException(status_code=400, detail="Only .c and .cpp source files are supported")
    
    # 暫存源碼
    src_path = f"/tmp/checker_{name}_{uuid.uuid4().hex[:8]}{ext}"
    out_path = os.path.join(CHECKERS_DIR, name)
    
    try:
        async with aiofiles.open(src_path, 'wb') as f:
            await f.write(content)
        
        # 編譯
        compiler = "g++" if ext == ".cpp" else "gcc"
        cmd = [compiler, "-O2", "-o", out_path, src_path]
        
        result = subprocess.run(cmd, capture_output=True, text=True, timeout=30)
        
        if result.returncode != 0:
            raise HTTPException(status_code=400, detail=f"Compilation failed: {result.stderr}")
        
        os.chmod(out_path, 0o755)
        logger.info(f"Checker '{name}' compiled from {source.filename}")
        
        return {
            "success": True, 
            "message": f"Checker '{name}' compiled successfully",
            "source_file": source.filename,
            "output_path": out_path
        }
    except subprocess.TimeoutExpired:
        raise HTTPException(status_code=408, detail="Compilation timed out")
    except HTTPException:
        raise
    except Exception as e:
        raise HTTPException(status_code=500, detail=f"Compilation error: {str(e)}")
    finally:
        # 清理暫存檔
        if os.path.exists(src_path):
            os.unlink(src_path)

@app.post("/api/v1/evaluate-package", status_code=status.HTTP_202_ACCEPTED, dependencies=[Depends(verify_api_key)])
async def evaluate_package(
    submission_id: str = Form(..., description="唯一的提交 ID"),
    package: UploadFile = File(..., description="完整題包 ZIP (包含 submission, testcase, checker)"),
    package_hash: str = Form(..., description="題包 SHA256 Hash"),
    
    time_limit: float = Form(2.0, description="CPU 時間限制 (秒)"),
    memory_limit: int = Form(256000, description="記憶體限制 (KB)"),
    
    callback_url: Optional[str] = Form(None, description="Webhook URL"),
    callback_token: Optional[str] = Form(None, description="Webhook Token"),
    priority: int = Form(10, description="優先級"),
):
    """
    評測完整題包
    
    題包結構：
    - meta.json: 題目元數據
    - testcase/*.in, testcase/*.out: 測試用例
    - checker/checker.{c,cpp}: Checker 源代碼
    - (submission code 透過後端 API 取得)
    """
    logger.info(f"Received package evaluation {submission_id}")
    
    # 1. 建立暫存目錄
    work_dir = f"/tmp/sandbox/packages/{submission_id}"
    os.makedirs(work_dir, exist_ok=True)
    
    try:
        # 2. 儲存並驗證題包
        package_path = os.path.join(work_dir, "package.zip")
        async with aiofiles.open(package_path, 'wb') as out_file:
            content = await package.read()
            await out_file.write(content)
        
        # 驗證 Hash
        sha256_hash = hashlib.sha256(content).hexdigest()
        if sha256_hash != package_hash:
            shutil.rmtree(work_dir, ignore_errors=True)
            raise HTTPException(status_code=400, detail=f"Package hash mismatch. Expected {package_hash}, got {sha256_hash}")
        
        # 3. 解壓題包
        extract_dir = os.path.join(work_dir, "extracted")
        os.makedirs(extract_dir, exist_ok=True)
        
        with zipfile.ZipFile(package_path) as z:
            z.extractall(extract_dir)
        
        # 4. 讀取 meta.json
        meta_path = os.path.join(extract_dir, "meta.json")
        if not os.path.exists(meta_path):
            raise HTTPException(status_code=400, detail="Missing meta.json in package")
        
        async with aiofiles.open(meta_path, 'r') as f:
            meta = json.loads(await f.read())
        
        # 5. 從後端取得 submission code
        code_text, language_raw = await fetch_submission_code(submission_id)

        lang_map = {
            "python": "python",
            "python3": "python",
            "py": "python",
            "c": "c",
            "cpp": "cpp",
            "c++": "cpp",
            "java": None, # 不支援
            "javascript": None, # 不支援
        }

        language_key = str(language_raw).lower()
        language = lang_map.get(language_key)
        if not language:
            raise HTTPException(status_code=400, detail=f"Unsupported language from backend: {language_raw}")

        ext_map = {"python": ".py", "c": ".c", "cpp": ".cpp"}
        submission_path = os.path.join(extract_dir, f"submission{ext_map[language]}")

        async with aiofiles.open(submission_path, "w", encoding="utf-8") as f:
            await f.write(code_text)
        
        # 6. 處理 Checker（支援多種格式）
        checker_path = None
        checker_dir = os.path.join(extract_dir, "checker")
        
        if os.path.exists(checker_dir):
            # 優先順序：已編譯 binary > C++ > C > Python
            checker_candidates = [
                ("checker", None),           # 已編譯的 binary
                ("checker.cpp", "cpp"),      # C++ 源碼
                ("checker.c", "c"),          # C 源碼  
                ("checker.py", "python"),    # Python 腳本
            ]
            
            for checker_file, checker_type in checker_candidates:
                potential_checker = os.path.join(checker_dir, checker_file)
                if os.path.exists(potential_checker):
                    if checker_type == "cpp":
                        # 編譯 C++ checker（包含 testlib.h 支援）
                        compiled_checker = os.path.join(work_dir, "checker_compiled")
                        
                        # 檢查是否有 testlib.h
                        testlib_path = os.path.join(checker_dir, "testlib.h")
                        include_flag = ["-I", checker_dir] if os.path.exists(testlib_path) else []
                        
                        cmd = ["g++", "-O2", "-o", compiled_checker] + include_flag + [potential_checker, "-lm"]
                        
                        logger.info(f"Compiling C++ checker: {' '.join(cmd)}")
                        result = subprocess.run(cmd, capture_output=True, text=True, timeout=30)
                        
                        if result.returncode != 0:
                            logger.error(f"Checker compilation failed: {result.stderr}")
                            raise HTTPException(status_code=400, detail=f"Checker compilation error: {result.stderr}")
                        
                        checker_path = compiled_checker
                        os.chmod(checker_path, 0o755)
                        
                    elif checker_type == "c":
                        # 編譯 C checker
                        compiled_checker = os.path.join(work_dir, "checker_compiled")
                        cmd = ["gcc", "-O2", "-o", compiled_checker, potential_checker, "-lm"]
                        
                        logger.info(f"Compiling C checker: {' '.join(cmd)}")
                        result = subprocess.run(cmd, capture_output=True, text=True, timeout=30)
                        
                        if result.returncode != 0:
                            logger.error(f"Checker compilation failed: {result.stderr}")
                            raise HTTPException(status_code=400, detail=f"Checker compilation error: {result.stderr}")
                        
                        checker_path = compiled_checker
                        os.chmod(checker_path, 0o755)
                        
                    elif checker_type == "python":
                        # Python checker 不需要編譯，直接使用
                        # 複製到 work_dir 確保路徑一致
                        python_checker_path = os.path.join(work_dir, "checker.py")
                        shutil.copy(potential_checker, python_checker_path)
                        checker_path = python_checker_path
                        os.chmod(checker_path, 0o755)
                        logger.info(f"Using Python checker: {checker_path}")
                        
                    else:
                        # 已編譯的 binary
                        checker_path = potential_checker
                        os.chmod(checker_path, 0o755)
                        logger.info(f"Using pre-compiled checker: {checker_path}")
                    
                    break
        
        # 也檢查 meta.json 中是否指定了 checker
        if not checker_path and meta.get("checker"):
            checker_name = meta.get("checker")
            # 從預設 checkers 目錄尋找
            default_checker = os.path.join(CHECKERS_DIR, checker_name)
            if os.path.exists(default_checker):
                checker_path = os.path.abspath(default_checker)
                logger.info(f"Using default checker from meta.json: {checker_name}")
        
        # 7. 讀取測試用例
        testcase_dir = os.path.join(extract_dir, "testcase")
        if not os.path.exists(testcase_dir):
            raise HTTPException(status_code=400, detail="Missing testcase directory")
        
        # 8. 建構 Request Data
        limits = LimitObject(
            time_limit_sec=time_limit,
            memory_limit_kb=memory_limit,
            wall_time_limit_sec=time_limit * 2.5
        )
        
        request_data = SubmissionRequest(
            submission_id=submission_id,
            code="<PACKAGE>",
            language=language,
            priority=priority,
            limits=limits,
            callback_url=callback_url,
            callback_token=callback_token
        )
        
        # 9. 建立 Job (使用特殊的 problem_id 表示這是 package 模式)
        job = Job(
            submission_id=submission_id,
            language=language,
            priority=priority,
            timestamp=time.time(),
            request_data=request_data,
            
            problem_id=f"package_{submission_id}",  # 特殊標記
            mode="package",  # 新模式：package
            file_path=submission_path,
            file_hash=package_hash,
            problem_hash=package_hash,
            
            use_checker=bool(checker_path),
            checker_file_path=checker_path,
            
            stdin=None,  # Package 模式下使用測試用例文件
            
            allow_network=meta.get("allow_network", False),
            network_whitelist=meta.get("network_whitelist", []),
            sidecar_image=meta.get("sidecar_image", None)
        )
        
        # 10. 推入隊列
        await job_queue.push(job)
        logger.info(f"Package evaluation job {submission_id} added to queue")
        
        return {
            "success": True,
            "submission_id": submission_id,
            "status": "queued",
            "language": language,
            "has_checker": bool(checker_path),
            "queue_position": job_queue.size()
        }
        
    except HTTPException:
        shutil.rmtree(work_dir, ignore_errors=True)
        raise
    except Exception as e:
        shutil.rmtree(work_dir, ignore_errors=True)
        logger.error(f"Error processing package: {e}")
        raise HTTPException(status_code=500, detail=f"Failed to process package: {str(e)}")

async def compile_problem_checker(problem_dir: str):
    """
    編譯題包內的 checker（如果存在）
    支援 C/C++ 源碼編譯和 Python checker
    """
    checker_dir = os.path.join(problem_dir, "checker")
    if not os.path.exists(checker_dir):
        return None
    
    compiled_checker_path = os.path.join(problem_dir, "checker_compiled")
    
    # 優先順序：已編譯 binary > C++ > C > Python
    checker_candidates = [
        ("checker", None),           # 已編譯的 binary
        ("checker.cpp", "cpp"),      # C++ 源碼
        ("checker.c", "c"),          # C 源碼  
        ("checker.py", "python"),    # Python 腳本
    ]
    
    for checker_file, checker_type in checker_candidates:
        potential_checker = os.path.join(checker_dir, checker_file)
        if os.path.exists(potential_checker):
            try:
                if checker_type == "cpp":
                    # 編譯 C++ checker（包含 testlib.h 支援）
                    testlib_path = os.path.join(checker_dir, "testlib.h")
                    include_flag = ["-I", checker_dir] if os.path.exists(testlib_path) else []
                    
                    cmd = ["g++", "-O2", "-o", compiled_checker_path] + include_flag + [potential_checker, "-lm"]
                    
                    logger.info(f"Compiling C++ checker for problem: {' '.join(cmd)}")
                    result = subprocess.run(cmd, capture_output=True, text=True, timeout=30)
                    
                    if result.returncode != 0:
                        logger.error(f"Checker compilation failed: {result.stderr}")
                        return None
                    
                    os.chmod(compiled_checker_path, 0o755)
                    return compiled_checker_path
                    
                elif checker_type == "c":
                    # 編譯 C checker
                    cmd = ["gcc", "-O2", "-o", compiled_checker_path, potential_checker, "-lm"]
                    
                    logger.info(f"Compiling C checker for problem: {' '.join(cmd)}")
                    result = subprocess.run(cmd, capture_output=True, text=True, timeout=30)
                    
                    if result.returncode != 0:
                        logger.error(f"Checker compilation failed: {result.stderr}")
                        return None
                    
                    os.chmod(compiled_checker_path, 0o755)
                    return compiled_checker_path
                    
                elif checker_type == "python":
                    # Python checker 不需要編譯
                    # 複製到題目目錄的根目錄
                    python_checker_path = os.path.join(problem_dir, "checker.py")
                    shutil.copy(potential_checker, python_checker_path)
                    os.chmod(python_checker_path, 0o755)
                    logger.info(f"Using Python checker: {python_checker_path}")
                    return python_checker_path
                    
                else:
                    # 已編譯的 binary，複製到題目根目錄
                    shutil.copy(potential_checker, compiled_checker_path)
                    os.chmod(compiled_checker_path, 0o755)
                    logger.info(f"Using pre-compiled checker: {compiled_checker_path}")
                    return compiled_checker_path
                    
            except Exception as e:
                logger.error(f"Error processing checker: {e}")
                return None
    
    return None

async def fetch_problem_checksum(problem_id: str) -> str:
    """
    從後端取得題目測資的 MD5 checksum
    使用 Sandbox Token 驗證
    """
    backend_url = os.getenv("BACKEND_API_URL", "").strip()
    
    # 如果沒有設定後端 URL，拋出異常讓 caller 處理 (使用本地題包)
    if not backend_url:
        raise HTTPException(status_code=503, detail="BACKEND_API_URL not configured, using local packages only")
    
    sandbox_token = os.getenv("SANDBOX_TOKEN", "")
    
    url = f"{backend_url}/problem/{problem_id}/checksum?token={sandbox_token}"
    
    try:
        async with httpx.AsyncClient() as client:
            resp = await client.get(url, timeout=30.0)
            
            if resp.status_code == 401:
                raise HTTPException(status_code=401, detail="Invalid sandbox token")
            if resp.status_code == 404:
                raise HTTPException(status_code=404, detail=f"Problem {problem_id} or testdata not found")
            if resp.status_code != 200:
                raise HTTPException(status_code=502, detail=f"Failed to fetch checksum: {resp.status_code}")
            
            data = resp.json()
            return data.get("data", {}).get("checksum", "")
    except HTTPException:
        raise
    except Exception as e:
        logger.error(f"Error fetching checksum for problem {problem_id}: {e}")
        raise HTTPException(status_code=502, detail=f"Failed to fetch checksum: {str(e)}")

async def fetch_problem_meta(problem_id: str) -> dict:
    """
    從後端取得題目測資的元資料（tasks 結構）
    使用 Sandbox Token 驗證
    """
    backend_url = os.getenv("BACKEND_API_URL", "").strip()
    
    # 如果沒有設定後端 URL，拋出異常
    if not backend_url:
        raise HTTPException(status_code=503, detail="BACKEND_API_URL not configured")
    
    sandbox_token = os.getenv("SANDBOX_TOKEN", "")
    
    url = f"{backend_url}/problem/{problem_id}/meta?token={sandbox_token}"
    
    try:
        async with httpx.AsyncClient() as client:
            resp = await client.get(url, timeout=30.0)
            
            if resp.status_code == 401:
                raise HTTPException(status_code=401, detail="Invalid sandbox token")
            if resp.status_code == 404:
                raise HTTPException(status_code=404, detail=f"Problem {problem_id} or testdata not found")
            if resp.status_code != 200:
                raise HTTPException(status_code=502, detail=f"Failed to fetch meta: {resp.status_code}")
            
            data = resp.json()
            return data.get("data", {})
    except HTTPException:
        raise
    except Exception as e:
        logger.error(f"Error fetching meta for problem {problem_id}: {e}")
        raise HTTPException(status_code=502, detail=f"Failed to fetch meta: {str(e)}")

async def download_problem_testdata(problem_id: str) -> bytes:
    """
    從後端下載題目測資包 (ZIP)
    使用 Sandbox Token 驗證
    """
    backend_url = os.getenv("BACKEND_API_URL", "").strip()
    
    # 如果沒有設定後端 URL，拋出異常
    if not backend_url:
        raise HTTPException(status_code=503, detail="BACKEND_API_URL not configured")
    
    sandbox_token = os.getenv("SANDBOX_TOKEN", "")
    
    url = f"{backend_url}/problem/{problem_id}/testdata?token={sandbox_token}"
    
    try:
        async with httpx.AsyncClient() as client:
            resp = await client.get(url, timeout=120.0)  # 較長的超時時間以應對大檔案
            
            if resp.status_code == 401:
                raise HTTPException(status_code=401, detail="Invalid sandbox token")
            if resp.status_code == 404:
                raise HTTPException(status_code=404, detail=f"Problem {problem_id} or testdata not found")
            if resp.status_code != 200:
                raise HTTPException(status_code=502, detail=f"Failed to download testdata: {resp.status_code}")
            
            return resp.content
    except HTTPException:
        raise
    except Exception as e:
        logger.error(f"Error downloading testdata for problem {problem_id}: {e}")
        raise HTTPException(status_code=502, detail=f"Failed to download testdata: {str(e)}")

async def ensure_problem_package(problem_id: str, problem_hash: str):
    """
    確保題目測資包存在且為最新版本。
    
    工作流程：
    1. 先檢查本地是否已有題包且 checksum 匹配（優先使用本地）
    2. 如果本地不存在或 checksum 不匹配，嘗試從後端下載
    3. 驗證下載的檔案完整性（MD5）
    4. 解壓並編譯 checker（如果存在）
    5. 生成 meta.json 供 worker 使用
    """
    problem_dir = os.path.join("test_data", "problems", problem_id)
    hash_file = os.path.join(problem_dir, ".checksum")  # 改用 checksum 檔名
    meta_file = os.path.join(problem_dir, "meta.json")
    
    # 1. 先檢查本地題包是否存在且 checksum 匹配
    if os.path.exists(problem_dir) and os.path.exists(hash_file) and os.path.exists(meta_file):
        try:
            async with aiofiles.open(hash_file, "r") as f:
                local_checksum = (await f.read()).strip()
            
            # 如果傳入的 problem_hash 與本地 checksum 匹配，直接使用本地題包
            if local_checksum == problem_hash:
                logger.info(f"Problem {problem_id} using local package (checksum: {problem_hash})")
                return  # 使用本地題包
        except Exception as e:
            logger.warning(f"Error reading local checksum for {problem_id}: {e}")
    
    # 2. 嘗試從後端獲取最新版本
    try:
        # 從後端取得最新 checksum
        remote_checksum = await fetch_problem_checksum(problem_id)
        
        # 檢查本地是否已有相同 checksum 的測資包
        if os.path.exists(problem_dir) and os.path.exists(hash_file):
            try:
                async with aiofiles.open(hash_file, "r") as f:
                    local_checksum = (await f.read()).strip()
                if local_checksum == remote_checksum:
                    logger.info(f"Problem {problem_id} testdata is up-to-date (checksum match)")
                    return  # 已是最新版本
            except Exception:
                pass  # 讀取失敗，重新下載
        
        # 3. 下載測資包
        logger.info(f"Downloading testdata for problem {problem_id}...")
        zip_content = await download_problem_testdata(problem_id)
        
        logger.info(f"Problem {problem_id} testdata downloaded (checksum: {remote_checksum})")
        
        # 5. 解壓測資包 - 直接放到題目根目錄（無 testcase 子目錄）
        if os.path.exists(problem_dir):
            shutil.rmtree(problem_dir)
        os.makedirs(problem_dir, exist_ok=True)
        
        meta_from_zip = None
        
        with zipfile.ZipFile(io.BytesIO(zip_content)) as z:
            for member in z.namelist():
                filename = os.path.basename(member)
                if not filename:
                    continue
                
                # 處理 meta.json - 放到題目根目錄並讀取內容
                if filename == 'meta.json':
                    target_path = os.path.join(problem_dir, filename)
                    with z.open(member) as source:
                        content = source.read()
                        with open(target_path, 'wb') as target:
                            target.write(content)
                        # 解析 meta.json 內容
                        try:
                            meta_from_zip = json.loads(content.decode('utf-8'))
                        except Exception as e:
                            logger.warning(f"Failed to parse meta.json from ZIP: {e}")
                
                # 處理 .in 和 .out 檔案 - 直接放到題目根目錄
                elif filename.endswith('.in') or filename.endswith('.out'):
                    target_path = os.path.join(problem_dir, filename)
                    with z.open(member) as source, open(target_path, 'wb') as target:
                        target.write(source.read())
                
                # 處理 checker 檔案
                elif 'checker' in filename.lower() or filename.endswith('.cpp') or filename.endswith('.py'):
                    if 'checker' in member.lower():
                        target_path = os.path.join(problem_dir, filename)
                        with z.open(member) as source, open(target_path, 'wb') as target:
                            target.write(source.read())
        
        # 6. 如果 ZIP 內有 meta.json，補充 checksum 並保存
        if meta_from_zip:
            meta_from_zip["checksum"] = remote_checksum
            async with aiofiles.open(meta_file, "w") as f:
                await f.write(json.dumps(meta_from_zip, indent=2))
            logger.info(f"Problem {problem_id} using meta.json from ZIP package")
        else:
            # 如果 ZIP 內沒有 meta.json，嘗試從後端 API 取得
            logger.warning(f"Problem {problem_id} ZIP has no meta.json, fetching from API...")
            try:
                meta_data = await fetch_problem_meta(problem_id)
                meta_json = {
                    "checksum": remote_checksum,
                    "tasks": meta_data.get("tasks", []),
                    "allow_network": False,
                    "network_whitelist": [],
                    "sidecar_image": None
                }
                async with aiofiles.open(meta_file, "w") as f:
                    await f.write(json.dumps(meta_json, indent=2))
            except Exception as e:
                logger.error(f"Failed to fetch meta for {problem_id}: {e}")
                # 建立最小 meta.json
                meta_json = {"checksum": remote_checksum, "tasks": []}
                async with aiofiles.open(meta_file, "w") as f:
                    await f.write(json.dumps(meta_json, indent=2))
        
        # 7. 寫入 checksum 檔案
        async with aiofiles.open(hash_file, "w") as f:
            await f.write(remote_checksum)
        
        # 8. 編譯題包內的 checker（如果存在）
        checker_path = await compile_problem_checker(problem_dir)
        if checker_path:
            logger.info(f"Problem {problem_id} checker compiled: {checker_path}")
        
        logger.info(f"Problem {problem_id} testdata package ready.")
        
    except HTTPException as he:
        # 如果本地有題包（無論版本），優先使用
        if os.path.exists(problem_dir) and os.path.exists(meta_file):
            logger.warning(f"Failed to fetch remote problem {problem_id} ({he.detail}), using local version.")
            return
        raise
    except Exception as e:
        logger.error(f"Error ensuring problem package {problem_id}: {e}")
        # 如果本地有題包（無論版本），優先使用
        if os.path.exists(problem_dir) and os.path.exists(meta_file):
            logger.warning(f"Error updating problem {problem_id} ({e}), using local version.")
            return
        raise HTTPException(status_code=503, detail=f"Problem package missing and fetch failed: {str(e)}")

# ===== Problem Package Manual Upload API =====

@app.get("/api/v1/problems", dependencies=[Depends(verify_api_key)])
async def list_problems():
    """
    列出所有已快取的題目
    """
    problems_dir = os.path.join("test_data", "problems")
    if not os.path.exists(problems_dir):
        return {"success": True, "problems": [], "count": 0}
    
    problems = []
    for d in os.listdir(problems_dir):
        problem_path = os.path.join(problems_dir, d)
        if os.path.isdir(problem_path):
            meta_file = os.path.join(problem_path, "meta.json")
            hash_file = os.path.join(problem_path, ".checksum")
            
            # 讀取 meta.json
            meta = {}
            if os.path.exists(meta_file):
                try:
                    async with aiofiles.open(meta_file, "r") as f:
                        meta = json.loads(await f.read())
                except:
                    pass
            
            # 讀取 checksum
            checksum = None
            if os.path.exists(hash_file):
                try:
                    async with aiofiles.open(hash_file, "r") as f:
                        checksum = (await f.read()).strip()
                except:
                    pass
            
            # 統計測資數量 - 直接在題目根目錄
            testcase_count = len([f for f in os.listdir(problem_path) if f.endswith('.in')])
            
            # 檢查 checker
            has_checker = os.path.exists(os.path.join(problem_path, "checker_compiled")) or \
                          os.path.exists(os.path.join(problem_path, "checker.py"))
            
            problems.append({
                "problem_id": d,
                "checksum": checksum,
                "testcase_count": testcase_count,
                "has_checker": has_checker,
                "task_count": meta.get("task_count", 0)
            })
    
    return {"success": True, "problems": problems, "count": len(problems)}

@app.get("/api/v1/problems/{problem_id}", dependencies=[Depends(verify_api_key)])
async def get_problem_info(problem_id: str):
    """
    查詢單個題目詳情
    """
    problem_dir = os.path.join("test_data", "problems", problem_id)
    if not os.path.exists(problem_dir):
        raise HTTPException(status_code=404, detail=f"Problem '{problem_id}' not found")
    
    meta_file = os.path.join(problem_dir, "meta.json")
    hash_file = os.path.join(problem_dir, ".checksum")
    
    # 讀取 meta.json
    meta = {}
    if os.path.exists(meta_file):
        async with aiofiles.open(meta_file, "r") as f:
            meta = json.loads(await f.read())
    
    # 讀取 checksum
    checksum = None
    if os.path.exists(hash_file):
        async with aiofiles.open(hash_file, "r") as f:
            checksum = (await f.read()).strip()
    
    # 列出測資檔案 - 直接在題目根目錄
    testcases = []
    for f in sorted(os.listdir(problem_dir)):
        if f.endswith('.in'):
            stem = f[:-3]
            testcases.append({
                "input": f,
                "output": f"{stem}.out",
                "input_exists": True,
                "output_exists": os.path.exists(os.path.join(problem_dir, f"{stem}.out"))
            })
    
    # Checker 資訊
    checker_info = None
    if os.path.exists(os.path.join(problem_dir, "checker_compiled")):
        checker_info = {"type": "compiled", "path": "checker_compiled"}
    elif os.path.exists(os.path.join(problem_dir, "checker.py")):
        checker_info = {"type": "python", "path": "checker.py"}
    elif os.path.exists(os.path.join(problem_dir, "checker.cpp")):
        checker_info = {"type": "source_cpp", "path": "checker.cpp", "compiled": False}
    elif os.path.exists(os.path.join(problem_dir, "checker.c")):
        checker_info = {"type": "source_c", "path": "checker.c", "compiled": False}
    
    return {
        "success": True,
        "problem_id": problem_id,
        "checksum": checksum,
        "meta": meta,
        "testcases": testcases,
        "testcase_count": len(testcases),
        "checker": checker_info
    }

@app.get("/api/v1/problems/{problem_id}/download", dependencies=[Depends(verify_api_key)])
async def download_problem_package(problem_id: str):
    """
    下載題包為 ZIP 檔案
    """
    problem_dir = os.path.join("test_data", "problems", problem_id)
    if not os.path.exists(problem_dir):
        raise HTTPException(status_code=404, detail=f"Problem '{problem_id}' not found")
    
    # 在記憶體中建立 ZIP
    zip_buffer = io.BytesIO()
    
    with zipfile.ZipFile(zip_buffer, 'w', zipfile.ZIP_DEFLATED) as zf:
        for root, dirs, files in os.walk(problem_dir):
            for file in files:
                file_path = os.path.join(root, file)
                # 計算相對路徑（相對於題目目錄）
                arcname = os.path.relpath(file_path, problem_dir)
                zf.write(file_path, arcname)
    
    zip_buffer.seek(0)
    
    return StreamingResponse(
        zip_buffer,
        media_type="application/zip",
        headers={
            "Content-Disposition": f"attachment; filename={problem_id}.zip"
        }
    )

@app.post("/api/v1/problems/{problem_id}/upload", dependencies=[Depends(verify_api_key)])
async def upload_problem_package(
    problem_id: str,
    package: UploadFile = File(..., description="題包 ZIP"),
    force: bool = Form(False, description="強制覆蓋已存在的題包")
):
    """
    手動上傳題包（用於測試或離線環境）
    
    支援兩種題包結構：
    
    新格式（推薦，無子目錄）：
    ```
    package.zip
    ├── 0001.in
    ├── 0001.out
    ├── 0002.in
    ├── 0002.out
    ├── meta.json
    └── checker.py (可選)
    ```
    
    舊格式（有 testcase/ 子目錄，向後兼容）：
    ```
    package.zip
    ├── testcase/
    │   ├── 0001.in
    │   └── 0001.out
    └── meta.json (可選)
    ```
    """
    problem_dir = os.path.join("test_data", "problems", problem_id)
    
    if os.path.exists(problem_dir) and not force:
        raise HTTPException(
            status_code=409, 
            detail=f"Problem '{problem_id}' already exists. Use force=true to overwrite"
        )
    
    content = await package.read()
    
    try:
        # 驗證是有效的 ZIP
        with zipfile.ZipFile(io.BytesIO(content)) as z:
            names = z.namelist()
            
            # 檢查結構：新格式（根目錄有 .in 檔案）或舊格式（有 testcase/ 目錄）
            has_in_files = any(n.endswith('.in') and '/' not in n for n in names)
            has_testcase_dir = any(n.startswith("testcase/") or "/testcase/" in n for n in names)
            
            if not has_in_files and not has_testcase_dir:
                raise HTTPException(
                    status_code=400, 
                    detail="Invalid package: must contain .in files or 'testcase/' directory"
                )
            
            # 清理並建立目錄
            if os.path.exists(problem_dir):
                shutil.rmtree(problem_dir)
            os.makedirs(problem_dir, exist_ok=True)
            
            # 解壓到題目目錄
            meta_from_zip = None
            
            for member in names:
                filename = os.path.basename(member)
                if not filename:
                    continue
                
                # 處理 meta.json
                if filename == 'meta.json':
                    target_path = os.path.join(problem_dir, filename)
                    with z.open(member) as source:
                        content_bytes = source.read()
                        with open(target_path, 'wb') as target:
                            target.write(content_bytes)
                        try:
                            meta_from_zip = json.loads(content_bytes.decode('utf-8'))
                        except:
                            pass
                
                # 處理測資檔案
                elif filename.endswith('.in') or filename.endswith('.out'):
                    target_path = os.path.join(problem_dir, filename)
                    with z.open(member) as source, open(target_path, 'wb') as target:
                        target.write(source.read())
                
                # 處理 checker 檔案
                elif 'checker' in filename.lower() or (filename.endswith('.cpp') and 'checker' in member.lower()):
                    target_path = os.path.join(problem_dir, filename)
                    with z.open(member) as source, open(target_path, 'wb') as target:
                        target.write(source.read())
        
        # 計算 checksum (SHA256)
        checksum = hashlib.sha256(content).hexdigest()
        hash_file = os.path.join(problem_dir, ".checksum")
        async with aiofiles.open(hash_file, "w") as f:
            await f.write(checksum)
        
        # 統計測資 - 直接在題目根目錄
        testcases = []
        for f in sorted(os.listdir(problem_dir)):
            if f.endswith('.in'):
                stem = f[:-3]
                testcases.append({
                    "no": len(testcases) + 1,
                    "stem": stem,
                    "in": f,
                    "out": f"{stem}.out"
                })
        
        # 生成或更新 meta.json
        meta_file = os.path.join(problem_dir, "meta.json")
        
        if meta_from_zip:
            # 使用 ZIP 內的 meta.json，補充 checksum
            meta_from_zip["checksum"] = checksum
            async with aiofiles.open(meta_file, "w") as f:
                await f.write(json.dumps(meta_from_zip, indent=2))
        else:
            # 自動生成 meta.json
            meta_json = {
                "checksum": checksum,
                "tasks": [{
                    "caseCount": len(testcases),
                    "taskScore": 100,
                    "timeLimit": 2000,
                    "memoryLimit": 256
                }],
                "allow_network": False,
                "network_whitelist": [],
                "sidecar_image": None
            }
            async with aiofiles.open(meta_file, "w") as f:
                await f.write(json.dumps(meta_json, indent=2))
        
        # 編譯 checker（如果有源碼）
        checker_path = await compile_problem_checker(problem_dir)
        
        logger.info(f"Problem '{problem_id}' uploaded manually. Testcases: {len(testcases)}, Checker: {checker_path is not None}")
        
        return {
            "success": True,
            "problem_id": problem_id,
            "checksum": checksum,
            "testcase_count": len(testcases),
            "checker_compiled": checker_path is not None,
            "message": f"Problem '{problem_id}' uploaded successfully"
        }
        
    except zipfile.BadZipFile:
        raise HTTPException(status_code=400, detail="Invalid ZIP file")
    except HTTPException:
        raise
    except Exception as e:
        # 清理失敗的上傳
        if os.path.exists(problem_dir):
            shutil.rmtree(problem_dir, ignore_errors=True)
        logger.error(f"Failed to upload problem package {problem_id}: {e}")
        raise HTTPException(status_code=500, detail=f"Upload failed: {str(e)}")

@app.delete("/api/v1/problems/{problem_id}", dependencies=[Depends(verify_api_key)])
async def delete_problem(problem_id: str):
    """
    刪除本地快取的題包
    """
    problem_dir = os.path.join("test_data", "problems", problem_id)
    if not os.path.exists(problem_dir):
        raise HTTPException(status_code=404, detail=f"Problem '{problem_id}' not found")
    
    try:
        shutil.rmtree(problem_dir)
        logger.info(f"Problem '{problem_id}' deleted")
        return {"success": True, "message": f"Problem '{problem_id}' deleted successfully"}
    except Exception as e:
        raise HTTPException(status_code=500, detail=f"Failed to delete problem: {str(e)}")

@app.post("/api/v1/submissions", status_code=status.HTTP_202_ACCEPTED, dependencies=[Depends(verify_api_key)])
async def submit_job(
    submission_id: str = Form(..., description="唯一的提交 ID"),
    problem_id: int = Form(..., description="題目 ID"),
    problem_hash: str = Form(..., description="題目包 Hash"),
    mode: str = Form(..., description="模式: zip | normal", regex="^(zip|normal)$"),
    language: str = Form(..., description="程式語言: c, cpp, python, java, go"),
    
    file: UploadFile = File(..., description="程式碼檔案或 ZIP"),
    file_hash: str = Form(..., description="檔案 SHA256 Hash"),
    
    time_limit: float = Form(2.0, description="CPU 時間限制 (秒)"),
    memory_limit: int = Form(256000, description="記憶體限制 (KB)"),
    
    use_checker: bool = Form(False, description="是否使用 Checker"),
    checker_name: str = Form("diff", description="Checker 名稱 (預設: diff)"),
    
    use_static_analysis: bool = Form(False, description="是否使用靜態分析"),
    static_analysis_config: Optional[str] = Form(None, description="靜態分析設定 (e.g. --forbid-loops)"),
    
    callback_url: Optional[str] = Form(None, description="Webhook URL"),
    callback_token: Optional[str] = Form(None, description="Webhook Token"),
    priority: int = Form(10, description="優先級"),
    stdin: Optional[str] = Form(None, description="標準輸入 (Stdin)"),
    
    allow_network: bool = Form(False, description="是否允許網路連線"),
    network_whitelist: Optional[str] = Form(None, description="網路白名單 (逗號分隔 domain)"),
    
    sidecar_image: Optional[str] = Form(None, description="Sidecar 容器映像檔 (e.g. redis:alpine)"),
):
    """
    提交評測任務 (Async V2) - Aligned with User Spec
    """
    # 0. Log 接收到的請求
    logger.info(f"Received submission {submission_id}: problem={problem_id}, lang={language}, mode={mode}, priority={priority}")

    # 0.5 Ensure Problem Package
    await ensure_problem_package(str(problem_id), problem_hash)

    # 1. 建立暫存目錄
    upload_dir = f"/tmp/sandbox/submissions/{submission_id}"
    os.makedirs(upload_dir, exist_ok=True)
    
    # 2. 儲存主檔案
    file_path = os.path.join(upload_dir, file.filename)
    try:
        async with aiofiles.open(file_path, 'wb') as out_file:
            content = await file.read()
            await out_file.write(content)
    except Exception as e:
        shutil.rmtree(upload_dir, ignore_errors=True)
        raise HTTPException(status_code=500, detail=f"File upload failed: {str(e)}")

    # 3. 驗證 Hash
    sha256_hash = hashlib.sha256(content).hexdigest()
    if sha256_hash != file_hash:
        shutil.rmtree(upload_dir, ignore_errors=True)
        raise HTTPException(status_code=400, detail=f"File hash mismatch. Expected {file_hash}, got {sha256_hash}")

    # 4. 處理 Checker
    checker_path = None
    problem_dir = os.path.join("test_data", "problems", str(problem_id))
    
    if use_checker:
        # use_checker=True 表示要使用自訂 checker，必須指定 checker 名稱
        normalized_checker_name = (checker_name or "").strip().lower()
        logger.info(f"Custom Checker requested: use_checker={use_checker}, checker_name='{checker_name}', normalized='{normalized_checker_name}'")
        
        if normalized_checker_name in ("diff", ""):
            # use_checker=True 但沒有指定 checker 名稱（或只填 diff），報錯
            shutil.rmtree(upload_dir, ignore_errors=True)
            raise HTTPException(status_code=400, detail="use_checker=True requires a valid checker_name (not 'diff' or empty)")
        
        # 優先順序：1. 指定的 checker 名稱 2. 題包內編譯好的 checker 3. 題包內的 Python checker
        
        # 1. 從 checkers 目錄尋找指定名稱
        potential_path = os.path.join(CHECKERS_DIR, checker_name)
        if os.path.exists(potential_path):
            checker_path = os.path.abspath(potential_path)
        else:
            # 2. 從題包尋找已編譯的 checker
            problem_checker_compiled = os.path.join(problem_dir, "checker_compiled")
            problem_checker_py = os.path.join(problem_dir, "checker.py")
            
            if os.path.exists(problem_checker_compiled):
                checker_path = os.path.abspath(problem_checker_compiled)
                logger.info(f"Using problem's compiled checker: {checker_path}")
            elif os.path.exists(problem_checker_py):
                checker_path = os.path.abspath(problem_checker_py)
                logger.info(f"Using problem's Python checker: {checker_path}")
            else:
                # 嘗試編譯題包內的 checker
                compiled_path = await compile_problem_checker(problem_dir)
                if compiled_path:
                    checker_path = os.path.abspath(compiled_path)
                    logger.info(f"Compiled and using problem's checker: {checker_path}")
                else:
                    # Cleanup and fail
                    shutil.rmtree(upload_dir, ignore_errors=True)
                    raise HTTPException(status_code=404, detail=f"Checker '{checker_name}' not found")
    else:
        # use_checker=False 表示使用預設的 diff 比對
        checker_path = "diff"
        logger.info(f"Using default diff checker (use_checker={use_checker})")
            
    # static_analysis_config is a string, no need to save file

    # 5. 建構 Request Data (For backward compatibility / internal use)
    limits = LimitObject(
        time_limit_sec=time_limit,
        memory_limit_kb=memory_limit,
        wall_time_limit_sec=time_limit * 2.5 # Default heuristic
    )
    
    request_data = SubmissionRequest(
        submission_id=submission_id,
        code="<FILE_UPLOADED>", 
        language=language,
        priority=priority,
        limits=limits,
        callback_url=callback_url,
        callback_token=callback_token
    )

    # 6. 建立 Job
    whitelist_list = []
    if network_whitelist:
        whitelist_list = [d.strip() for d in network_whitelist.split(",") if d.strip()]

    job = Job(
        submission_id=submission_id,
        language=language,
        priority=priority,
        timestamp=time.time(),
        request_data=request_data,
        
        problem_id=str(problem_id),
        mode=mode,
        file_path=file_path,
        file_hash=file_hash,
        problem_hash=problem_hash,
        
        use_checker=True,  # 永遠使用 checker 邏輯（diff 或自訂）
        checker_file_path=checker_path,  # "diff" 或實際路徑
        
        use_static_analysis=use_static_analysis,
        static_analysis_config=static_analysis_config,
        
        stdin=stdin,
        allow_network=allow_network,
        network_whitelist=whitelist_list,
        sidecar_image=sidecar_image,
        is_selftest=False  # 正式提交，非 selftest
    )

    # 7. 推入隊列
    try:
        await job_queue.push(job)
        logger.info(f"Job {submission_id} added to queue (Position: {job_queue.size()})")
    except asyncio.QueueFull:
        logger.warning(f"Job queue full, rejected {submission_id}")
        # Cleanup on failure
        shutil.rmtree(upload_dir, ignore_errors=True)
        raise HTTPException(
            status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
            detail="系統忙碌中，請稍後再試 (Queue Full)"
        )

    return {
        "success": True,
        "submission_id": submission_id,
        "status": "queued",
        "queue_position": job_queue.size()
    }

@app.post("/api/v1/selftest-submissions", status_code=status.HTTP_202_ACCEPTED, dependencies=[Depends(verify_api_key)])
async def submit_selftest_job(
    submission_id: str = Form(..., description="唯一的提交 ID"),
    language: str = Form(..., description="程式語言: c, cpp, python, java, go"),
    
    file: UploadFile = File(..., description="程式碼檔案或 ZIP"),
    file_hash: str = Form(..., description="檔案 SHA256 Hash"),
    
    # 自定義測試核心：stdin 輸入
    stdin: Optional[str] = Form(None, description="標準輸入 (Stdin) - 自定義測試的輸入"),
    
    # 可選的題目相關參數（如果需要用題包測資）
    problem_id: Optional[int] = Form(None, description="題目 ID (可選，若提供則使用題包測資)"),
    problem_hash: Optional[str] = Form(None, description="題目包 Hash (可選)"),
    mode: str = Form("normal", description="模式: zip | normal", regex="^(zip|normal)$"),
    
    time_limit: float = Form(2.0, description="CPU 時間限制 (秒)"),
    memory_limit: int = Form(256000, description="記憶體限制 (KB)"),
    
    use_checker: bool = Form(False, description="是否使用 Checker"),
    checker_name: str = Form("diff", description="Checker 名稱 (預設: diff)"),
    
    use_static_analysis: bool = Form(False, description="是否使用靜態分析"),
    static_analysis_config: Optional[str] = Form(None, description="靜態分析設定 (e.g. --forbid-loops)"),
    
    callback_url: Optional[str] = Form(None, description="Webhook URL"),
    callback_token: Optional[str] = Form(None, description="Webhook Token"),
    priority: int = Form(-1, description="優先級 (預設 -1，自定義測試優先)"),
    
    allow_network: bool = Form(False, description="是否允許網路連線"),
    network_whitelist: Optional[str] = Form(None, description="網路白名單 (逗號分隔 domain)"),
    
    sidecar_image: Optional[str] = Form(None, description="Sidecar 容器映像檔 (e.g. redis:alpine)"),
):
    """
    學生自行測試提交 (Async)
    
    兩種使用模式：
    1. 純自定義測試：只提供 stdin，不需要 problem_id，直接執行並返回 stdout/stderr
    2. 題包測試：提供 problem_id 和 problem_hash，使用題包內的測資進行評測
    """
    # 0. Log 接收到的請求
    logger.info(f"Received selftest submission {submission_id}: problem={problem_id}, lang={language}, mode={mode}, priority={priority}")

    # 0.5 Selftest 永遠走單一執行模式，不需要題包
    # 即使後端傳了 problem_id 和 problem_hash，也直接忽略
    # 這樣可以避免因為題包不存在而報錯
    logger.info(f"Selftest mode: ignoring problem_id/problem_hash, will run single execution mode")
    if not stdin:
        logger.info(f"Selftest {submission_id} has no stdin input, will run with empty input")

    # 1. 建立暫存目錄
    upload_dir = f"/tmp/sandbox/submissions/{submission_id}"
    os.makedirs(upload_dir, exist_ok=True)
    
    # 2. 儲存主檔案
    file_path = os.path.join(upload_dir, file.filename)
    try:
        async with aiofiles.open(file_path, 'wb') as out_file:
            content = await file.read()
            await out_file.write(content)
    except Exception as e:
        shutil.rmtree(upload_dir, ignore_errors=True)
        raise HTTPException(status_code=500, detail=f"File upload failed: {str(e)}")

    # 3. 驗證 Hash
    sha256_hash = hashlib.sha256(content).hexdigest()
    if sha256_hash != file_hash:
        shutil.rmtree(upload_dir, ignore_errors=True)
        raise HTTPException(status_code=400, detail=f"File hash mismatch. Expected {file_hash}, got {sha256_hash}")

    # 4. Selftest 不需要 Checker，直接回傳 stdout/stderr
    checker_path = None
    logger.info(f"Selftest mode: No checker needed, will return stdout/stderr directly")
            
    # static_analysis_config is a string, no need to save file

    # 5. 建構 Request Data (For backward compatibility / internal use)
    limits = LimitObject(
        time_limit_sec=time_limit,
        memory_limit_kb=memory_limit,
        wall_time_limit_sec=time_limit * 2.5 # Default heuristic
    )
    
    request_data = SubmissionRequest(
        submission_id=submission_id,
        code="<FILE_UPLOADED>", 
        language=language,
        priority=priority,
        limits=limits,
        callback_url=callback_url,
        callback_token=callback_token
    )

    # 6. 建立 Job
    whitelist_list = []
    if network_whitelist:
        whitelist_list = [d.strip() for d in network_whitelist.split(",") if d.strip()]

    job = Job(
        submission_id=submission_id,
        language=language,
        priority=priority,
        timestamp=time.time(),
        request_data=request_data,
        
        problem_id=str(problem_id) if problem_id else None,  # 可能沒有 problem_id
        mode=mode,
        file_path=file_path,
        file_hash=file_hash,
        problem_hash=problem_hash if problem_hash else None,
        
        use_checker=checker_path is not None,  # 純自定義測試時不使用 checker
        checker_file_path=checker_path,  # None, "diff" 或實際路徑
        
        use_static_analysis=use_static_analysis,
        static_analysis_config=static_analysis_config,
        
        stdin=stdin,
        allow_network=allow_network,
        network_whitelist=whitelist_list,
        sidecar_image=sidecar_image,
        is_selftest=True  # 自定義測試，callback 格式會不同
    )

    # 7. 推入隊列
    try:
        await job_queue.push(job)
        logger.info(f"Selftest job {submission_id} added to queue (Position: {job_queue.size()})")
    except asyncio.QueueFull:
        logger.warning(f"Job queue full, rejected {submission_id}")
        # Cleanup on failure
        shutil.rmtree(upload_dir, ignore_errors=True)
        raise HTTPException(
            status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
            detail="系統忙碌中，請稍後再試 (Queue Full)"
        )

    return {
        "success": True,
        "submission_id": submission_id,
        "status": "queued",
        "queue_position": job_queue.size()
    }

@app.post("/api/v1/test_local/{problem_id}", dependencies=[Depends(verify_api_key)])
async def test_local_problem(problem_id: str):
    """
    測試本地題目 (Dev Only)
    讀取 test_data/problems/{problem_id} 下的 solution.py 與 test cases
    """
    problem_dir = os.path.join("test_data", "problems", problem_id)
    if not os.path.exists(problem_dir):
        raise HTTPException(status_code=404, detail=f"Problem {problem_id} not found")

    # Detect solution file and language
    solution_files = {
        "solution.py": "python",
        "solution.c": "c",
        "solution.cpp": "cpp",
        "solution.java": "java",
        "solution.go": "go"
    }
    
    solution_path = None
    language = "python" # Default
    
    for fname, lang in solution_files.items():
        path = os.path.join(problem_dir, fname)
        if os.path.exists(path):
            solution_path = path
            language = lang
            break
            
    if not solution_path:
        raise HTTPException(status_code=404, detail="No valid solution file (py/c/cpp/java/go) found")

    # Read solution code
    async with aiofiles.open(solution_path, "r", encoding="utf-8") as f:
        code_content = await f.read()
    
    # Read meta.json for extra config (Network/Sidecar)
    meta_path = os.path.join(problem_dir, "meta.json")
    allow_network = False
    network_whitelist = []
    sidecar_image = None
    
    if os.path.exists(meta_path):
        try:
            async with aiofiles.open(meta_path, "r") as f:
                meta = json.loads(await f.read())
                allow_network = meta.get("allow_network", False)
                network_whitelist = meta.get("network_whitelist", [])
                sidecar_image = meta.get("sidecar_image", None)
        except Exception as e:
            logger.warning(f"Failed to parse meta.json for {problem_id}: {e}")

    # Create ONE job for the whole problem (Multi-case mode)
    submission_id = f"test-{problem_id}-{int(time.time())}-{uuid.uuid4().hex[:8]}"
    
    upload_dir = f"/tmp/sandbox/submissions/{submission_id}"
    os.makedirs(upload_dir, exist_ok=True)
    
    # Determine extension
    ext_map = {"python": ".py", "c": ".c", "cpp": ".cpp", "java": ".java", "go": ".go"}
    ext = ext_map.get(language, ".txt")
    file_path = os.path.join(upload_dir, f"solution{ext}")
    
    async with aiofiles.open(file_path, "w", encoding="utf-8") as f:
        await f.write(code_content)
        
    file_hash = hashlib.sha256(code_content.encode()).hexdigest()
    
    limits = LimitObject(
        time_limit_sec=2.0,
        memory_limit_kb=256000,
        wall_time_limit_sec=5.0
    )
    
    request_data = SubmissionRequest(
        submission_id=submission_id,
        code=code_content,
        language=language,
        priority=10,
        limits=limits
    )
    
    job = Job(
        submission_id=submission_id,
        language=language,
        priority=10,
        timestamp=time.time(),
        request_data=request_data,
        problem_id=problem_id,
        mode="normal",
        file_path=file_path,
        file_hash=file_hash,
        problem_hash="local-dev",
        stdin=None, # None triggers multi-case mode in worker
        expected_output=None,
        allow_network=allow_network,
        network_whitelist=network_whitelist,
        sidecar_image=sidecar_image
    )
    
    await job_queue.push(job)
        
    return {
        "message": f"Triggered 1 submission for {problem_id}",
        "submission_ids": [submission_id]
    }

@app.get("/api/v1/auth-history", dependencies=[Depends(verify_api_key)])
async def get_auth_history(limit: int = 100):
    """獲取認證失敗紀錄"""
    history = []
    if not os.path.exists(AUTH_LOG_FILE):
        return []
        
    try:
        async with aiofiles.open(AUTH_LOG_FILE, "r") as f:
            lines = await f.readlines()
            # Get last N lines
            for line in lines[-limit:]:
                try:
                    history.append(json.loads(line))
                except json.JSONDecodeError:
                    continue
        return list(reversed(history)) # Newest first
    except Exception as e:
        logger.error(f"Error reading auth history: {e}")
        return []

@app.get("/api/v1/history", dependencies=[Depends(verify_api_key)])
async def get_history(limit: int = 1000):
    """
    從 Log 檔案獲取歷史紀錄
    """
    history = []
    log_path = os.path.join("logs", "submission_history.jsonl")
    
    if os.path.exists(log_path):
        try:
            # 讀取所有行並反轉 (最新的在最前)
            async with aiofiles.open(log_path, "r", encoding="utf-8") as f:
                lines = await f.readlines()
                
            # 如果 limit 為 -1，則回傳全部
            target_lines = reversed(lines)
            
            count = 0
            for line in target_lines:
                if limit != -1 and count >= limit:
                    break
                try:
                    if line.strip():
                        history.append(json.loads(line))
                        count += 1
                except json.JSONDecodeError:
                    continue
        except Exception as e:
            logger.error(f"Error reading history log: {e}")
            
    return history

@app.get("/api/v1/submissions/{submission_id}")
async def get_submission_result(submission_id: str):
    """
    查詢特定提交的結果
    """
    # 1. 搜尋歷史紀錄 (已完成的任務)
    for job in worker_manager.history:
        if job["submission_id"] == submission_id:
            return job
    
    # 2. 搜尋正在執行的任務
    for worker in worker_manager.workers:
        if worker.current_job and worker.current_job.submission_id == submission_id:
            return {
                "submission_id": submission_id,
                "status": "processing",
                "message": "Job is currently running"
            }
            
    # 3. 搜尋佇列中的任務 (尚未開始)
    # 這邊需要遍歷 queue，但 asyncio.Queue 不容易遍歷，暫時略過或假設在 queue 中
    # 簡單回傳 404 代表 "Not Found in History or Active" -> 可能還在 Queue 或 ID 錯誤
    
    raise HTTPException(status_code=404, detail="Submission not found (might be queued or invalid ID)")

from fastapi.concurrency import run_in_threadpool

# ...existing code...

def get_docker_stats_sync():
    """Synchronous function to get Docker stats"""
    try:
        # Use standard from_env which usually works best if environment is set correctly
        # If unix socket is needed, it should be handled by env vars or default
        client = docker.from_env(timeout=3)
        stats_data = []
        
        containers = client.containers.list()
        target_containers = [c for c in containers if "sandbox" in c.name or "static-analysis" in c.name]
        
        for container in target_containers:
            try:
                stats = container.stats(stream=False)
                
                # Calculate CPU
                cpu_delta = stats['cpu_stats']['cpu_usage']['total_usage'] - stats['precpu_stats']['cpu_usage']['total_usage']
                system_cpu_usage = stats['cpu_stats'].get('system_cpu_usage')
                precpu_system_cpu_usage = stats['precpu_stats'].get('system_cpu_usage')
                
                cpu_percent = 0.0
                if system_cpu_usage and precpu_system_cpu_usage:
                    system_delta = system_cpu_usage - precpu_system_cpu_usage
                    if system_delta > 0 and cpu_delta > 0:
                        online_cpus = stats['cpu_stats'].get('online_cpus', 1)
                        cpu_percent = (cpu_delta / system_delta) * online_cpus * 100.0
                
                # Calculate Memory
                mem_usage = stats['memory_stats'].get('usage', 0)
                mem_limit = stats['memory_stats'].get('limit', 1)
                mem_percent = (mem_usage / mem_limit) * 100.0
                
                stats_data.append({
                    "name": container.name,
                    "id": container.short_id,
                    "cpu_percent": round(cpu_percent, 2),
                    "memory_usage_mb": round(mem_usage / (1024 * 1024), 2),
                    "memory_limit_mb": round(mem_limit / (1024 * 1024), 2),
                    "memory_percent": round(mem_percent, 2)
                })
            except Exception as e:
                logger.error(f"Error getting stats for {container.name}: {e}")
                continue
        
        return {"containers": stats_data}
    except Exception as e:
        logger.error(f"Docker stats error: {e}")
        return {"containers": [], "error": str(e)}

@app.get("/api/v1/system_stats", dependencies=[Depends(verify_api_key)])
async def get_system_stats():
    """
    獲取 Docker 容器資源使用狀態 (Non-blocking)
    """
    return await run_in_threadpool(get_docker_stats_sync)

@app.get("/health")
async def health():
    status_data = worker_manager.get_status()
    
    # 檢查 LLM judge 狀態
    llm_judge = get_llm_judge()
    llm_available = llm_judge.is_available()
    
    return {
        "status": "ok", 
        "queue_size": job_queue.size(),
        "queue_paused": job_queue.is_paused(),
        "service": "sandbox-isolate-api-v2",
        "workers": status_data,
        "llm_judge": {
            "available": llm_available,
            "model": os.getenv("LLM_JUDGE_MODEL", "gpt-4o-mini") if llm_available else None
        }
    }

@app.post("/api/v1/queue/pause", dependencies=[Depends(verify_api_key)])
async def pause_queue():
    """暫停隊列處理"""
    job_queue.pause()
    return {"success": True, "message": "Queue paused", "queue_paused": True}

@app.post("/api/v1/queue/resume", dependencies=[Depends(verify_api_key)])
async def resume_queue():
    """恢復隊列處理"""
    job_queue.resume()
    return {"success": True, "message": "Queue resumed", "queue_paused": False}

@app.post("/api/v1/queue/clear", dependencies=[Depends(verify_api_key)])
async def clear_queue():
    """清空隊列"""
    count = job_queue.clear()
    return {"success": True, "message": f"Queue cleared. Removed {count} jobs.", "removed_count": count}

@app.get("/dashboard", response_class=HTMLResponse)
async def dashboard():
    """即時監控儀表板"""
    if os.path.exists("dashboard.html"):
        with open("dashboard.html", "r", encoding="utf-8") as f:
            return f.read()
    return "Dashboard not found"

@app.get("/logs/view", response_class=HTMLResponse)
async def view_logs():
    """即時 Log 檢視頁面"""
    html_content = """
    <!DOCTYPE html>
    <html>
    <head>
        <title>Sandbox Live Logs</title>
        <meta charset="utf-8">
        <style>
            body { background-color: #1e1e1e; color: #d4d4d4; font-family: 'Consolas', 'Monaco', monospace; margin: 0; padding: 20px; font-size: 14px; }
            h1 { font-size: 18px; margin-bottom: 10px; color: #569cd6; }
            #controls { margin-bottom: 10px; position: sticky; top: 0; background: #1e1e1e; padding: 10px 0; border-bottom: 1px solid #333; }
            button { background: #0e639c; color: white; border: none; padding: 5px 10px; cursor: pointer; }
            button:hover { background: #1177bb; }
            #log-container { white-space: pre-wrap; word-wrap: break-word; }
            .error { color: #f48771; }
            .info { color: #d4d4d4; }
            .warning { color: #cca700; }
        </style>
    </head>
    <body>
        <div id="controls">
            <h1>Sandbox Live Logs</h1>
            <button onclick="fetchLogs()">Refresh Now</button>
            <span id="status" style="margin-left: 10px; font-size: 12px; color: #888;">Auto-refreshing every 2s...</span>
            <div id="debug-info" style="font-size: 10px; color: #666; margin-top: 5px;"></div>
        </div>
        <div id="log-container">Loading...</div>
        <script>
            const logContainer = document.getElementById('log-container');
            const debugInfo = document.getElementById('debug-info');
            let isScrolledToBottom = true;

            window.onscroll = function() {
                isScrolledToBottom = (window.innerHeight + window.scrollY) >= document.body.offsetHeight - 50;
            };

            async function fetchLogs() {
                try {
                    const response = await fetch('/logs/raw?t=' + new Date().getTime());
                    if (!response.ok) throw new Error("Failed to fetch: " + response.status);
                    const text = await response.text();
                    logContainer.textContent = text;
                    
                    if (isScrolledToBottom) {
                        window.scrollTo(0, document.body.scrollHeight);
                    }
                    document.getElementById('status').textContent = "Last updated: " + new Date().toLocaleTimeString();
                    debugInfo.textContent = "Length: " + text.length + " chars";
                } catch (e) {
                    document.getElementById('status').textContent = "Error: " + e.message;
                }
            }
            
            // Initial load
            fetchLogs();
            // Auto refresh
            setInterval(fetchLogs, 2000);
        </script>
    </body>
    </html>
    """
    return html_content

@app.get("/logs/raw")
async def get_raw_logs():
    """獲取原始 Log 內容"""
    abs_log_file = os.path.abspath(log_file)
    if not os.path.exists(abs_log_file):
        # Debug info if file not found
        return f"Log file not found at {abs_log_file}.\nCWD: {os.getcwd()}\nLS logs/: {os.listdir('logs') if os.path.exists('logs') else 'logs dir missing'}"
    
    try:
        # Use standard open to ensure we get content and avoid async locking issues if any
        with open(abs_log_file, mode='r', encoding='utf-8', errors='ignore') as f:
            content = f.read()
            if not content:
                return f"Log file exists at {abs_log_file} but is empty."
            return content
    except Exception as e:
        return f"Error reading log file: {str(e)}"

@app.get("/")
async def root():
    return {
        "service": app.title, 
        "version": app.version,
        "endpoints": ["/health", "/api/v1/submissions", "/dashboard"]
    }