AICrossSim · gwjm · Jun 5, 2026 · Jun 11, 2026
diff --git a/benchmark_all.sh b/benchmark_all.sh
@@ -0,0 +1,132 @@
+#!/bin/bash
+# Benchmarking script for Fast-dLLM models using the PLENA eval_dllm harness.
+# Streams output to the terminal; no per-run log files are retained.
+# Only the tabulated results (all_results.jsonl) persist.
+
+export TRANSFORMERS_OFFLINE=1
+export PYTHONPATH=$PYTHONPATH:/home/gm1425/Fast-dLLM/PLENA_Software/.uv_cache/git-v0/checkouts/c53f323a805b777e/cb963b8/src/
+
+LOG_DIR="logs/benchmarks"
+mkdir -p $LOG_DIR
+
+MODELS=(
+    "Efficient-Large-Model/Fast_dLLM_v2_7B"
+)
+
+# Sweep dimensions are env-overridable (space-separated) so the sweep can be
+# sharded across GPUs, e.g.:
+#   GPU=1 BENCH_BL="8"  BENCH_BS="64 128" RESULTS_FILE=logs/benchmarks/gpu1.jsonl bash benchmark_all.sh
+# Larger batches (64/128) saturate compute on the eager-quantized configs;
+# memory is dominated by the ~15GB of weights and barely grows with batch.
+BLOCK_LENGTHS=(${BENCH_BL:-8 16 32})
+BATCH_SIZES=(${BENCH_BS:-16 32})
+
+# Active sweep: configs 1-3 run with no external prerequisites.
+#   01 fp16          — baseline (no quant_config)
+#   02 w4 RTN        — weight-only INT4 (Linear quant)
+#   03 w4+act4+kv4   — adds self_attn module quant (qk/av/kv-cache INT4).
+#                      Needs the Fast_dLLM_QwenAttentionMXInt support added to
+#                      chop (chop/nn/quantized/modules/fast_dllm_qwen).
+#
+# Configs 04-06 are GPTQ and are DEFERRED — they require, in addition to the
+# chop support above:
+#   - calib/Qwen_Qwen3-8B_gsm8k_n64_s1024.pt  (Step 0 calibrate; see
+#     plena_experiments/table9/README.md), which needs Qwen/Qwen3-8B (online).
+#   - a writable gptq.checkpoint_dir (the TOMLs point at /data/models/cx922,
+#     owned by another user — override to a local path before enabling).
+#   - config 06 additionally reuses config 05's rotation-search checkpoint.
+# Move them into CONFIGS once those prerequisites are in place.
+CONFIGS=(
+    # "plena_experiments/table9/configs/gsm8k/01_fp16.toml"
+    # "plena_experiments/table9/configs/gsm8k/02_w4_rtn.toml"
+    # "plena_experiments/table9/configs/gsm8k/03_w4_act4_kv4_rtn.toml"
+    # "plena_experiments/table9/configs/gsm8k/04_w4_act4_kv4_gptq.toml"
+    # "plena_experiments/table9/configs/gsm8k/05_w4_act4_kv4_gptq_erryclip.toml"
+    "plena_experiments/table9/configs/gsm8k/06_w4_act4_kv4_gptq_erryclip_selrot.toml"
+)
+# Override the config set via BENCH_CONFIGS (space-separated TOML paths).
+if [ -n "${BENCH_CONFIGS:-}" ]; then
+    CONFIGS=(${BENCH_CONFIGS})
+fi
+
+RESULTS_FILE="${RESULTS_FILE:-$LOG_DIR/all_results.jsonl}"
+if [ ! -f "$RESULTS_FILE" ]; then
+    echo "[]" > "$RESULTS_FILE"
+fi
+
+DEFAULT_PY=$([[ -x .venv/bin/python ]] && echo .venv/bin/python || echo python)
+PY=${PY:-$DEFAULT_PY}
+
+for MODEL in "${MODELS[@]}"; do
+    for CONFIG in "${CONFIGS[@]}"; do
+        for BL in "${BLOCK_LENGTHS[@]}"; do
+            for BS in "${BATCH_SIZES[@]}"; do
+                MODEL_NAME=$(basename "$MODEL")
+                CONFIG_NAME=$(basename "$CONFIG" .toml)
+                RUN_ID="${MODEL_NAME}_${CONFIG_NAME}_BL${BL}_BS${BS}"
+
+                if grep -q "\"run_id\": \"$RUN_ID\"" "$RESULTS_FILE"; then
+                    echo "Skipping $RUN_ID (already exists)..."
+                    continue
+                fi
+
+                echo "Benchmarking: $MODEL_NAME config=$CONFIG_NAME BL=$BL BS=$BS..."
+
+                # Build args for PLENA eval_dllm jsonargparse CLI.
+                # No --log_dir: skip persistent per-run args.json/results.json.
+                ARGS=(
+                    --model_name "$MODEL"
+                    --tasks gsm8k
+                    --device_id cuda:0
+                    --batch_size "$BS"
+                    --bd_size "$BL"
+                    --small_block_size 8
+                    --max_new_tokens 512
+                    --threshold 1.0
+                    --temperature 0.0
+                    --num_fewshot 0
+                    --show_speed true
+                )
+
+                # Baseline fp16 runs: no quant_config; quantized runs: pass config
+                if [[ "$CONFIG_NAME" != *fp16* ]]; then
+                    ARGS+=(--quant_config "$CONFIG")
+                fi
+
+                # Stream to terminal while capturing to a temp file for metric
+                # extraction; the temp file is deleted once results are tabulated.
+                TMP_LOG=$(mktemp)
+                CUDA_VISIBLE_DEVICES=2 $PY -m quant_eval.cli.eval_dllm "${ARGS[@]}" \
+                    2>&1 | tee "$TMP_LOG"
+
+                # gsm8k reports two exact_match scores: "strict-match" (needs a
+                # literal "#### <n>", which our \boxed{} prompt never produces ->
+                # always ~0) and "flexible-extract" (last number in the output ->
+                # the real score). Prefer flexible-extract; fall back to the last
+                # exact_match line for tasks (e.g. minerva_math) with no filters.
+                ACCURACY=$(grep -oP 'exact_match,flexible-extract: \K[0-9.]+' "$TMP_LOG" | head -n 1)
+                [ -z "$ACCURACY" ] && ACCURACY=$(grep -oP 'exact_match[^,]*?: \K[0-9.]+' "$TMP_LOG" | tail -n 1)
+                TOKENS_PER_SEC=$(grep -oP 'Tokens/s: \K[0-9.]+' "$TMP_LOG" | tail -n 1)
+                TTFT_MS=$(grep -oP 'TTFT: \K[0-9.]+' "$TMP_LOG" | tail -n 1)
+                TOTAL_TIME=$(grep -oP 'Total workload time: \K[0-9.]+' "$TMP_LOG" | tail -n 1)
+
+                rm -f "$TMP_LOG"
+
+                # Treat a run as failed (crash/OOM) if no speed line was emitted.
+                # Don't record failed runs, so the skip-check retries them later.
+                if [ -z "$TOKENS_PER_SEC" ]; then
+                    echo "FAILED $RUN_ID — no result (crash/incomplete); will retry on rerun"
+                    continue
+                fi
+
+                [ -z "$ACCURACY" ] && ACCURACY=0
+                [ -z "$TTFT_MS" ] && TTFT_MS=0
+                [ -z "$TOTAL_TIME" ] && TOTAL_TIME=0
+
+                echo "{\"run_id\": \"$RUN_ID\", \"model\": \"$MODEL_NAME\", \"config\": \"$CONFIG_NAME\", \"block_length\": $BL, \"batch_size\": $BS, \"accuracy\": $ACCURACY, \"tokens_per_sec\": $TOKENS_PER_SEC, \"ttft_ms\": $TTFT_MS, \"total_time\": $TOTAL_TIME}" >> "$RESULTS_FILE"
+
+                echo "Done $RUN_ID — Acc: $ACCURACY  Tps: $TOKENS_PER_SEC  TTFT: ${TTFT_MS}ms"
+            done
+        done
+    done
+done
diff --git a/benchmark_memory.sh b/benchmark_memory.sh
@@ -0,0 +1,158 @@
+#!/bin/bash
+# Memory profiling for Fast-dLLM models using the PLENA profile_memory CLI.
+#
+# For each quant config the model is loaded ONCE, then a single generation
+# batch is run for every (block_length, batch_size) cell to record:
+#   * model_footprint_mb — resident model weights (CUDA allocated after load)
+#   * peak_memory_mb     — peak CUDA allocation while generating one batch
+#
+# Results are merged into the SAME all_results.jsonl produced by
+# benchmark_all.sh, matched by run_id (latency/accuracy fields are preserved;
+# the memory fields are added). Cells with no existing run_id are appended.
+
+export TRANSFORMERS_OFFLINE=1
+export PYTHONPATH=$PYTHONPATH:/home/gm1425/Fast-dLLM/PLENA_Software/.uv_cache/git-v0/checkouts/c53f323a805b777e/cb963b8/src/
+
+LOG_DIR="logs/benchmarks"
+mkdir -p $LOG_DIR
+
+MODELS=(
+    "Efficient-Large-Model/Fast_dLLM_v2_7B"
+)
+
+BLOCK_LENGTHS=(8 16 32 64)
+BATCH_SIZES=(1 4 8 16 32)
+
+CONFIGS=(
+    "plena_experiments/table9/configs/gsm8k/01_fp16.toml"
+    "plena_experiments/table9/configs/gsm8k/02_w4_rtn.toml"
+    "plena_experiments/table9/configs/gsm8k/03_w4_act4_kv4_rtn.toml"
+    "plena_experiments/table9/configs/gsm8k/04_w4_act4_kv4_gptq.toml"
+    "plena_experiments/table9/configs/gsm8k/05_w4_act4_kv4_gptq_erryclip.toml"
+    "plena_experiments/table9/configs/gsm8k/06_w4_act4_kv4_gptq_erryclip_selrot.toml"
+)
+
+RESULTS_FILE="$LOG_DIR/all_results.jsonl"
+if [ ! -f "$RESULTS_FILE" ]; then
+    echo "[]" > "$RESULTS_FILE"
+fi
+
+DEFAULT_PY=$([[ -x .venv/bin/python ]] && echo .venv/bin/python || echo python)
+PY=${PY:-$DEFAULT_PY}
+
+# JSON list literals for the profiler CLI (jsonargparse).
+BL_JSON=$(printf '%s,' "${BLOCK_LENGTHS[@]}"); BL_JSON="[${BL_JSON%,}]"
+BS_JSON=$(printf '%s,' "${BATCH_SIZES[@]}");   BS_JSON="[${BS_JSON%,}]"
+
+for MODEL in "${MODELS[@]}"; do
+    for CONFIG in "${CONFIGS[@]}"; do
+        MODEL_NAME=$(basename "$MODEL")
+        CONFIG_NAME=$(basename "$CONFIG" .toml)
+
+        echo "Profiling memory: $MODEL_NAME config=$CONFIG_NAME (single batch per cell)..."
+
+        ARGS=(
+            --model_name "$MODEL"
+            --device_id cuda:0
+            --bd_sizes "$BL_JSON"
+            --batch_sizes "$BS_JSON"
+            --small_block_size 8
+            --max_new_tokens 512
+            --threshold 1.0
+            --temperature 0.0
+        )
+        if [[ "$CONFIG_NAME" != *fp16* ]]; then
+            ARGS+=(--quant_config "$CONFIG")
+        fi
+
+        TMP_LOG=$(mktemp)
+        CUDA_VISIBLE_DEVICES=3 $PY -m quant_eval.cli.profile_memory "${ARGS[@]}" \
+            2>&1 | tee "$TMP_LOG"
+
+        if ! grep -q "MEM_RESULT" "$TMP_LOG"; then
+            echo "FAILED $MODEL_NAME/$CONFIG_NAME — no MEM_RESULT lines (crash/incomplete)"
+            rm -f "$TMP_LOG"
+            continue
+        fi
+
+        # Merge each MEM_RESULT cell into all_results.jsonl by run_id.
+        "$PY" - "$RESULTS_FILE" "$TMP_LOG" "$MODEL_NAME" "$CONFIG_NAME" <<'PYEOF'
+import json, re, sys
+
+results_path, log_path, model_name, config_name = sys.argv[1:5]
+
+# Parse memory cells from the profiler log.
+cells = {}  # run_id -> (footprint_mb, peak_mb, peak_reserved_mb, bl, bs)
+pat = re.compile(
+    r"MEM_RESULT\|bd=(\d+)\|bs=(\d+)\|cache=(true|false)\|footprint_mb=([\d.]+)"
+    r"\|peak_mb=([\d.]+)\|peak_reserved_mb=([\d.]+)"
+)
+with open(log_path) as f:
+    for line in f:
+        m = pat.search(line)
+        if not m:
+            continue
+        bl, bs, cache = int(m.group(1)), int(m.group(2)), m.group(3)
+        # run_id must match benchmark_all.sh exactly: ..._BL{bl}_BS{bs}_CACHE{cache}
+        run_id = f"{model_name}_{config_name}_BL{bl}_BS{bs}_CACHE{cache}"
+        cells[run_id] = {
+            "block_length": bl,
+            "batch_size": bs,
+            "use_block_cache": cache == "true",
+            "model_footprint_mb": round(float(m.group(4)), 2),
+            "peak_memory_mb": round(float(m.group(5)), 2),
+            "peak_reserved_mb": round(float(m.group(6)), 2),
+        }
+
+# Read existing lines, updating matching records in place.
+out_lines = []
+seen = set()
+with open(results_path) as f:
+    for raw in f:
+        s = raw.rstrip("\n")
+        if not s.strip():
+            continue
+        try:
+            obj = json.loads(s)
+        except json.JSONDecodeError:
+            out_lines.append(s)
+            continue
+        if isinstance(obj, dict) and obj.get("run_id") in cells:
+            rid = obj["run_id"]
+            obj.update(
+                model_footprint_mb=cells[rid]["model_footprint_mb"],
+                peak_memory_mb=cells[rid]["peak_memory_mb"],
+                peak_reserved_mb=cells[rid]["peak_reserved_mb"],
+            )
+            seen.add(rid)
+            out_lines.append(json.dumps(obj))
+        else:
+            out_lines.append(s)
+
+# Append cells that had no existing latency/accuracy record.
+for rid, c in cells.items():
+    if rid in seen:
+        continue
+    out_lines.append(json.dumps({
+        "run_id": rid,
+        "model": model_name,
+        "config": config_name,
+        "block_length": c["block_length"],
+        "batch_size": c["batch_size"],
+        "use_block_cache": c["use_block_cache"],
+        "model_footprint_mb": c["model_footprint_mb"],
+        "peak_memory_mb": c["peak_memory_mb"],
+        "peak_reserved_mb": c["peak_reserved_mb"],
+    }))
+
+with open(results_path, "w") as f:
+    f.write("\n".join(out_lines) + "\n")
+
+print(f"Merged {len(cells)} memory cells "
+      f"({len(seen)} updated, {len(cells) - len(seen)} appended) for {config_name}.")
+PYEOF
+
+        rm -f "$TMP_LOG"
+        echo "Done memory profiling $MODEL_NAME/$CONFIG_NAME."
+    done
+done
diff --git a/plena_experiments/table9/configs/gsm8k/04_w4_act4_kv4_gptq.toml b/plena_experiments/table9/configs/gsm8k/04_w4_act4_kv4_gptq.toml
@@ -6,15 +6,15 @@
 by = "regex_name"
 
 [gptq]
-model_name        = "Qwen/Qwen3-8B"
-dataset           = "file:calib/Qwen_Qwen3-8B_gsm8k_n64_s1024.pt"
+model_name        = "Efficient-Large-Model/Fast_dLLM_v2_7B"
+dataset           = "file:calib/Fast_dLLM_v2_7B_gsm8k_n64_s1024.pt"
 nsamples          = 64
 seqlen            = 1024
 format            = "mxint"
 quantile_search   = true
 clip_search_y     = false
 cali_batch_size   = 32
-checkpoint_dir    = "/data/models/cx922/plena_camera/gsm8k_04_w4_act4_kv4_gptq"
+checkpoint_dir    = "/data/models/gm1425/plena_gptq/fastdllm_gsm8k_04_w4_act4_kv4_gptq"
 
 [gptq.weight_config]
 weight_block_size = 16

diff --git a/plena_experiments/table9/configs/gsm8k/05_w4_act4_kv4_gptq_erryclip.toml b/plena_experiments/table9/configs/gsm8k/05_w4_act4_kv4_gptq_erryclip.toml
@@ -6,15 +6,15 @@
 by = "regex_name"
 
 [gptq]
-model_name        = "Qwen/Qwen3-8B"
-dataset           = "file:calib/Qwen_Qwen3-8B_gsm8k_n64_s1024.pt"
+model_name        = "Efficient-Large-Model/Fast_dLLM_v2_7B"
+dataset           = "file:calib/Fast_dLLM_v2_7B_gsm8k_n64_s1024.pt"
 nsamples          = 64
 seqlen            = 1024
 format            = "mxint"
 quantile_search   = true
 clip_search_y     = true       # ← the only diff from 04
 cali_batch_size   = 32
-checkpoint_dir    = "/data/models/cx922/plena_camera/gsm8k_05_w4_act4_kv4_gptq_erryclip"
+checkpoint_dir    = "/data/models/gm1425/plena_gptq/fastdllm_gsm8k_05_w4_act4_kv4_gptq_erryclip"
 
 [gptq.weight_config]
 weight_block_size = 16

diff --git a/plena_experiments/table9/configs/gsm8k/06_w4_act4_kv4_gptq_erryclip_selrot.toml b/plena_experiments/table9/configs/gsm8k/06_w4_act4_kv4_gptq_erryclip_selrot.toml
@@ -11,15 +11,15 @@
 by = "regex_name"
 
 [gptq]
-model_name        = "Qwen/Qwen3-8B"
-dataset           = "file:calib/Qwen_Qwen3-8B_gsm8k_n64_s1024.pt"
+model_name        = "Efficient-Large-Model/Fast_dLLM_v2_7B"
+dataset           = "file:calib/Fast_dLLM_v2_7B_gsm8k_n64_s1024.pt"
 nsamples          = 64
 seqlen            = 1024
 format            = "mxint"
 quantile_search   = true
 clip_search_y     = true
 cali_batch_size   = 32
-checkpoint_dir    = "/data/models/cx922/plena_camera/gsm8k_05_w4_act4_kv4_gptq_erryclip"
+checkpoint_dir    = "/data/models/gm1425/plena_gptq/fastdllm_gsm8k_05_w4_act4_kv4_gptq_erryclip"
 
 [gptq.weight_config]
 weight_block_size = 16