async_detection.sh

#!/usr/bin/env bash
set -euo pipefail

MODEL_NAMES=(
    "LRM-Conta-Detection-Arena/sft-conta-qwen2.5-7b-no-rl"
    "LRM-Conta-Detection-Arena/sft-conta-qwen2.5-7b-grpo-step64"
    "LRM-Conta-Detection-Arena/sft-conta-llama3-8b-no-rl"
    "LRM-Conta-Detection-Arena/sft-conta-llama3-8b-gpro-step64"

    "LRM-Conta-Detection-Arena/sft-conta-deepseek-distill-llama3-8b"
    "LRM-Conta-Detection-Arena/sft-conta-deepseek-distill-qwen2.5-7b"
)

DATASETS=(
    "minerva_math"
    "aime25"
    "amc23"
    "aime24"
    "olympiadbench"
    "gpqa_diamond"
)

MIAS=( 
    "loss"
    # "perturb"
    # "min-k++"
    # "ref" 
)

GPU_LIST=(8)   # visible GPUs indices
GPUS_PER_JOB=1 # adjust the GPUS_PER_JOB based on your hardware resources
NUM_RESPONSES=8

############################ sanity checks ###################################
if (( ${#GPU_LIST[@]} % GPUS_PER_JOB != 0 )); then
  echo "❌  ${#GPU_LIST[@]} GPUs cannot be split into chunks of $GPUS_PER_JOB" >&2
  exit 1
fi

# ---------------------------------------------------------------------------#
# Build an array whose elements are the comma‑separated GPU slices           
#    e.g.  SLICES=( "0,1" "2,3" "4,5" "6,7" )                                
# ---------------------------------------------------------------------------#
declare -a SLICES=()
for ((i=0;i<${#GPU_LIST[@]}; i+=GPUS_PER_JOB)); do
  slice=("${GPU_LIST[@]:i:GPUS_PER_JOB}")
  SLICES+=( "$(IFS=','; echo "${slice[*]}")" )
done
NUM_SLICES=${#SLICES[@]}             # also the --global_size


POOL=$(mktemp -u)
mkfifo "$POOL"
exec 3<>"$POOL"                    
rm "$POOL"                          


for s in "${SLICES[@]}"; do printf '%s\n' "$s" >&3; done

collect_one () {
  local slice=$1 model=$2 data=$3 mia=$4 shard=$5
  CUDA_VISIBLE_DEVICES="$slice" \
    python collect_result.py \
      --model_name   "$model" \
      --dataset_name "$data" \
      --mia          "$mia" \
      --global_size "$NUM_SLICES" \
      --sharding "$shard" \
      --num_responses "$NUM_RESPONSES"
   
  printf '%s\n' "$slice" >&3     # return slice to pool
}

delete_npz() {
    local model=$1 data=$2 mia=$3 
    CUDA_VISIBLE_DEVICES="$slice" \
    python remove_mia_results.py --model_name "$model" \
                --dataset_name "$data" \
                --mia "$mia" \
                --num_responses "$NUM_RESPONSES"
    printf '%s\n' "$slice" >&3     # return slice to pool
}


for model in "${MODEL_NAMES[@]}"; do
    for data in "${DATASETS[@]}"; do
        for mia in "${MIAS[@]}"; do
            IFS= read -r slice <&3 || true
            delete_npz "$model" "$data" "$mia" &
        done
    done
done
wait
echo "ℹ️  Remove existing MIA results"


echo "ℹ️  Launching collection jobs asynchronously…"
# queue every (model,data,mia) triple
for model in "${MODEL_NAMES[@]}"; do
    for data in "${DATASETS[@]}"; do
        for mia in "${MIAS[@]}"; do
            for ((shard=1; shard<=NUM_SLICES; shard++)); do
                IFS= read -r slice <&3 || true
                collect_one "$slice" "$model" "$data" "$mia" "$shard" &
            done
        done
    done
done
wait
echo "✔ All collection shards finished."