progen-speculative-decoding/run_sample.sh at main · amyxlu/progen-speculative-decoding · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
#!/bin/bash

MODEL=${MODEL:-"progen2-small"}
NUM_SAMPLES=${NUM_SAMPLES:-1}
MAX_LENGTH=${MAX_LENGTH:-8}

# Sampling
TOP_P=${TOP_P:-0.95}
TEMP=${TEMP:-0.2}
FREQ_PENALTY=${FREQ_PENALTY:-0.0}

# Speculative decoding
SPEC_MODEL=${SPEC_MODEL:-"None"}
NUM_SPEC_TOKENS=${NUM_SPEC_TOKENS:="None"}
NGRAM_PROMPT_LOOKUP_MAX=${NGRAM_PROMPT_LOOKUP_MAX:-4}

USE_VLLM=${USE_VLLM:-True}
SEPARATE_TOKENIZER=${SEPARATE_TOKENIZER:-False}

# Run mode
SANITY=${SANITY:-False}
SAMPLE=${SAMPLE:-False}
BENCHMARK=${BENCHMARK:-True}
LOG_SPEC_DECODE_METRICS=${LOG_SPEC_DECODE_METRICS:-False}

python sample.py \
    --model=$MODEL \
    --num-samples=$NUM_SAMPLES \
    --max-length=$MAX_LENGTH \
    --p=$TOP_P \
    --t=$TEMP \
    --frequency_penalty=$FREQ_PENALTY \
    --speculative_model=$SPEC_MODEL \
    --num_speculative_tokens=$NUM_SPEC_TOKENS \
    --ngram_prompt_lookup_max=$NGRAM_PROMPT_LOOKUP_MAX \
    --use_vllm=$USE_VLLM \
    --separate_tokenizer=$SEPARATE_TOKENIZER \
    --sanity=$SANITY \
    --sample=$SAMPLE \
    --benchmark=$BENCHMARK \
    --log_spec_decode_metrics=$LOG_SPEC_DECODE_METRICS