MOOSE-Chem/Assumption2-Reason2Unknown.sh at main · SenticNet/MOOSE-Chem · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
#!/bin/bash
#SBATCH -J exp5_llama
#SBATCH -o logs/exp5_llama
#SBATCH -e logs/exp5_llama
#SBATCH -p AI4Chem
#SBATCH -N 1
#SBATCH -n 1
#SBATCH --gres=gpu:0


api_key="sk-"

model_name_insp_retrieval="gpt4"
model_name_gene="llama318b"
model_name_eval="gpt4"


## Experiment 5: Main Table for Assumption 2 (Main Table 1 and Main Table 2) (when if_use_background_survey == 1)
## Experiment 6: Alation Table on the role of background survey (when if_use_background_survey == 0)

for if_use_background_survey in 1
do
        echo "\n\nif_use_background_survey: $if_use_background_survey"
        for bkg_q_id in {0..50}
        do
                echo "\n\nEntering loop for bkg_q_id: $bkg_q_id"
                python -u hypothesis_generation.py --model_name ${model_name_gene} \
                        --api_type 0 --api_key ${api_key} \
                        --chem_annotation_path ./Data/chem_research_2024.xlsx --corpus_size 300 --if_use_strict_survey_question 1 --if_use_background_survey ${if_use_background_survey} \
                        --inspiration_dir ./Checkpoints/coarse_inspiration_search_${model_name_insp_retrieval}_corpusSize_300_survey_${if_use_background_survey}_strict_1_numScreen_15_round_4_similarity_0_bkgid_${bkg_q_id}.json \
                        --output_dir ./Checkpoints/hypothesis_generation_${model_name_gene}_corpus_300_survey_${if_use_background_survey}_gdthInsp_1_intraEA_1_interEA_1_bkgid_${bkg_q_id}.json \
                        --if_save 1 --if_load_from_saved 0 \
                        --if_use_gdth_insp 1 --idx_round_of_first_step_insp_screening 0 \
                        --num_mutations 3 --num_itr_self_refine 3  --num_self_explore_steps_each_line 3 --num_screening_window_size 12 --num_screening_keep_size 3 \
                        --if_mutate_inside_same_bkg_insp 1 --if_mutate_between_diff_insp 1 --if_self_explore 0 --if_consider_external_knowledge_feedback_during_second_refinement 0 \
                        --inspiration_ids -1  --recom_inspiration_ids -1 --recom_num_beam_size 10  --self_explore_inspiration_ids  --self_explore_num_beam_size 10 \
                        --max_inspiration_search_steps 3 --background_question_id ${bkg_q_id} \
                        --baseline_type 0

                echo "\n\nRunning evaluate.py for bkg_q_id: $bkg_q_id"
                python -u evaluate.py --model_name ${model_name_eval} \
                        --api_type 0 --api_key ${api_key} \
                        --chem_annotation_path ./Data/chem_research_2024.xlsx --corpus_size 300 \
                        --hypothesis_dir ./Checkpoints/hypothesis_generation_${model_name_gene}_corpus_300_survey_${if_use_background_survey}_gdthInsp_1_intraEA_1_interEA_1_bkgid_${bkg_q_id}.json \
                        --output_dir ./Checkpoints/evaluation_${model_name_eval}_corpus_300_survey_${if_use_background_survey}_gdthInsp_1_intraEA_1_interEA_1_bkgid_${bkg_q_id}.json \
                        --if_save 1 --if_load_from_saved 0
        done
done
echo "Experiment 5 and Experiment 6 finished successfully"


## Experiment 12: (MOOSE-Chem w/o significance feedback) (if_use_background_survey == 1 and if_use_gdth_insp == 1; similar to exp5)

# echo "Experiment 12: MOOSE-Chem w/o significance feedback"
# baseline_type=3

# for bkg_q_id in {0..50}
# do
#         echo "\n\nEntering loop for bkg_q_id: $bkg_q_id"
#         python -u hypothesis_generation.py --model_name ${model_name_gene} \
#                 --api_type 1 --api_key ${api_key} \
#                 --chem_annotation_path ./Data/chem_research_2024.xlsx --corpus_size 300 --if_use_strict_survey_question 1 --if_use_background_survey 1 \
#                 --inspiration_dir ./Checkpoints/coarse_inspiration_search_${model_name_insp_retrieval}_corpusSize_300_survey_1_strict_1_numScreen_15_round_4_similarity_0_bkgid_${bkg_q_id}.json \
#                 --output_dir ./Checkpoints/hypothesis_generation_${model_name_gene}_baseline_${baseline_type}_corpus_300_survey_1_gdthInsp_1_intraEA_1_interEA_1_bkgid_${bkg_q_id}.json \
#                 --if_save 1 --if_load_from_saved 0 \
#                 --if_use_gdth_insp 1 --idx_round_of_first_step_insp_screening 0 \
#                 --num_mutations 3 --num_itr_self_refine 3  --num_self_explore_steps_each_line 3 --num_screening_window_size 12 --num_screening_keep_size 3 \
#                 --if_mutate_inside_same_bkg_insp 1 --if_mutate_between_diff_insp 1 --if_self_explore 0 --if_consider_external_knowledge_feedback_during_second_refinement 0 \
#                 --inspiration_ids -1  --recom_inspiration_ids -1 --recom_num_beam_size 10  --self_explore_inspiration_ids  --self_explore_num_beam_size 10 \
#                 --max_inspiration_search_steps 3 --background_question_id ${bkg_q_id} \
#                 --baseline_type ${baseline_type}

#         echo "\n\nRunning evaluate.py for bkg_q_id: $bkg_q_id"
#         python -u evaluate.py --model_name ${model_name_eval} \
#                 --api_type 0 --api_key ${api_key} \
#                 --chem_annotation_path ./Data/chem_research_2024.xlsx --corpus_size 300 \
#                 --hypothesis_dir ./Checkpoints/hypothesis_generation_${model_name_gene}_baseline_${baseline_type}_corpus_300_survey_1_gdthInsp_1_intraEA_1_interEA_1_bkgid_${bkg_q_id}.json \
#                 --output_dir ./Checkpoints/evaluation_${model_name_eval}_baseline_${baseline_type}_corpus_300_survey_1_gdthInsp_1_intraEA_1_interEA_1_bkgid_${bkg_q_id}.json \
#                 --if_save 1 --if_load_from_saved 0
# done
# echo "Experiment 12 finished successfully"


## Experiment 7/8/9/10/11

experiment_id=11

## Experiment 7: MOOSE
if [[ ${experiment_id} -eq 7 ]]; then
        echo "Experiment 7: MOOSE"
        round_of_insp_screening=1
        recom_num_beam_size=15
        # intra-EA
        if_mutate_inside_same_bkg_insp=0
        # inter-EA
        if_mutate_between_diff_insp=0
        # not baseline
        baseline_type=0
## Experiment 8: MOOSE-Chem
elif [[ ${experiment_id} -eq 8 ]]; then
        echo "Experiment 8: MOOSE-Chem"
        # use round 1 to save inference time, can be an ablation to also test round 0
        round_of_insp_screening=1
        # maybe 15 beam size can be a good tradeoff between performance and inference time
        recom_num_beam_size=15
        # intra-EA
        if_mutate_inside_same_bkg_insp=1
        # inter-EA
        if_mutate_between_diff_insp=1
        # not baseline
        baseline_type=0
## Experiment 9: MOOSE-Chem w/o intraEA
elif [[ ${experiment_id} -eq 9 ]]; then
        echo "Experiment 9: MOOSE-Chem w/o intraEA"
        round_of_insp_screening=1
        recom_num_beam_size=15
        # intra-EA
        if_mutate_inside_same_bkg_insp=0
        # inter-EA
        if_mutate_between_diff_insp=1
        # not baseline
        baseline_type=0
## Experiment 10: (baseline 1) MOOSE w/o valid and clarity feedback
elif [[ ${experiment_id} -eq 10 ]]; then
        echo "Experiment 10: MOOSE w/o valid and clarity feedback (baseline 1)"
        round_of_insp_screening=1
        recom_num_beam_size=15
        # intra-EA
        if_mutate_inside_same_bkg_insp=0
        # inter-EA
        if_mutate_between_diff_insp=0
        # baseline 1
        baseline_type=1
## Experiment 11: (baseline 2) MOOSE w/o inspiration retrieval
elif [[ ${experiment_id} -eq 11 ]]; then
        echo "Experiment 11: MOOSE w/o inspiration retrieval (baseline 2)"
        round_of_insp_screening=1
        recom_num_beam_size=15
        # intra-EA
        if_mutate_inside_same_bkg_insp=0
        # inter-EA
        if_mutate_between_diff_insp=0
        # baseline 2
        baseline_type=2
else
        echo "Invalid experiment_id: ${experiment_id}"
        exit 1
fi


# hypothesis_dir
if [ ${experiment_id} -eq 7 ] || [ ${experiment_id} -eq 8 ] || [ ${experiment_id} -eq 9 ]; then
        hypothesis_dir="./Checkpoints/hypothesis_generation_${model_name_gene}_corpus_300_survey_1_gdthInsp_0_roundInsp_${round_of_insp_screening}_intraEA_${if_mutate_inside_same_bkg_insp}_interEA_${if_mutate_between_diff_insp}_beamsize_${recom_num_beam_size}_bkgid_"
elif [ ${experiment_id} -eq 10 ] || [ ${experiment_id} -eq 11 ]; then
        hypothesis_dir="./Checkpoints/hypothesis_generation_${model_name_gene}_baseline_${baseline_type}_corpus_300_survey_1_gdthInsp_0_roundInsp_${round_of_insp_screening}_intraEA_${if_mutate_inside_same_bkg_insp}_interEA_${if_mutate_between_diff_insp}_beamsize_${recom_num_beam_size}_bkgid_"
fi


# echo "Experiment begins..."
# for bkg_q_id in {0..50}
# do
#         # echo "\n\nEntering loop for bkg_q_id: $bkg_q_id"
#         # python -u hypothesis_generation.py --model_name ${model_name_gene} \
#         #         --api_type 0 --api_key ${api_key} \
#         #         --chem_annotation_path ./Data/chem_research_2024.xlsx --corpus_size 300 --if_use_strict_survey_question 1 --if_use_background_survey 1 \
#         #         --inspiration_dir ./Checkpoints/coarse_inspiration_search_${model_name_insp_retrieval}_corpusSize_300_survey_1_strict_1_numScreen_15_round_4_similarity_0_bkgid_${bkg_q_id}.json \
#         #         --output_dir ./Checkpoints/hypothesis_generation_${model_name_gene}_baseline_${baseline_type}_corpus_300_survey_1_gdthInsp_0_roundInsp_${round_of_insp_screening}_intraEA_${if_mutate_inside_same_bkg_insp}_interEA_${if_mutate_between_diff_insp}_beamsize_${recom_num_beam_size}_bkgid_${bkg_q_id}.json \
#         #         --if_save 1 --if_load_from_saved 0 \
#         #         --if_use_gdth_insp 0 --idx_round_of_first_step_insp_screening ${round_of_insp_screening} \
#         #         --num_mutations 3 --num_itr_self_refine 3  --num_self_explore_steps_each_line 3 --num_screening_window_size 12 --num_screening_keep_size 3 \
#         #         --if_mutate_inside_same_bkg_insp ${if_mutate_inside_same_bkg_insp} --if_mutate_between_diff_insp ${if_mutate_between_diff_insp} --if_self_explore 0 --if_consider_external_knowledge_feedback_during_second_refinement 0 \
#         #         --inspiration_ids -1  --recom_inspiration_ids  --recom_num_beam_size ${recom_num_beam_size}  --self_explore_inspiration_ids  --self_explore_num_beam_size 10 \
#         #         --max_inspiration_search_steps 3 --background_question_id ${bkg_q_id} \
#         #         --baseline_type ${baseline_type}

#         echo "\n\nRunning evaluate.py for bkg_q_id: $bkg_q_id"
#         python -u evaluate.py --model_name ${model_name_eval} \
#                 --api_type 0 --api_key ${api_key} \
#                 --chem_annotation_path ./Data/chem_research_2024.xlsx --corpus_size 300 \
#                 --hypothesis_dir ${hypothesis_dir}${bkg_q_id}.json \
#                 --output_dir ./Checkpoints/evaluation_${model_name_eval}_baseline_${baseline_type}_corpus_300_survey_1_gdthInsp_0_roundInsp_${round_of_insp_screening}_intraEA_${if_mutate_inside_same_bkg_insp}_interEA_${if_mutate_between_diff_insp}_beamsize_${recom_num_beam_size}_bkgid_${bkg_q_id}.json \
#                 --if_save 1 --if_load_from_saved 0
# done
# echo "Experiment ${experiment_id} finished successfully"