DriveRLR/example.py at master · Simula-COMPLEX/DriveRLR · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
# from src.DriveRLR import MutateScenarios, LLMAPI, ParseResults
from DriveRLR import MutateScenarios, LLMAPI, ParseResults


road_list = ["road1", "road2", "road3", "road4"]
road_name_list = ["Road1", "Road2", "Road3", "Road4"]
road_description = {
    "road1": "In the following scenario, Ego0's driving intention is to first drive on a dual-way road, then turn right to a one-way road of four lanes.\n",
    "road2": "In the following scenario, Ego0's driving intention is to first drive on a straight one-way road with four lanes, and then switch three lanes.\n",
    "road3": "In the following scenario, Ego0's driving intention is to first perform a left turn to switch to a straight downhill lane and then drive without turning.\n",
    "road4": "In the following scenario, Ego0's driving intention is to first turn left and then drive on the right-hand side of the road.\n"
}
weather_list = ["rain_day", "rain_night", "sunny_day", "clear_night"]
weather_name_list = ["Rainy Day", "Rainy Night", "Sunny Day", "Clear Night"]
parameter_list = ["position", "rotation", "velocity", "angular_velocity"]
parameter_name_list = ["Position", "Rotation", "Velocity", "Angular Velocity"]
mutation_per_list = [0.01, -0.01]
extra_list = ["increase" if v > 0 else "decrease" for v in mutation_per_list]
mutation_dimension = False
s_num = 4
scenario_number = s_num * len(road_list) * len(weather_list)
total_time_step = 6
randomly_select_scenarios_folder = "./data/deepscenario/randomly_selected_scenarios"
outputs_results_folder = "./data/outputs_results"


# mutate_scenarios = MutateScenarios()
# for road in road_list:
#     for weather in weather_list:
#         randomly_select_scenarios_path = f"{randomly_select_scenarios_folder}/" + road + "-" + weather + "-scenarios/"
#         mutate_scenarios.original_scenarios_2json(randomly_select_scenarios_path, total_time_step, parameter_list)
#         mutate_scenarios.extra_mutate_scenarios_timestep(randomly_select_scenarios_path, total_time_step,
#                                                          parameter_list, mutation_per_list, mutation_dimension)


# model = "gpt-3.5-turbo-1106"
# model = "accounts/fireworks/models/llama-v2-13b-chat"
# model = "accounts/fireworks/models/mistral-7b-instruct-4k"
model = "gpt-5-2025-08-07"
# model = "accounts/fireworks/models/llama4-maverick-instruct-basic"
# model = "mistral-small-2506"
model_list = ["gpt-5-2025-08-07", "llama4-maverick-instruct-basic", "mistral-small-2506"]
model_name_list = ["GPT-5", "Llama4-M", "Mistral-Small"]
# model_list = ["gpt-3.5-turbo-1106", "llama-v2-13b-chat", "mistral-7b-instruct-4k"]
# model_name_list = ["GPT-3.5", "Llama2-13B", "Mistral-7B"]

temperature = 1 if "gpt-5" in model else 0
repeat_time = 10
api_key = ""


# llmapi = LLMAPI()
# for road in road_list:
#     for weather in weather_list:
#         randomly_select_scenarios_path = f"{randomly_select_scenarios_folder}/" + road + "-" + weather + "-scenarios/"
#         llmapi.evaluate_R_MR_extra_4full_experiments(randomly_select_scenarios_path,
#                                                      road, road_description[road], weather,
#                                                      total_time_step, parameter_list, extra_list,
#                                                      model, repeat_time, temperature, api_key, outputs_results_folder)


# parseResults = ParseResults()
#
# for road in road_list:
#     for weather in weather_list:
#         evaluate_R_MR_extra_4full_experiments_path = f"{outputs_results_folder}/" + road + "-" + weather + "-scenarios/"
#         parseResults.parse_extra_mutated_scenarios(evaluate_R_MR_extra_4full_experiments_path, parameter_list, extra_list,
#                                                    model_list, repeat_time)
#
# parseResults.count_extra_mutated_scenarios_robustness_RQ(model_list, outputs_results_folder,
#                                                          road_list, road_name_list, weather_list, weather_name_list,
#                                                          parameter_list, parameter_name_list, extra_list)
# parseResults.count_extra_mutated_scenarios_robustness_RQ1(outputs_results_folder, model_list, parameter_name_list,
#                                                           scenario_number)
# parseResults.count_extra_mutated_scenarios_robustness_RQ2(outputs_results_folder, model_list, road_name_list,
#                                                           weather_name_list, parameter_name_list)
# parseResults.count_extra_mutated_scenarios_robustness_RQ3(outputs_results_folder, model_list, road_name_list,
#                                                           weather_name_list, parameter_name_list, s_num)
#
# parseResults.sort_robustness_RQ_a_c(outputs_results_folder, model_list, model_name_list, parameter_name_list,
#                                     road_name_list, weather_name_list)
# parseResults.sort_rank_robustness_RQ_b_d(outputs_results_folder, model_list, model_name_list, parameter_name_list,
#                                          road_name_list, weather_name_list)
#
# parseResults.draw_violin_diagram_RQ0_robustness(outputs_results_folder, model_list, model_name_list)
# parseResults.draw_violin_diagram_RQ1_robustness(outputs_results_folder, model_list, model_name_list,
#                                                 scenario_number, parameter_name_list)
# parseResults.draw_violin_diagram_RQ2_robustness(outputs_results_folder, model_list, model_name_list, road_name_list,
#                                                 weather_name_list, parameter_name_list, s_num)
# parseResults.draw_violin_diagram_RQ3_robustness(outputs_results_folder, model_list, model_name_list, road_name_list,
#                                                 weather_name_list, parameter_name_list, s_num)