diff --git a/demo/demo_data_farming_model.py b/demo/demo_data_farming_model.py deleted file mode 100644 index 3687e1373..000000000 --- a/demo/demo_data_farming_model.py +++ /dev/null @@ -1,79 +0,0 @@ -"""Demo for Data Farming Models. - -This script is intended to help with running a data-farming experiment on -a simulation model. It creates a design of model factors and runs multiple -replications at each configuration of the model. Outputs are printed to a file. -""" - -import sys -from pathlib import Path - -# Append the parent directory (simopt package) to the system path -sys.path.append(str(Path(__file__).resolve().parent.parent)) - -from simopt.data_farming_base import DataFarmingExperiment - - -def main() -> None: - """Main function to run the data farming experiment.""" - # Specify the name of the model as it appears in directory.py - model_name = "CNTNEWS" - - # Specify the names of the model factors (in order) that will be varied. - factor_headers = [ - "purchase_price", - "sales_price", - "salvage_price", - "order_quantity", - ] - - # If creating the design, provide the name of a .txt file containing - # the following: - # - one row corresponding to each model factor being varied - # - three columns: - # - first column: lower bound for factor value - # - second column: upper bound for factor value - # - third column: (integer) number of digits for discretizing values - # (e.g., 0 corresponds to integral values for the factor) - # factor_settings_filename = "model_factor_settings" - factor_settings_filename = None - - # OR, if the design has been created, provide the name of a .text file - # containing the following: - # - one row corresponding to each design point - # - the number of columns equal to the number of factors being varied - # - each value in the table gives the value of the factor (col index) - # for the design point (row index) - # E.g., design_filename = "model_factor_settings_design" - # design_filename = None - design_filename = "model_factor_settings_design" - - # Specify a common number of replications to run of the model at each - # design point. - n_reps = 10 - - # Specify whether to use common random numbers across different versions - # of the model. - crn_across_design_pts = True - - # Specify filename for outputs. - output_filename = "cntnews_data_farming_output" - - # No code beyond this point needs to be edited. - - # Create DataFarmingExperiment object. - myexperiment = DataFarmingExperiment( - model_name=model_name, - factor_settings_file_name=factor_settings_filename, - factor_headers=factor_headers, - design_path=design_filename, - model_fixed_factors={}, - ) - - # Run replications and print results to file. - myexperiment.run(n_reps=n_reps, crn_across_design_pts=crn_across_design_pts) - myexperiment.print_to_csv(csv_file_name=output_filename) - - -if __name__ == "__main__": - main() diff --git a/demo/demo_data_farming_over_solver_and_problem.py b/demo/demo_data_farming_over_solver_and_problem.py deleted file mode 100644 index a8a1a5330..000000000 --- a/demo/demo_data_farming_over_solver_and_problem.py +++ /dev/null @@ -1,133 +0,0 @@ -"""Demo for Data Farming over Solvers and Problems. - -This script is intended to help with running a data-farming experiment on -a solver. It creates a design of solver factors and runs multiple -macroreplications at each version of the solver. Outputs are printed to a file. -""" - -import sys -from pathlib import Path - -# Append the parent directory (simopt package) to the system path -sys.path.append(str(Path(__file__).resolve().parent.parent)) - -from simopt.experiment_base import ProblemsSolvers, create_design - - -def main() -> None: - """Main function to run the data farming experiment.""" - # Specify the name of the solver as it appears in directory.py - solver_name = "ASTRODF" - # Specify the name of the problem as it appears in directory.py - problem_name = "CNTNEWS-1" - # Specify the name of the model as it appears in directory.py - model_name = "CNTNEWS" - - # Specify the names of the sovler factors (in order) that will be varied. - solver_factor_headers = ["eta_1", "eta_2", "lambda_min"] - # Specify the names of the model factors (in order) that will be varied. - model_factor_headers = ["purchase_price", "sales_price", "order_quantity"] - - # OPTIONAL: factors chosen for cross design - # factor name followed by list containing factor values to cross design over - solver_cross_design_factors = {"crn_across_solns": [True, False]} - # model_cross_design_factors = {} - - # OPTIONAL: Provide additional overrides for default factors. - # If empty, default factor settings are used. - solver_fixed_factors = {} - model_fixed_factors = {"salvage_price": 5, "Burr_c": 1} - - # Provide the name of a file .txt locatated in the datafarming_experiments folder - # containing the following: - # - one row corresponding to each solver factor being varied - # - three columns: - # - first column: lower bound for factor value - # - second column: upper bound for factor value - # - third column: (integer) number of digits for discretizing values - # (e.g., 0 corresponds to integral values for the factor) - solver_factor_settings_filename = "astrodf_testing" - model_factor_settings_filename = "testing_model_cntnews_1" - - # Specify the number stacks to use for ruby design creation - solver_n_stacks = 1 - problem_n_stacks = 1 - - # Specify a common number of macroreplications of each unique solver/problem - # combination (i.e., the number of runs at each design point.) - n_macroreps = 3 - - # Specify the number of postreplications to take at each recommended solution - # from each macroreplication at each design point. - n_postreps = 100 - - # Specify the number of postreplications to take at x0 and x*. - n_postreps_init_opt = 200 - - # Specify the CRN control for postreplications. - crn_across_budget = True # Default - crn_across_macroreps = False # Default - crn_across_init_opt = True # Default - - # Create DataFarmingExperiment object for sovler design - solver_design_list = create_design( - name=solver_name, - factor_headers=solver_factor_headers, - factor_settings_filename=solver_factor_settings_filename, - n_stacks=solver_n_stacks, - fixed_factors=solver_fixed_factors, # optional - cross_design_factors=solver_cross_design_factors, # optional - ) - # Create DataFarmingExperiment object for model design - model_design_list = create_design( - name=model_name, - factor_headers=model_factor_headers, - factor_settings_filename=model_factor_settings_filename, - n_stacks=problem_n_stacks, - fixed_factors=model_fixed_factors, # optional - # cross_design_factors=model_cross_design_factors, #optional - ) - - # create solver name list for ProblemsSolvers (do not edit) - solver_names = [] - for _ in range(len(solver_design_list)): - solver_names.append(solver_name) - - # create proble name list for ProblemsSolvers (do not edit) - problem_names = [] - for _ in range(len(model_design_list)): - problem_names.append(problem_name) - - # Create ProblemsSovlers experiment with solver and model design - experiment = ProblemsSolvers( - solver_factors=solver_design_list, - problem_factors=model_design_list, - solver_names=solver_names, - problem_names=problem_names, - ) - - # check compatibility of selected solvers and problems - experiment.check_compatibility() - - # Run macroreplications at each design point. - experiment.run(n_macroreps) - - # Postprocess the experimental results from each design point. - experiment.post_replicate( - n_postreps=n_postreps, - crn_across_budget=crn_across_budget, - crn_across_macroreps=crn_across_macroreps, - ) - experiment.post_normalize( - n_postreps_init_opt=n_postreps_init_opt, - crn_across_init_opt=crn_across_init_opt, - ) - - # Record and log results - experiment.record_group_experiment_results() - experiment.log_group_experiment_results() - experiment.report_group_statistics() - - -if __name__ == "__main__": - main() diff --git a/demo/demo_data_farming_problem.py b/demo/demo_data_farming_problem.py deleted file mode 100644 index 12c696521..000000000 --- a/demo/demo_data_farming_problem.py +++ /dev/null @@ -1,123 +0,0 @@ -"""Demo for Data Farming over Problems. - -This script is intended to help with running a data-farming experiment on -a problem. It creates a design of problem factors and runs multiple -macroreplications at each version of the problem. Outputs are printed to a file. -""" - -import sys -from pathlib import Path - -# Append the parent directory (simopt package) to the system path -sys.path.append(str(Path(__file__).resolve().parent.parent)) - -from simopt.experiment_base import ProblemsSolvers, create_design - - -def main() -> None: - """Main function to run the data farming experiment.""" - # Specify the name of the problem as it appears in directory.py - problem_name = "CNTNEWS-1" - # Specify the name of the model as it appears in directory.py - model_name = "CNTNEWS" - # Specify the name of the solver as it appears in directory.py - solver_names = ["ASTRODF", "RNDSRCH"] - - # Specify the names of the model factors (in order) that will be varied. - model_factor_headers = ["purchase_price", "sales_price", "order_quantity"] - - # OPTIONAL: factors chosen for cross design - # factor name followed by list containing factor values to cross design over - # model_cross_design_factors = {} - - # OPTIONAL: Provide additional overrides for model default factors. - # If empty, default factor settings are used. - model_fixed_factors = {"salvage_price": 5, "Burr_c": 1} - - # OPTIONAL: Provide additional overrides for solver default factors. - # If empty, default factor settings are used. - # list of dictionaries that provide fixed factors for problems when you don't want - # to use the default values. if you want to use all default values use empty - # dictionary, order must match problem names - solver_fixed_factors = [{"eta_1": 0.5, "eta_2": 0.4}, {"sample_size": 15}] - - # uncomment this version to run w/ only default solver factors - # sp;ver_fixed_factors = [{},{}] - - # Provide the name of a file .txt locatated in the datafarming_experiments folder - # containing the following: - # - one row corresponding to each solver factor being varied - # - three columns: - # - first column: lower bound for factor value - # - second column: upper bound for factor value - # - third column: (integer) number of digits for discretizing values - # (e.g., 0 corresponds to integral values for the factor) - model_factor_settings_filename = "testing_model_cntnews_1" - - # Specify the number stacks to use for ruby design creation - problem_n_stacks = 1 - - # Specify a common number of macroreplications of each unique solver/problem - # combination (i.e., the number of runs at each design point.) - n_macroreps = 3 - - # Specify the number of postreplications to take at each recommended solution - # from each macroreplication at each design point. - n_postreps = 100 - - # Specify the number of postreplications to take at x0 and x*. - n_postreps_init_opt = 200 - - # Specify the CRN control for postreplications. - crn_across_budget = True # Default - crn_across_macroreps = False # Default - crn_across_init_opt = True # Default - - # Create DataFarmingExperiment object for model design - model_design_list = create_design( - name=model_name, - factor_headers=model_factor_headers, - factor_settings_filename=model_factor_settings_filename, - n_stacks=problem_n_stacks, - fixed_factors=model_fixed_factors, # optional - # cross_design_factors=model_cross_design_factors, #optional - ) - - # create proble name list for ProblemsSolvers (do not edit) - problem_names = [] - for _ in range(len(model_design_list)): - problem_names.append(problem_name) - - # Create ProblemsSovlers experiment with solver and model design - experiment = ProblemsSolvers( - solver_factors=solver_fixed_factors, - problem_factors=model_design_list, - solver_names=solver_names, - problem_names=problem_names, - ) - - # check compatibility of selected solvers and problems - experiment.check_compatibility() - - # Run macroreplications at each design point. - experiment.run(n_macroreps) - - # Postprocess the experimental results from each design point. - experiment.post_replicate( - n_postreps=n_postreps, - crn_across_budget=crn_across_budget, - crn_across_macroreps=crn_across_macroreps, - ) - experiment.post_normalize( - n_postreps_init_opt=n_postreps_init_opt, - crn_across_init_opt=crn_across_init_opt, - ) - - # Record and log results - experiment.record_group_experiment_results() - experiment.log_group_experiment_results() - experiment.report_group_statistics() - - -if __name__ == "__main__": - main() diff --git a/demo/demo_data_farming_solver.py b/demo/demo_data_farming_solver.py deleted file mode 100644 index f36a132f9..000000000 --- a/demo/demo_data_farming_solver.py +++ /dev/null @@ -1,123 +0,0 @@ -"""Demo for Data Farming over Solvers. - -This script is intended to help with running a data-farming experiment on -a solver. It creates a design of solver factors and runs multiple -macroreplications at each version of the solver. Outputs are printed to a file. -""" - -import sys -from pathlib import Path - -# Append the parent directory (simopt package) to the system path -sys.path.append(str(Path(__file__).resolve().parent.parent)) - -from simopt.experiment_base import ProblemsSolvers, create_design - - -def main() -> None: - """Main function to run the data farming experiment.""" - # Specify the name of the solver as it appears in directory.py - solver_name = "ASTRODF" - # list of problem names for solver design to be run on - # (if more than one version of same problem, repeat name) - # Specify the name of the problem as it appears in directory.py - problem_names = ["SSCONT-1", "SAN-1"] - - # Specify the names of the sovler factors (in order) that will be varied. - solver_factor_headers = ["eta_1", "eta_2", "lambda_min"] - - # OPTIONAL: factors chosen for cross design - # factor name followed by list containing factor values to cross design over - solver_cross_design_factors = {"crn_across_solns": [True, False]} - - # OPTIONAL: Provide additional overrides for solver default factors. - # If empty, default factor settings are used. - solver_fixed_factors = {} - # OPTIONAL: Provide additional overrides for problem default factors. - # If empty, default factor settings are used. - # list of dictionaries that provide fixed factors for problems when you don't want - # to use the default values. if you want to use all default values use empty - # dictionary, order must match problem names - problem_fixed_factors = [ - {"budget": 2000, "demand_mean": 90.0, "fixed_cost": 25}, - {"budget": 500}, - ] - - # Provide the name of a file .txt locatated in the datafarming_experiments - # folder containing - # the following: - # - one row corresponding to each solver factor being varied - # - three columns: - # - first column: lower bound for factor value - # - second column: upper bound for factor value - # - third column: (integer) number of digits for discretizing values - # (e.g., 0 corresponds to integral values for the factor) - solver_factor_settings_filename = "astrodf_testing" - - # Specify the number stacks to use for ruby design creation - solver_n_stacks = 1 - - # Specify a common number of macroreplications of each unique solver/problem - # combination (i.e., the number of runs at each design point.) - n_macroreps = 3 - - # Specify the number of postreplications to take at each recommended solution - # from each macroreplication at each design point. - n_postreps = 100 - - # Specify the number of postreplications to take at x0 and x*. - n_postreps_init_opt = 200 - - # Specify the CRN control for postreplications. - crn_across_budget = True # Default - crn_across_macroreps = False # Default - crn_across_init_opt = True # Default - - # Create DataFarmingExperiment object for sovler design - solver_design_list = create_design( - name=solver_name, - factor_headers=solver_factor_headers, - factor_settings_filename=solver_factor_settings_filename, - n_stacks=solver_n_stacks, - fixed_factors=solver_fixed_factors, # optional - cross_design_factors=solver_cross_design_factors, # optional - ) - - # create solver name list for ProblemsSolvers (do not edit) - solver_names = [] - for _ in range(len(solver_design_list)): - solver_names.append(solver_name) - - # Create ProblemsSovlers experiment with solver and model design - experiment = ProblemsSolvers( - solver_factors=solver_design_list, - problem_factors=problem_fixed_factors, - solver_names=solver_names, - problem_names=problem_names, - ) - - # check compatibility of selected solvers and problems - experiment.check_compatibility() - - # Run macroreplications at each design point. - experiment.run(n_macroreps) - - # Postprocess the experimental results from each design point. - experiment.post_replicate( - n_postreps=n_postreps, - crn_across_budget=crn_across_budget, - crn_across_macroreps=crn_across_macroreps, - ) - experiment.post_normalize( - n_postreps_init_opt=n_postreps_init_opt, - crn_across_init_opt=crn_across_init_opt, - ) - - # Record and log results - experiment.record_group_experiment_results() - experiment.log_group_experiment_results() - experiment.report_group_statistics() - - -if __name__ == "__main__": - main() diff --git a/demo/demo_load_solver_design.py b/demo/demo_load_solver_design.py deleted file mode 100644 index 6007ed98b..000000000 --- a/demo/demo_load_solver_design.py +++ /dev/null @@ -1,108 +0,0 @@ -"""Demo for Loading Solver Design Points. - -This script is intended to help with loading a design of solver factors -to run on one or more problems. If more than one problem are used a cross-design of -problems will be run. The design file should be a txt or csv file with headers as -factor names and each row representing an individual design point. Design files -generated by the simopt GUI can also be run using this script. Outputs are printed to -the experiments folder in simopt. -""" - -import sys -from pathlib import Path - -import pandas as pd - -# Append the parent directory (simopt package) to the system path -sys.path.append(str(Path(__file__).resolve().parent.parent)) - -from simopt.experiment_base import ProblemsSolvers - - -def main() -> None: - """Main function to run the data farming experiment.""" - # run this script in the terminal from the simopt directory - # name of solver that design was created on - solver_name = "RNDSRCH" - # list of problem names for solver design to be run on - # (if more than one version of same problem, repeat name) - problem_names = [ - "SSCONT-1", - "SAN-1", - ] - - # name of file containing design points (csv or excel): column headers must exactly - # match names of solver factors w/ each row representing a design point - # (can also use csv's generated by GUI) - design_filename = Path("data_farming_experiments", "RNDSRCH_design.csv") - - # list of dictionaries that provide fixed factors for problems when you don't want - # to use the default values, if you want to use all default values use empty - # dictionary, order must match problem names - problem_fixed_factors = [ - {"budget": 2000, "demand_mean": 90.0, "fixed_cost": 25}, - {"budget": 500}, - ] - - # uncomment this version to run w/ only default problem factors - # problem_fixed_factors = [{},{}] - - # use this dictionary to change any default solver factors that were not included - # in the design - solver_fixed_factors = {} - - # number of macroreplication to run and each solver design point - n_macroreps = 2 - # number of post replications to run on each macro replication - n_postreps = 100 - # number of normalization postreplications to run at initial solution and optimal - # solution - n_postreps_init_opt = 200 - - # turn design file into df & retrive dp information - design_table = pd.read_csv(design_filename) - - design_factor_names = design_table.columns.tolist() - - # remove GUI columns from list if present (ignore if not using design files - # generated by GUI) - design_factor_names.remove("Design #") - design_factor_names.remove("Solver Name") - design_factor_names.remove("Design Type") - design_factor_names.remove("Number Stacks") - - dp_list = [] # list of all design points - - for _, row in design_table.iterrows(): - dp = {} # dictionary of current dp - for factor in design_factor_names: - dp[factor] = row[factor] - dp_list.append(dp) - - # add fixed solver factors to dps - for fixed_factor in solver_fixed_factors: - for dp in dp_list: - dp[fixed_factor] = solver_fixed_factors[fixed_factor] - - n_dp = len(dp_list) - solver_names = [] - for _ in range(n_dp): - solver_names.append(solver_name) - - experiment = ProblemsSolvers( - solver_factors=dp_list, - problem_factors=problem_fixed_factors, - solver_names=solver_names, - problem_names=problem_names, - ) - - experiment.run(n_macroreps) - experiment.post_replicate(n_postreps) - experiment.post_normalize(n_postreps_init_opt) - experiment.record_group_experiment_results() - experiment.log_group_experiment_results() - experiment.report_group_statistics() - - -if __name__ == "__main__": - main() diff --git a/demo/demo_model.py b/demo/demo_model.py deleted file mode 100644 index 92a5e3015..000000000 --- a/demo/demo_model.py +++ /dev/null @@ -1,71 +0,0 @@ -"""Demo for Model Debugging. - -This script is intended to help with debugging a model. -It imports a model, initializes a model object with given factors, -sets up pseudorandom number generators, and runs one or more replications. -""" - -import sys -from pathlib import Path - -# Append the parent directory (simopt package) to the system path -sys.path.append(str(Path(__file__).resolve().parent.parent)) - -# Import random number generator. -from mrg32k3a.mrg32k3a import MRG32k3a - - -def main() -> None: - """Main function to run the data farming experiment.""" - # Import model. - # from models. import - # Replace with name of .py file containing model class. - # Replace with name of model class. - # Fix factors of model. Specify a dictionary of factors. - # fixed_factors = {} # Resort to all default values. - # Look at Model class definition to get names of factors. - # Initialize an instance of the specified model class. - # mymodel = (fixed_factors) - # Replace with name of model class. - # Working example for MM1 model. - # ----------------------------------------------- - from simopt.models.mm1queue import MM1Queue - - fixed_factors = {"lambda": 3.0, "mu": 8.0} - mymodel = MM1Queue(fixed_factors=fixed_factors) - # ----------------------------------------------- - - # The rest of this script requires no changes. - - # Check that all factors describe a simulatable model. - # Check fixed factors individually. - for key, value in mymodel.factors.items(): - print( - f"The factor {key} is set as {value}. " - f"Is this simulatable? {bool(mymodel.check_simulatable_factor(key))}." - ) - # Check all factors collectively. - print( - "Is the specified model simulatable? " - f"{bool(mymodel.check_simulatable_factors())}." - ) - - # Create a list of RNG objects for the simulation model to use when - # running replications. - rng_list = [MRG32k3a(s_ss_sss_index=[0, ss, 0]) for ss in range(mymodel.n_rngs)] - - # Run a single replication of the model. - responses, gradients = mymodel.replicate(rng_list) - print("\nFor a single replication:") - print("\nResponses:") - for key, value in responses.items(): - print(f"\t {key} is {value}.") - print("\n Gradients:") - for outerkey in gradients: - print(f"\tFor the response {outerkey}:") - for innerkey, value in gradients[outerkey].items(): - print(f"\t\tThe gradient w.r.t. {innerkey} is {value}.") - - -if __name__ == "__main__": - main() diff --git a/demo/demo_plots.py b/demo/demo_plots.py deleted file mode 100644 index ce8d88f97..000000000 --- a/demo/demo_plots.py +++ /dev/null @@ -1,98 +0,0 @@ -"""Demo Plotting Script. - -This script is intended to help with debugging problems and solvers. -It create a problem-solver pairing (using the directory) and runs multiple -macroreplications of the solver on the problem. -""" - -import sys -from pathlib import Path - -# Append the parent directory (simopt package) to the system path -sys.path.append(str(Path(__file__).resolve().parent.parent)) - -# Import the ProblemSolver class and other useful functions -from simopt.experiment_base import ( - PlotType, - ProblemSolver, - plot_terminal_progress, - post_normalize, - read_experiment_results, -) - - -def main() -> None: - """Main function to run the data farming experiment.""" - solver_names = {"RNDSRCH", "ASTRODF", "NELDMD"} - problem_names = {"SAN-1"} # CNTNEWS-1"} #, "SAN-1"} - # solver_name = "RNDSRCH" # Random search solver - # problem_name = "CNTNEWS-1" # Continuous newsvendor problem - # solver_name = - # problem_name = - - for problem_name in problem_names: - problem_experiments = [] - for solver_name in solver_names: - print(f"Testing solver {solver_name} on problem {problem_name}.") - # Initialize an instance of the experiment class. - myexperiment = ProblemSolver(solver_name, problem_name) - - file_name_path = ( - "experiments/outputs/" + solver_name + "_on_" + problem_name + ".pickle" - ) - - # Run a fixed number of macroreplications of the solver on the problem. - # myexperiment.run(n_macroreps=10) - - # If the solver runs have already been performed, uncomment the - # following pair of lines (and uncommmen the myexperiment.run(...) - # line above) to read in results from a .pickle file. - myexperiment = read_experiment_results(file_name_path) - - print("Post-processing results.") - # Run a fixed number of postreplications at all recommended solutions. - myexperiment.post_replicate(n_postreps=200) - - problem_experiments.append(myexperiment) - - # Find an optimal solution x* for normalization. - post_normalize(problem_experiments, n_postreps_init_opt=200) - - # Re-compile problem-solver results. - myexperiments = [] - for solver_name in solver_names: - # solver_experiments = [] - for problem_name in problem_names: - file_name_path = ( - "experiments/outputs/" + solver_name + "_on_" + problem_name + ".pickle" - ) - myexperiment = read_experiment_results(file_name_path) - myexperiments.append(myexperiment) - # solver_experiments.append(myexperiment) - # myexperiments.append(solver_experiments) - - print("Plotting results.") - # Produce basic plots. - plot_terminal_progress( - experiments=myexperiments, plot_type=PlotType.BOX, normalize=False - ) - plot_terminal_progress( - experiments=myexperiments, plot_type=PlotType.BOX, normalize=True - ) - plot_terminal_progress( - experiments=myexperiments, - plot_type=PlotType.VIOLIN, - normalize=False, - all_in_one=False, - ) - plot_terminal_progress( - experiments=myexperiments, plot_type=PlotType.VIOLIN, normalize=True - ) - # plot_terminal_scatterplots(experiments = myexperiments, all_in_one=False) - - # Plots will be saved in the folder experiments/plots. - print("Finished. Plots can be found in experiments/plots folder.") - - -if __name__ == "__main__": - main() diff --git a/demo/demo_problem.py b/demo/demo_problem.py deleted file mode 100644 index 5755d10cb..000000000 --- a/demo/demo_problem.py +++ /dev/null @@ -1,122 +0,0 @@ -"""Demo script for the Problem class. - -This script is intended to help with debugging a problem. -It imports a problem, initializes a problem object with given factors, -sets up pseudorandom number generators, and runs multiple replications -at a given solution. -""" - -import sys -from pathlib import Path - -# Append the parent directory (simopt package) to the system path -sys.path.append(str(Path(__file__).resolve().parent.parent)) - -# Import random number generator. -from mrg32k3a.mrg32k3a import MRG32k3a - -# Import the Solution class. -from simopt.base import Solution - - -def main() -> None: - """Main function to run the data farming experiment.""" - # Import problem. - # from models. import - # Replace with name of .py file containing problem class. - # Replace with name of problem class. - # Fix factors of problem. Specify a dictionary of factors. - # fixed_factors = {} # Resort to all default values. - # Look at Problem class definition to get names of factors. - # Initialize an instance of the specified problem class. - # myproblem = (fixed_factors=fixed_factors) - # Replace with name of problem class. - # Initialize a solution x corresponding to the problem. - # x = (,) - # Look at the Problem class definition to identify the decision variables. - # x will be a tuple consisting of the decision variables. - # The following line does not need to be changed. - # mysolution = Solution(x, myproblem) - # Working example for CntNVMaxProfit problem. - # ----------------------------------------------- - from simopt.models.cntnv import CntNVMaxProfit - - fixed_factors = {"initial_solution": (2,), "budget": 500} - myproblem = CntNVMaxProfit(fixed_factors=fixed_factors) - x = (3,) - mysolution = Solution(x, myproblem) - # ----------------------------------------------- - - # Another working example for FacilitySizingTotalCost problem. (Commented out) - # This example has stochastic constraints. - # ----------------------------------------------- - # from models.facilitysizing import FacilitySizingTotalCost - # fixed_factors = {"epsilon": 0.1} - # myproblem = FacilitySizingTotalCost(fixed_factors=fixed_factors) - # x = (200, 200, 200) - # mysolution = Solution(x, myproblem) - # ----------------------------------------------- - - # The rest of this script requires no changes. - - # Create and attach rngs to solution - rng_list = [ - MRG32k3a(s_ss_sss_index=[0, ss, 0]) for ss in range(myproblem.model.n_rngs) - ] - mysolution.attach_rngs(rng_list, copy=False) - - # Simulate a fixed number of replications (n_reps) at the solution x. - n_reps = 10 - myproblem.simulate(mysolution, num_macroreps=n_reps) - - # Print results to console. - print( - f"Ran {n_reps} replications of the {myproblem.name} problem " - f"at solution x = {x}.\n" - ) - obj_mean = round(mysolution.objectives_mean[0], 4) - obj_stderr = round(mysolution.objectives_stderr[0], 4) - print( - f"The mean objective estimate was {obj_mean} with standard error {obj_stderr}." - ) - print("The individual observations of the objective were:") - for idx in range(n_reps): - print(f"\t {round(mysolution.objectives[idx][0], 4)}") - if myproblem.gradient_available: - print("\nThe individual observations of the gradients of the objective were:") - for idx in range(n_reps): - grads = mysolution.objectives_gradients[idx][0] - print(f"\t {[round(g, 4) for g in grads]}") - else: - print("\nThis problem has no known gradients.") - if myproblem.n_stochastic_constraints > 0: - print( - f"\nThis problem has {myproblem.n_stochastic_constraints} stochastic " - "constraints of the form E[LHS] <= 0." - ) - for stc_idx in range(myproblem.n_stochastic_constraints): - stoch_const_mean = mysolution.stoch_constraints_mean[stc_idx] - stoch_const_mean_round = round(stoch_const_mean, 4) - stoch_const_stderr = mysolution.stoch_constraints_stderr[stc_idx] - stoch_const_stderr_round = round(stoch_const_stderr, 4) - print( - f"\tFor stochastic constraint #{stc_idx + 1}, " - f"the mean of the LHS was {stoch_const_mean_round} " - f"with standard error {stoch_const_stderr_round}." - ) - print("\tThe observations of the LHSs were:") - for idx in range(n_reps): - # Quick check to make sure the stochastic constraints are not None. - if mysolution.stoch_constraints is None: - error_msg = "Stochastic constraints should not be None." - raise ValueError(error_msg) - # Print out the current stochastic constraint value. - stoch_const = mysolution.stoch_constraints[stc_idx][idx] - stoch_const_round = round(stoch_const, 4) - print(f"\t\t {stoch_const_round}") - else: - print("\nThis problem has no stochastic constraints.") - - -if __name__ == "__main__": - main() diff --git a/demo/demo_problem_solver.py b/demo/demo_problem_solver.py deleted file mode 100644 index ba704895f..000000000 --- a/demo/demo_problem_solver.py +++ /dev/null @@ -1,91 +0,0 @@ -"""Demo script for the ProblemSolver class. - -This script is intended to help with debugging problems and solvers. -It create a problem-solver pairing (using the directory) and runs multiple -macroreplications of the solver on the problem. -""" - -import sys -from pathlib import Path - -# Append the parent directory (simopt package) to the system path -sys.path.append(str(Path(__file__).resolve().parent.parent)) - -# Import the ProblemSolver class and other useful functions -from simopt.experiment_base import ( - PlotType, - ProblemSolver, - plot_progress_curves, - plot_solvability_cdfs, - post_normalize, -) - - -def main() -> None: - """Main function to run the demo script.""" - # !! When testing a new solver/problem, first go to directory.py. - # There you should add the import statement and an entry in the respective - # dictionary (or dictionaries). - # See directory.py for more details. - - # Specify the names of the solver and problem to test. - # solver_name = - # problem_name = - # These names are strings and should match those input to directory.py. - - # Example with random search solver on continuous newsvendor problem. - # ----------------------------------------------- - solver_name = "RNDSRCH" # Random search solver - problem_name = "CNTNEWS-1" # Continuous newsvendor problem - # ----------------------------------------------- - - print(f"Testing solver {solver_name} on problem {problem_name}.") - - # Specify file path name for storing experiment outputs in .pickle file. - file_name_path = ( - "experiments/outputs/" + solver_name + "_on_" + problem_name + ".pickle" - ) - print(f"Results will be stored as {file_name_path}.") - - # Initialize an instance of the experiment class. - myexperiment = ProblemSolver(solver_name, problem_name) - - # Run a fixed number of macroreplications of the solver on the problem. - myexperiment.run(n_macroreps=10) - - # If the solver runs have already been performed, uncomment the - # following pair of lines (and uncommmen the myexperiment.run(...) - # line above) to read in results from a .pickle file. - # myexperiment = read_experiment_results(file_name_path) - - print("Post-processing results.") - # Run a fixed number of postreplications at all recommended solutions. - myexperiment.post_replicate(n_postreps=200) - # Find an optimal solution x* for normalization. - post_normalize([myexperiment], n_postreps_init_opt=200) - - # Log results. - myexperiment.log_experiment_results() - - print("Plotting results.") - # Produce basic plots of the solver on the problem. - plot_progress_curves( - experiments=[myexperiment], plot_type=PlotType.ALL, normalize=False - ) - plot_progress_curves( - experiments=[myexperiment], plot_type=PlotType.MEAN, normalize=False - ) - plot_progress_curves( - experiments=[myexperiment], - plot_type=PlotType.QUANTILE, - beta=0.90, - normalize=False, - ) - plot_solvability_cdfs(experiments=[myexperiment], solve_tol=0.1) - - # Plots will be saved in the folder experiments/plots. - print("Finished. Plots can be found in experiments/plots folder.") - - -if __name__ == "__main__": - main() diff --git a/demo/demo_problems_solvers.py b/demo/demo_problems_solvers.py deleted file mode 100644 index 3e583cbcc..000000000 --- a/demo/demo_problems_solvers.py +++ /dev/null @@ -1,59 +0,0 @@ -"""Demo for the ProblemsSolvers class. - -This script is intended to help with debugging problems and solvers. -It create problem-solver groups (using the directory) and runs multiple -macroreplications of each problem-solver pair. -""" - -import sys -from pathlib import Path - -# Append the parent directory (simopt package) to the system path -sys.path.append(str(Path(__file__).resolve().parent.parent)) - -# Import the ProblemsSolvers class and other useful functions -from simopt.experiment_base import PlotType, ProblemsSolvers, plot_solvability_profiles - - -def main() -> None: - """Main function to run the demo script.""" - # !! When testing a new solver/problem, first go to directory.py. - # There you should add the import statement and an entry in the respective - # dictionary (or dictionaries). - # See directory.py for more details. - - # Specify the names of the solver and problem to test. - # These names are strings and should match those input to directory.py. - # Ex: - solver_names = ["RNDSRCH", "ASTRODF", "NELDMD"] - problem_names = ["CNTNEWS-1", "SAN-1"] - - # Initialize an instance of the experiment class. - mymetaexperiment = ProblemsSolvers( - solver_names=solver_names, problem_names=problem_names - ) - - # Write to log file. - mymetaexperiment.log_group_experiment_results() - - # Run a fixed number of macroreplications of each solver on each problem. - mymetaexperiment.run(n_macroreps=3) - - print("Post-processing results.") - # Run a fixed number of postreplications at all recommended solutions. - mymetaexperiment.post_replicate(n_postreps=50) - # Find an optimal solution x* for normalization. - mymetaexperiment.post_normalize(n_postreps_init_opt=50) - - print("Plotting results.") - # Produce basic plots of the solvers on the problems. - plot_solvability_profiles( - experiments=mymetaexperiment.experiments, plot_type=PlotType.CDF_SOLVABILITY - ) - - # Plots will be saved in the folder experiments/plots. - print("Finished. Plots can be found in experiments/plots folder.") - - -if __name__ == "__main__": - main() diff --git a/demo/demo_san-sscont-ironorecont_experiment.py b/demo/demo_san-sscont-ironorecont_experiment.py deleted file mode 100644 index d46cdc7d9..000000000 --- a/demo/demo_san-sscont-ironorecont_experiment.py +++ /dev/null @@ -1,413 +0,0 @@ -"""Demo for an Experiment with SAN, SSCONT, and IRONORECONT problems. - -This script is intended to help with a large experiment with -5 solvers (two versions of random search, ASTRO-DF, STRONG, and Nelder-Mead) -and 60 problems (20 unique instances of problems from -(s, S) inventory, iron ore, and stochastic activity network). -Produces plots appearing in the INFORMS Journal on Computing submission. -""" - -import sys -from pathlib import Path - -# Append the parent directory (simopt package) to the system path -sys.path.append(str(Path(__file__).resolve().parent.parent)) - -from simopt.experiment_base import ( - PlotType, - ProblemSolver, - plot_area_scatterplots, - plot_progress_curves, - plot_solvability_cdfs, - plot_solvability_profiles, - plot_terminal_progress, - plot_terminal_scatterplots, - post_normalize, - read_experiment_results, -) - - -def main() -> None: - """Main function to run the demo script.""" - # Problems factors used in experiments - # SAN - all_random_costs = [ - (1, 2, 2, 7, 17, 7, 2, 13, 1, 9, 18, 16, 7), - (2, 1, 10, 13, 15, 13, 12, 9, 12, 15, 5, 8, 10), - (2, 6, 7, 11, 13, 5, 1, 2, 2, 3, 15, 16, 13), - (3, 4, 18, 8, 10, 17, 14, 19, 15, 15, 7, 10, 6), - (3, 6, 9, 15, 1, 19, 1, 13, 2, 19, 6, 7, 14), - (4, 4, 2, 4, 5, 3, 19, 4, 17, 5, 16, 8, 8), - (5, 14, 14, 7, 10, 14, 16, 16, 8, 7, 14, 11, 17), - (7, 9, 17, 19, 1, 7, 4, 3, 9, 9, 13, 17, 14), - (8, 14, 1, 10, 18, 10, 17, 1, 2, 11, 1, 16, 6), - (8, 17, 5, 17, 4, 14, 2, 5, 5, 5, 8, 8, 16), - (10, 3, 2, 7, 15, 12, 7, 9, 12, 17, 9, 1, 2), - (10, 5, 17, 12, 13, 14, 6, 5, 19, 17, 1, 7, 17), - (10, 16, 10, 13, 9, 1, 1, 16, 5, 7, 7, 12, 15), - (11, 5, 15, 13, 15, 17, 12, 12, 16, 11, 18, 19, 2), - (12, 11, 13, 4, 15, 11, 16, 2, 7, 7, 13, 8, 3), - (13, 3, 14, 2, 15, 18, 17, 13, 5, 17, 17, 5, 18), - (14, 8, 8, 14, 8, 8, 18, 16, 8, 18, 12, 6, 7), - (14, 18, 7, 8, 13, 17, 10, 17, 19, 1, 13, 6, 12), - (15, 1, 2, 6, 14, 18, 11, 19, 15, 18, 15, 1, 4), - (18, 4, 19, 2, 13, 11, 9, 2, 17, 18, 11, 7, 14), - ] - - num_problems = len(all_random_costs) - - # SSCONT - demand_means = [25.0, 50.0, 100.0, 200.0, 400.0] - lead_means = [1.0, 3.0, 6.0, 9.0] - - # IRONORECONT - st_devs = [1, 2, 3, 4, 5] - holding_costs = [1, 100] - inven_stops = [1000, 10000] - - # RUNNING AND POST-PROCESSING EXPERIMENTS - num_macroreps = 10 - num_postreps = 100 - num_postnorms = 200 - - # Five solvers. - solvers = ["RNDSRCH_ss=10", "RNDSRCH_ss=50", "ASTRODF", "NELDMD", "STRONG"] - # Two versions of random search with varying sample sizes. - # rs_sample_sizes = [10, 50] - # ASTRODF factors - - # First problem: SAN - # Loop over problem instances. - for i in range(num_problems): - model_fixed_factors = {} - problem_fixed_factors = { - "budget": 10000, - "arc_costs": all_random_costs[i], - } - problem_rename = f"SAN-1_rc={all_random_costs[i]}" - - # Temporarily store experiments on the same problem for post-normalization. - experiments_same_problem = [] - solver_fixed_factors = {} - - for solver in solvers: - solver_name = solver - if solver == "RNDSRCH_ss=10": - solver_name = "RNDSRCH" - solver_fixed_factors = {"sample_size": 10} - elif solver == "RNDSRCH_ss=50": - solver_name = "RNDSRCH" - solver_fixed_factors = {"sample_size": 50} - - # Loop over solvers: - new_experiment = ProblemSolver( - solver_name=solver_name, - solver_rename=solver, - solver_fixed_factors=solver_fixed_factors, - problem_name="SAN-1", - problem_rename=problem_rename, - problem_fixed_factors=problem_fixed_factors, - model_fixed_factors=model_fixed_factors, - ) - # Run experiment with M. - new_experiment.run(n_macroreps=num_macroreps) - # Post replicate experiment with N. - new_experiment.post_replicate(n_postreps=num_postreps) - experiments_same_problem.append(new_experiment) - - # Post-normalize experiments with L. - # Provide NO proxies for f(x0), f(x*), or f(x). - post_normalize( - experiments=experiments_same_problem, - n_postreps_init_opt=num_postnorms, - ) - - # Second problem: SSCONT - for dm in demand_means: - for lm in lead_means: - model_fixed_factors = {"demand_mean": dm, "lead_mean": lm} - # Default budget for (s,S) inventory problem = 1000 replications. - # RS with sample size of 100 will get through only 10 iterations. - problem_fixed_factors = {"budget": 1000} - problem_rename = f"SSCONT-1_dm={dm}_lm={lm}" - - # Temporarily store experiments on the same problem for post-normalization. - experiments_same_problem = [] - solver_fixed_factors = {} - for solver in solvers: - solver_name = solver - if solver == "RNDSRCH_ss=10": - solver_name = "RNDSRCH" - solver_fixed_factors = {"sample_size": 10} - elif solver == "RNDSRCH_ss=50": - solver_name = "RNDSRCH" - solver_fixed_factors = {"sample_size": 50} - - # Loop over solvers: - new_experiment = ProblemSolver( - solver_name=solver_name, - solver_rename=solver, - solver_fixed_factors=solver_fixed_factors, - problem_name="SSCONT-1", - problem_rename=problem_rename, - problem_fixed_factors=problem_fixed_factors, - model_fixed_factors=model_fixed_factors, - ) - # Run experiment with M. - new_experiment.run(n_macroreps=num_macroreps) - # Post replicate experiment with N. - new_experiment.post_replicate(n_postreps=num_postreps) - experiments_same_problem.append(new_experiment) - - # Post-normalize experiments with L. - # Provide NO proxies for f(x0), f(x*), or f(x). - post_normalize( - experiments=experiments_same_problem, - n_postreps_init_opt=num_postnorms, - ) - - # Third problem: IRONORECONT - for sd in st_devs: - for hc in holding_costs: - for inv in inven_stops: - model_fixed_factors = { - "st_dev": sd, - "holding_cost": hc, - "inven_stop": inv, - } - problem_fixed_factors = {"budget": 1000} - problem_rename = f"IRONORECONT-1_sd={sd}_hc={hc}_inv={inv}" - - # Temporarily store experiments on the same problem for - # post-normalization. - experiments_same_problem = [] - solver_fixed_factors = {} - for solver in solvers: - solver_name = solver - if solver == "RNDSRCH_ss=10": - solver_name = "RNDSRCH" - solver_fixed_factors = {"sample_size": 10} - elif solver == "RNDSRCH_ss=50": - solver_name = "RNDSRCH" - solver_fixed_factors = {"sample_size": 50} - - # Loop over solvers: - new_experiment = ProblemSolver( - solver_name=solver_name, - solver_rename=solver, - solver_fixed_factors=solver_fixed_factors, - problem_name="IRONORECONT-1", - problem_rename=problem_rename, - problem_fixed_factors=problem_fixed_factors, - model_fixed_factors=model_fixed_factors, - ) - # Run experiment with M. - new_experiment.run(n_macroreps=num_macroreps) - # Post replicate experiment with N. - new_experiment.post_replicate(n_postreps=num_postreps) - experiments_same_problem.append(new_experiment) - - # Post-normalize experiments with L. - # Provide NO proxies for f(x0), f(x*), or f(x). - post_normalize( - experiments=experiments_same_problem, - n_postreps_init_opt=num_postnorms, - ) - - # LOAD DATA FROM .PICKLE FILES TO PREPARE FOR PLOTTING. - - # For plotting, "experiments" will be a list of list of ProblemSolver objects. - # outer list - indexed by solver - # inner list - index by problem - experiments = [] - - # Load .pickle files of past results. - # Load all experiments for a given solver, for all solvers. - # Load experiments belonging to the problems in: - problems = ["SAN", "SSCONT", "IRONORECONT"] - # problems = ["SAN"] - - for solver in solvers: - experiments_same_solver = [] - - solver_display = solver - if solver == "RNDSRCH_ss=10": - solver_display = "RS10" - elif solver == "RNDSRCH_ss=50": - solver_display = "RS50" - elif solver == "ASTRODF": - solver_display = "ASTRO-DF" - elif solver == "NELDMD": - solver_display = "Nelder-Mead" - - for problem in problems: - if problem == "SAN": - # Load SAN .pickle files - for i in range(num_problems): - problem_rename = f"{problem}-1_rc={all_random_costs[i]}" - file_name = f"{solver}_on_{problem_rename}" - # Load experiment. - new_experiment = read_experiment_results( - f"experiments/outputs/{file_name}.pickle" - ) - # Rename problem to produce nicer plot labels. - new_experiment.problem.name = ( - f"{problem}-1 with rc={all_random_costs[i]}" - ) - new_experiment.solver.name = solver_display - experiments_same_solver.append(new_experiment) - - elif problem == "SSCONT": - # Load SSCONT .pickle files - for dm in demand_means: - for lm in lead_means: - problem_rename = f"{problem}-1_dm={dm}_lm={lm}" - file_name = f"{solver}_on_{problem_rename}" - # Load experiment. - new_experiment = read_experiment_results( - f"experiments/outputs/{file_name}.pickle" - ) - # Rename problem to produce nicer plot labels. - new_experiment.problem.name = ( - f"{problem}-1 with " - rf"$\mu_D={round(dm)}$ and $\mu_L={round(lm)}$" - ) - new_experiment.solver.name = solver_display - experiments_same_solver.append(new_experiment) - - elif problem == "IRONORECONT": - # Load IRONORECONT .pickle files - for sd in st_devs: - for hc in holding_costs: - for inv in inven_stops: - problem_rename = f"{problem}-1_sd={sd}_hc={hc}_inv={inv}" - file_name = f"{solver}_on_{problem_rename}" - # Load experiment. - new_experiment = read_experiment_results( - f"experiments/outputs/{file_name}.pickle" - ) - # Rename problem to produce nicer plot labels. - new_experiment.problem.name = ( - f"{problem}-1 with " - rf"$\sigma={sd}$ and hc={hc} and inv={inv}" - ) - new_experiment.solver.name = solver_display - experiments_same_solver.append(new_experiment) - - experiments.append(experiments_same_solver) - - # PLOTTING - - n_solvers = len(experiments) - n_problems = len(experiments[0]) - - enable_confidence_intervals = True - alpha = 0.2 - - plot_solvability_profiles( - experiments, - plot_type=PlotType.CDF_SOLVABILITY, - solve_tol=alpha, - all_in_one=True, - plot_conf_ints=enable_confidence_intervals, - print_max_hw=enable_confidence_intervals, - ) - plot_solvability_profiles( - experiments, - plot_type=PlotType.QUANTILE_SOLVABILITY, - solve_tol=alpha, - beta=0.5, - all_in_one=True, - plot_conf_ints=enable_confidence_intervals, - print_max_hw=enable_confidence_intervals, - ) - plot_solvability_profiles( - experiments=experiments, - plot_type=PlotType.DIFF_CDF_SOLVABILITY, - solve_tol=alpha, - ref_solver="ASTRO-DF", - all_in_one=True, - plot_conf_ints=enable_confidence_intervals, - print_max_hw=enable_confidence_intervals, - ) - plot_solvability_profiles( - experiments=experiments, - plot_type=PlotType.DIFF_QUANTILE_SOLVABILITY, - solve_tol=alpha, - beta=0.5, - ref_solver="ASTRO-DF", - all_in_one=True, - plot_conf_ints=enable_confidence_intervals, - print_max_hw=enable_confidence_intervals, - ) - plot_area_scatterplots( - experiments, - all_in_one=True, - plot_conf_ints=enable_confidence_intervals, - print_max_hw=enable_confidence_intervals, - ) - plot_terminal_scatterplots(experiments, all_in_one=True) - - for i in range(n_problems): - plot_progress_curves( - [experiments[solver_idx][i] for solver_idx in range(n_solvers)], - plot_type=PlotType.MEAN, - all_in_one=True, - plot_conf_ints=enable_confidence_intervals, - print_max_hw=True, - ) - plot_terminal_progress( - [experiments[solver_idx][i] for solver_idx in range(n_solvers)], - plot_type=PlotType.VIOLIN, - normalize=True, - all_in_one=True, - ) - # plot_solvability_cdfs( - # [experiments[solver_idx][i] for solver_idx in range(n_solvers)], - # solve_tol=0.2, - # all_in_one=True, - # plot_CIs=True, - # print_max_hw=True, - # ) - - # Plots for mu_D = 400 and mu_L = 6 (appreared in the paper) - plot_progress_curves( - [experiments[solver_idx][0] for solver_idx in range(n_solvers)], - plot_type=PlotType.ALL, - all_in_one=True, - ) - - plot_progress_curves( - [experiments[solver_idx][0] for solver_idx in range(3, 4)], - plot_type=PlotType.ALL, - all_in_one=True, - normalize=False, - ) - - plot_progress_curves( - [experiments[solver_idx][0] for solver_idx in range(n_solvers)], - plot_type=PlotType.ALL, - all_in_one=True, - plot_conf_ints=True, - print_max_hw=False, - normalize=True, - ) - - plot_solvability_cdfs( - experiments=[experiments[solver_idx][0] for solver_idx in range(n_solvers)], - solve_tol=0.2, - all_in_one=True, - plot_conf_ints=True, - print_max_hw=False, - ) - - plot_terminal_progress( - [experiments[solver_idx][0] for solver_idx in range(n_solvers)], - plot_type=PlotType.VIOLIN, - normalize=False, - all_in_one=True, - ) - - -if __name__ == "__main__": - main() diff --git a/demo/demo_sscont_experiment.py b/demo/demo_sscont_experiment.py deleted file mode 100644 index 65948a879..000000000 --- a/demo/demo_sscont_experiment.py +++ /dev/null @@ -1,421 +0,0 @@ -"""Run five solvers on 20 versions of the (s, S) inventory problem. - -Produces plots appearing in the INFORMS Journal on Computing submission. -""" - -import sys -from pathlib import Path - -# Append the parent directory (simopt package) to the system path -sys.path.append(str(Path(__file__).resolve().parent.parent)) - -from simopt.experiment_base import ( - PlotType, - ProblemSolver, - plot_area_scatterplots, - plot_progress_curves, - plot_solvability_profiles, - plot_terminal_progress, - plot_terminal_scatterplots, - post_normalize, - read_experiment_results, -) - - -def main() -> None: - """Main function to run the demo script.""" - # Default values of the (s, S) model: - # "demand_mean": 100.0 - # "lead_mean": 6.0 - # "backorder_cost": 4.0 - # "holding_cost": 1.0 - # "fixed_cost": 36.0 - # "variable_cost": 2.0 - - # Create 20 problem instances by varying two factors, five levels. - demand_means = [25.0, 50.0, 100.0, 200.0, 400.0] - lead_means = [1.0, 3.0, 6.0, 9.0] - - demand_means = [400.0] - lead_means = [6.0] - - # Two versions of random search with varying sample sizes. - rs_sample_sizes = [10, 50] - - # RUNNING AND POST-PROCESSING EXPERIMENTS - num_macroreps = 10 - num_postreps = 100 - num_postnorms = 200 - # Loop over problem instances. - for dm in demand_means: - for lm in lead_means: - model_fixed_factors = {"demand_mean": dm, "lead_mean": lm} - # Default budget for (s,S) inventory problem = 1000 replications. - # RS with sample size of 100 will get through only 10 iterations. - problem_fixed_factors = {"budget": 1000} - problem_rename = f"SSCONT-1_dm={dm}_lm={lm}" - - # Temporarily store experiments on the same problem for post-normalization. - experiments_same_problem = [] - - # Loop over random search solvers. - for rs_ss in rs_sample_sizes: - solver_fixed_factors = {"sample_size": rs_ss} - solver_rename = f"RNDSRCH_ss={rs_ss}" - # Create experiment for the problem-solver pair. - new_experiment = ProblemSolver( - solver_name="RNDSRCH", - problem_name="SSCONT-1", - solver_rename=solver_rename, - problem_rename=problem_rename, - solver_fixed_factors=solver_fixed_factors, - problem_fixed_factors=problem_fixed_factors, - model_fixed_factors=model_fixed_factors, - ) - # Run experiment with M. - new_experiment.run(n_macroreps=num_macroreps) - # Post replicate experiment with N. - new_experiment.post_replicate(n_postreps=num_postreps) - experiments_same_problem.append(new_experiment) - - # Setup and run ASTRO-DF. - solver_fixed_factors = {} - new_experiment = ProblemSolver( - solver_name="ASTRODF", - problem_name="SSCONT-1", - problem_rename=problem_rename, - solver_fixed_factors=solver_fixed_factors, - problem_fixed_factors=problem_fixed_factors, - model_fixed_factors=model_fixed_factors, - ) - # Run experiment with M. - new_experiment.run(n_macroreps=num_macroreps) - # Post replicate experiment with N. - new_experiment.post_replicate(n_postreps=num_postreps) - experiments_same_problem.append(new_experiment) - - # Setup and run Nelder-Mead. - new_experiment = ProblemSolver( - solver_name="NELDMD", - problem_name="SSCONT-1", - problem_rename=problem_rename, - problem_fixed_factors=problem_fixed_factors, - model_fixed_factors=model_fixed_factors, - ) - # Run experiment withM. - new_experiment.run(n_macroreps=num_macroreps) - # Post replicate experiment with N. - new_experiment.post_replicate(n_postreps=num_postreps) - experiments_same_problem.append(new_experiment) - - # Setup and run STRONG.= - new_experiment = ProblemSolver( - solver_name="STRONG", - problem_name="SSCONT-1", - problem_rename=problem_rename, - problem_fixed_factors=problem_fixed_factors, - model_fixed_factors=model_fixed_factors, - ) - # Run experiment with M. - new_experiment.run(n_macroreps=num_macroreps) - # Post replicate experiment with N. - new_experiment.post_replicate(n_postreps=num_postreps) - experiments_same_problem.append(new_experiment) - - # Post-normalize experiments with L. - # Provide NO proxies for f(x0), f(x*), or f(x). - post_normalize( - experiments=experiments_same_problem, - n_postreps_init_opt=num_postnorms, - ) - - # LOAD DATA FROM .PICKLE FILES TO PREPARE FOR PLOTTING. - - # For plotting, "experiments" will be a list of list of Experiment objects. - # outer list - indexed by solver - # inner list - index by problem - experiments = [] - - # Load .pickle files of past results. - # Load all experiments for a given solver, for all solvers. - for rs_ss in rs_sample_sizes: - solver_rename = f"RNDSRCH_ss={rs_ss}" - experiments_same_solver = [] - for dm in demand_means: - for lm in lead_means: - problem_rename = f"SSCONT-1_dm={dm}_lm={lm}" - file_name = f"{solver_rename}_on_{problem_rename}" - # Load experiment. - new_experiment = read_experiment_results( - f"experiments/outputs/{file_name}.pickle" - ) - # Rename problem and solver to produce nicer plot labels. - new_experiment.solver.name = f"RS{rs_ss}" - new_experiment.problem.name = ( - rf"SSCONT-1 with $\mu_D={round(dm)}$ and $\mu_L={round(lm)}$" - ) - experiments_same_solver.append(new_experiment) - experiments.append(experiments_same_solver) - # Load ASTRO-DF results. - solver_rename = "ASTRODF" - experiments_same_solver = [] - for dm in demand_means: - for lm in lead_means: - problem_rename = f"SSCONT-1_dm={dm}_lm={lm}" - file_name = f"{solver_rename}_on_{problem_rename}" - # Load experiment. - new_experiment = read_experiment_results( - f"experiments/outputs/{file_name}.pickle" - ) - # Rename problem and solver to produce nicer plot labels. - new_experiment.solver.name = "ASTRO-DF" - new_experiment.problem.name = ( - rf"SSCONT-1 with $\mu_D={round(dm)}$ and $\mu_L={round(lm)}$" - ) - experiments_same_solver.append(new_experiment) - experiments.append(experiments_same_solver) - # Load Nelder-Mead results. - solver_rename = "NELDMD" - experiments_same_solver = [] - for dm in demand_means: - for lm in lead_means: - problem_rename = f"SSCONT-1_dm={dm}_lm={lm}" - file_name = f"{solver_rename}_on_{problem_rename}" - # Load experiment. - new_experiment = read_experiment_results( - f"experiments/outputs/{file_name}.pickle" - ) - # Rename problem and solver to produce nicer plot labels. - new_experiment.solver.name = "Nelder-Mead" - new_experiment.problem.name = ( - rf"SSCONT-1 with $\mu_D={round(dm)}$ and $\mu_L={round(lm)}$" - ) - experiments_same_solver.append(new_experiment) - experiments.append(experiments_same_solver) - # Load STRONG results. - solver_rename = "STRONG" - experiments_same_solver = [] - for dm in demand_means: - for lm in lead_means: - problem_rename = f"SSCONT-1_dm={dm}_lm={lm}" - file_name = f"{solver_rename}_on_{problem_rename}" - # Load experiment. - new_experiment = read_experiment_results( - f"experiments/outputs/{file_name}.pickle" - ) - # Rename problem and solver to produce nicer plot labels. - new_experiment.solver.name = "STRONG" - new_experiment.problem.name = ( - rf"SSCONT-1 with $\mu_D={round(dm)}$ and $\mu_L={round(lm)}$" - ) - experiments_same_solver.append(new_experiment) - experiments.append(experiments_same_solver) - # PLOTTING - - n_solvers = len(experiments) - n_problems = len(experiments[0]) - - plot_area_scatterplots( - experiments, all_in_one=True, plot_conf_ints=True, print_max_hw=True - ) - plot_solvability_profiles( - experiments, - plot_type=PlotType.CDF_SOLVABILITY, - solve_tol=0.1, - all_in_one=True, - plot_conf_ints=True, - print_max_hw=True, - ) - plot_solvability_profiles( - experiments, - plot_type=PlotType.QUANTILE_SOLVABILITY, - solve_tol=0.1, - beta=0.5, - all_in_one=True, - plot_conf_ints=True, - print_max_hw=True, - ) - plot_solvability_profiles( - experiments=experiments, - plot_type=PlotType.DIFF_CDF_SOLVABILITY, - solve_tol=0.1, - ref_solver="ASTRO-DF", - all_in_one=True, - plot_conf_ints=True, - print_max_hw=True, - ) - plot_solvability_profiles( - experiments=experiments, - plot_type=PlotType.DIFF_QUANTILE_SOLVABILITY, - solve_tol=0.1, - beta=0.5, - ref_solver="ASTRO-DF", - all_in_one=True, - plot_conf_ints=True, - print_max_hw=True, - ) - plot_terminal_scatterplots(experiments, all_in_one=True) - - for i in range(n_problems): - plot_progress_curves( - [experiments[solver_idx][i] for solver_idx in range(n_solvers)], - plot_type=PlotType.MEAN, - all_in_one=True, - plot_conf_ints=True, - print_max_hw=True, - normalize=False, - ) - plot_terminal_progress( - [experiments[solver_idx][i] for solver_idx in range(n_solvers)], - plot_type=PlotType.VIOLIN, - normalize=True, - all_in_one=True, - ) - - # from math import log, sqrt - # import matplotlib.pyplot as plt - # import numpy as np - # for mu_d in demand_means: - # for mu_l in lead_means: - # lq = mu_d * mu_l / 3 - # uq = mu_d * mu_l + 2 * sqrt(2 * mu_d**2 * mu_l) - # mu = (log(lq) + log(uq)) / 2 - # sigma = (log(uq) - mu) / 1.96 - # print( - # round(mu_d, 0), - # round(mu_l, 0), - # round(lq, 2), - # round(uq, 2), - # round(mu, 2), - # round(sigma, 2), - # ) - # s = np.random.lognormal(mu, sigma, 1000) - # plt.hist( - # s, - # density=True, - # alpha=0.5, - # label=str(round(mu_d, 0)) + "," + str(round(mu_l, 0)), - # bins=50, - # color="blue", - # ) - # plt.axvline(lq, color="red") - # plt.axvline(uq, color="red") - # plt.axis("tight") - # plt.legend() - # plt.show() - - # # Mean progress curves from all solvers on one problem. Problem instance 0. - # plot_progress_curves( - # experiments=[experiments[solver_idx][0] for solver_idx in range(n_solvers)], - # plot_type="mean", - # all_in_one=True, - # plot_CIs=True, - # print_max_hw=False, - # ) - - # # Mean progress curves from all solvers on one problem. Problem instance 22. - # plot_progress_curves( - # experiments=[experiments[solver_idx][22] for solver_idx in range(n_solvers)], - # plot_type="mean", - # all_in_one=True, - # plot_CIs=True, - # print_max_hw=False, - # ) - - # # Plot 0.9-quantile progress curves from all solvers on one problem. - # # Problem instance 0. - # plot_progress_curves( - # experiments=[experiments[solver_idx][0] for solver_idx in range(n_solvers)], - # plot_type="quantile", - # beta=0.9, - # all_in_one=True, - # plot_CIs=True, - # print_max_hw=False, - # ) - - # # Plot 0.9-quantile progress curves from all solvers on one problem. - # # Problem instance 22. - # plot_progress_curves( - # experiments=[experiments[solver_idx][22] for solver_idx in range(n_solvers)], - # plot_type="quantile", - # beta=0.9, - # all_in_one=True, - # plot_CIs=True, - # print_max_hw=False, - # ) - - # # Plot cdf of 0.2-solve times for all solvers on one problem. - # # Problem instance 0. - # plot_solvability_cdfs( - # experiments=[experiments[solver_idx][0] for solver_idx in range(n_solvers)], - # solve_tol=0.2, - # all_in_one=True, - # plot_CIs=True, - # print_max_hw=False, - # ) - - # # Plot cdf of 0.2-solve times for all solvers on one problem. - # # Problem instance 22. - # plot_solvability_cdfs( - # experiments=[experiments[solver_idx][22] for solver_idx in range(n_solvers)], - # solve_tol=0.2, - # all_in_one=True, - # plot_CIs=True, - # print_max_hw=False, - # ) - - # # Plot area scatterplots of all solvers on all problems. - # plot_area_scatterplots( - # experiments=experiments, all_in_one=True, plot_CIs=False, print_max_hw=False - # ) - - # # Plot cdf 0.1-solvability profiles of all solvers on all problems. - # plot_solvability_profiles( - # experiments=experiments, - # plot_type="cdf_solvability", - # all_in_one=True, - # plot_CIs=True, - # print_max_hw=False, - # solve_tol=0.1, - # ) - - # # Plot 0.5-quantile 0.1-solvability profiles of all solvers on all problems. - # plot_solvability_profiles( - # experiments=experiments, - # plot_type="quantile_solvability", - # all_in_one=True, - # plot_CIs=True, - # print_max_hw=False, - # solve_tol=0.1, - # beta=0.5, - # ) - - # # Plot difference of cdf 0.1-solvability profiles of all solvers on all problems. - # # Reference solver = ASTRO-DF. - # plot_solvability_profiles( - # experiments=experiments, - # plot_type="diff_cdf_solvability", - # all_in_one=True, - # plot_CIs=True, - # print_max_hw=False, - # solve_tol=0.1, - # ref_solver="ASTRO-DF", - # ) - - # # Plot difference of 0.5-quantile 0.1-solvability profiles of all solvers on - # # all problems. - # # Reference solver = ASTRO-DF. - # plot_solvability_profiles( - # experiments=experiments, - # plot_type="diff_quantile_solvability", - # all_in_one=True, - # plot_CIs=True, - # print_max_hw=False, - # solve_tol=0.1, - # beta=0.5, - # ref_solver="ASTRO-DF", - # ) - - -if __name__ == "__main__": - main() diff --git a/demo/exp_base_testing.py b/demo/exp_base_testing.py deleted file mode 100644 index d1525c1a3..000000000 --- a/demo/exp_base_testing.py +++ /dev/null @@ -1,64 +0,0 @@ -"""Demo for the ProblemsSolvers class. - -This script is intended to help with running a data-farming experiment on -a solver. It creates a design of solver factors and runs multiple -macroreplications at each version of the solver. Outputs are printed to a file. -""" - -import sys -from pathlib import Path - -# Append the parent directory (simopt package) to the system path -sys.path.append(str(Path(__file__).resolve().parent.parent)) - -from simopt.experiment_base import ProblemsSolvers - - -def main() -> None: - """Main function to run the demo script.""" - # Specify the name of the solver as it appears in directory.py - - solver_names = ["RNDSRCH", "RNDSRCH", "ASTRODF"] - - solver_renames = ["RND_test1", "RND_test2", "AST_test"] - - problem_names = ["EXAMPLE-1", "CNTNEWS-1"] - - problem_renames = ["EX_test", "NEWS_test"] - - experiment_name = "test_exp" - - solver_factors = [{}, {"sample_size": 2}, {}] - - problem_factors = [{}, {}] - - # Create ProblemsSovlers experiment with solver and model design - experiment = ProblemsSolvers( - solver_factors=solver_factors, - problem_factors=problem_factors, - solver_names=solver_names, - problem_names=problem_names, - solver_renames=solver_renames, - problem_renames=problem_renames, - experiment_name=experiment_name, - create_pair_pickles=True, - ) - - # check compatibility of selected solvers and problems - experiment.check_compatibility() - - # Run macroreplications at each design point. - experiment.run(2) - - # Postprocess the experimental results from each design point. - experiment.post_replicate(10) - experiment.post_normalize(10) - - # Record and log results - experiment.record_group_experiment_results() - experiment.log_group_experiment_results() - experiment.report_group_statistics() - - -if __name__ == "__main__": - main() diff --git a/notebooks/data_farming_model.ipynb b/notebooks/data_farming_model.ipynb new file mode 100644 index 000000000..32508a5c9 --- /dev/null +++ b/notebooks/data_farming_model.ipynb @@ -0,0 +1,192 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "0", + "metadata": {}, + "source": [ + "# Demo for Data Farming Models\n", + "\n", + "This script is intended to help with running a data-farming experiment on a simulation model.\n", + "\n", + "It creates a design of model factors and runs multiple replications at each configuration of the model.\n", + "\n", + "Outputs are printed to a file." + ] + }, + { + "cell_type": "markdown", + "id": "1", + "metadata": {}, + "source": [ + "## Append SimOpt Path\n", + "\n", + "Since the notebook is stored in simopt/notebooks, we need to append the parent simopt directory to the system path to import the necessary modules later on." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2", + "metadata": {}, + "outputs": [], + "source": [ + "import sys\n", + "from pathlib import Path\n", + "\n", + "# Take the current directory, find the parent, and add it to the system path\n", + "sys.path.append(str(Path.cwd().parent))" + ] + }, + { + "cell_type": "markdown", + "id": "3", + "metadata": {}, + "source": [ + "## Configuration Parameters\n", + "\n", + "This section defines the core parameters for the data farming experiment.\n", + "\n", + "To query model/problem/solver names, run `python scripts/list_directories.py`" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4", + "metadata": {}, + "outputs": [], + "source": [ + "model_abbr_name = \"CNTNEWS\"\n", + "\n", + "# Specify the names of the model factors (in order) that will be varied.\n", + "factor_headers = [\n", + " \"purchase_price\",\n", + " \"sales_price\",\n", + " \"salvage_price\",\n", + " \"order_quantity\",\n", + "]" + ] + }, + { + "cell_type": "markdown", + "id": "5", + "metadata": {}, + "source": [ + "## Design Settings\n", + "\n", + "You can either create a new design by providing a factor_settings_filename with factor bounds and discretization, OR use an existing design by providing a design_filename. Only one of these should be active at a time.\n", + "\n", + "### Factor Settings File\n", + "\n", + "If creating the design, provide the name of a .txt file containing the following:\n", + "- one row corresponding to each model factor being varied\n", + "- 3 columns:\n", + " 1. lower bound for factor value\n", + " 2. upper bound for factor value\n", + " 3. (integer) number of digits for discretizing values (e.g., 0 corresponds to integral values for the factor)\n", + "\n", + "### Design File\n", + "\n", + "If using an existing design, provide the name of a .txt file containing the design. The file should contain the following:\n", + "- one row corresponding to each design point\n", + "- the number of columns equal to the number of factors being varied\n", + "- each value in the table gives the value of the factor (col index) for the design point (row index)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6", + "metadata": {}, + "outputs": [], + "source": [ + "# factor_settings_filename = \"model_factor_settings\"\n", + "factor_settings_filename = None\n", + "\n", + "# design_filename = None\n", + "design_filename = \"model_factor_settings_design\"" + ] + }, + { + "cell_type": "markdown", + "id": "7", + "metadata": {}, + "source": [ + "## Replication and Output\n", + "These settings control the number of simulation runs per design point and whether common random numbers are used, as well as the name for the output file." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8", + "metadata": {}, + "outputs": [], + "source": [ + "# Specify a common number of replications to run of the model at each design point\n", + "n_reps = 10\n", + "\n", + "# Specify whether to use common random numbers across different versions of the model\n", + "crn_across_design_pts = True\n", + "\n", + "# Specify filename for outputs\n", + "output_filename = \"cntnews_data_farming_output\"" + ] + }, + { + "cell_type": "markdown", + "id": "9", + "metadata": {}, + "source": [ + "## Run Data Farming Experiment\n", + "This block initializes the DataFarmingExperiment object using the parameters defined above, executes the specified number of replications at each design point, and saves the results to a CSV file." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "10", + "metadata": {}, + "outputs": [], + "source": [ + "# Import experiment_base module, which contains functions for experimentation.\n", + "from simopt.data_farming_base import DataFarmingExperiment\n", + "\n", + "# Create DataFarmingExperiment object.\n", + "myexperiment = DataFarmingExperiment(\n", + " model_name=model_abbr_name,\n", + " factor_settings_file_name=factor_settings_filename,\n", + " factor_headers=factor_headers,\n", + " design_path=design_filename,\n", + " model_fixed_factors={},\n", + ")\n", + "\n", + "# Run replications and print results to file.\n", + "myexperiment.run(n_reps=n_reps, crn_across_design_pts=crn_across_design_pts)\n", + "myexperiment.print_to_csv(csv_file_name=output_filename)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "simopt", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.13.3" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/notebooks/data_farming_over_solver_and_problem.ipynb b/notebooks/data_farming_over_solver_and_problem.ipynb new file mode 100644 index 000000000..618a17a30 --- /dev/null +++ b/notebooks/data_farming_over_solver_and_problem.ipynb @@ -0,0 +1,245 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "0", + "metadata": {}, + "source": [ + "# Demo for Data Farming over Solvers and Problems.\n", + "\n", + "This script is intended to help with running a data-farming experiment on a solver.\n", + "\n", + "It creates a design of solver factors and runs multiple macroreplications at each version of the solver.\n", + "\n", + "Outputs are printed to a file." + ] + }, + { + "cell_type": "markdown", + "id": "1", + "metadata": {}, + "source": [ + "## Append SimOpt Path\n", + "\n", + "Since the notebook is stored in simopt/notebooks, we need to append the parent simopt directory to the system path to import the necessary modules later on." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2", + "metadata": {}, + "outputs": [], + "source": [ + "import sys\n", + "from pathlib import Path\n", + "\n", + "# Take the current directory, find the parent, and add it to the system path\n", + "sys.path.append(str(Path.cwd().parent))" + ] + }, + { + "cell_type": "markdown", + "id": "3", + "metadata": {}, + "source": [ + "## Configuration Parameters\n", + "\n", + "This section defines the core parameters for the data farming experiment.\n", + "\n", + "To query model/problem/solver names, run `python scripts/list_directories.py`" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4", + "metadata": {}, + "outputs": [], + "source": [ + "solver_abbr_name = \"ASTRODF\"\n", + "problem_abbr_name = \"CNTNEWS-1\"\n", + "model_abbr_name = \"CNTNEWS\"\n", + "\n", + "# Specify the names of the solver factors (in order) that will be varied.\n", + "solver_factor_headers = [\"eta_1\", \"eta_2\", \"lambda_min\"]\n", + "# Specify the names of the model factors (in order) that will be varied.\n", + "model_factor_headers = [\"purchase_price\", \"sales_price\", \"order_quantity\"]\n", + "\n", + "# OPTIONAL: factors chosen for cross design\n", + "# factor name followed by list containing factor values to cross design over\n", + "solver_cross_design_factors = {\"crn_across_solns\": [True, False]}\n", + "# model_cross_design_factors = {}\n", + "\n", + "# OPTIONAL: Provide additional overrides for default factors.\n", + "# If empty, default factor settings are used.\n", + "solver_fixed_factors = {}\n", + "model_fixed_factors = {\"salvage_price\": 5, \"Burr_c\": 1}\n", + "\n", + "# Provide the name of a file .txt locatated in the datafarming_experiments folder\n", + "# containing the following:\n", + "# - one row corresponding to each solver factor being varied\n", + "# - three columns:\n", + "# - first column: lower bound for factor value\n", + "# - second column: upper bound for factor value\n", + "# - third column: (integer) number of digits for discretizing values\n", + "# (e.g., 0 corresponds to integral values for the factor)\n", + "solver_factor_settings_filename = \"astrodf_testing\"\n", + "model_factor_settings_filename = \"testing_model_cntnews_1\"\n", + "\n", + "# Specify the number stacks to use for ruby design creation\n", + "solver_n_stacks = 1\n", + "problem_n_stacks = 1\n", + "\n", + "# Specify a common number of macroreplications of each unique solver/problem\n", + "# combination (i.e., the number of runs at each design point.)\n", + "n_macroreps = 3\n", + "\n", + "# Specify the number of postreplications to take at each recommended solution\n", + "# from each macroreplication at each design point.\n", + "n_postreps = 100\n", + "\n", + "# Specify the number of postreplications to take at x0 and x*.\n", + "n_postreps_init_opt = 200\n", + "\n", + "# Specify the CRN control for postreplications.\n", + "crn_across_budget = True # Default\n", + "crn_across_macroreps = False # Default\n", + "crn_across_init_opt = True # Default" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5", + "metadata": {}, + "outputs": [], + "source": [ + "from simopt.experiment_base import create_design\n", + "\n", + "# Create DataFarmingExperiment object for solver design\n", + "solver_design_list = create_design(\n", + " name=solver_abbr_name,\n", + " factor_headers=solver_factor_headers,\n", + " factor_settings_filename=solver_factor_settings_filename,\n", + " n_stacks=solver_n_stacks,\n", + " fixed_factors=solver_fixed_factors, # optional\n", + " cross_design_factors=solver_cross_design_factors, # optional\n", + ")\n", + "# Create DataFarmingExperiment object for model design\n", + "model_design_list = create_design(\n", + " name=model_abbr_name,\n", + " factor_headers=model_factor_headers,\n", + " factor_settings_filename=model_factor_settings_filename,\n", + " n_stacks=problem_n_stacks,\n", + " fixed_factors=model_fixed_factors, # optional\n", + " # cross_design_factors=model_cross_design_factors, #optional\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6", + "metadata": {}, + "outputs": [], + "source": [ + "from simopt.experiment_base import ProblemsSolvers\n", + "\n", + "# create solver name list for ProblemsSolvers\n", + "solver_names = []\n", + "for _ in range(len(solver_design_list)):\n", + " solver_names.append(solver_abbr_name)\n", + "\n", + "# create problem name list for ProblemsSolvers\n", + "problem_names = []\n", + "for _ in range(len(model_design_list)):\n", + " problem_names.append(problem_abbr_name)\n", + "\n", + "# Create ProblemsSolvers experiment with solver and model design\n", + "experiment = ProblemsSolvers(\n", + " solver_factors=solver_design_list,\n", + " problem_factors=model_design_list,\n", + " solver_names=solver_names,\n", + " problem_names=problem_names,\n", + ")\n", + "\n", + "# check compatibility of selected solvers and problems\n", + "experiment.check_compatibility()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7", + "metadata": {}, + "outputs": [], + "source": [ + "# Run macroreplications at each design point.\n", + "experiment.run(n_macroreps)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8", + "metadata": {}, + "outputs": [], + "source": [ + "# Postprocess the experimental results from each design point.\n", + "experiment.post_replicate(\n", + " n_postreps=n_postreps,\n", + " crn_across_budget=crn_across_budget,\n", + " crn_across_macroreps=crn_across_macroreps,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9", + "metadata": {}, + "outputs": [], + "source": [ + "experiment.post_normalize(\n", + " n_postreps_init_opt=n_postreps_init_opt,\n", + " crn_across_init_opt=crn_across_init_opt,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "10", + "metadata": {}, + "outputs": [], + "source": [ + "# Record and log results\n", + "experiment.record_group_experiment_results()\n", + "experiment.log_group_experiment_results()\n", + "experiment.report_group_statistics()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "simopt", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.13.3" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/notebooks/data_farming_problem.ipynb b/notebooks/data_farming_problem.ipynb new file mode 100644 index 000000000..19f9d4de9 --- /dev/null +++ b/notebooks/data_farming_problem.ipynb @@ -0,0 +1,224 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "0", + "metadata": {}, + "source": [ + "# Demo for Data Farming over Problems.\n", + "\n", + "This script is intended to help with running a data-farming experiment on a problem.\n", + "\n", + "It creates a design of problem factors and runs multiple macroreplications at each version of the problem.\n", + "\n", + "Outputs are printed to a file." + ] + }, + { + "cell_type": "markdown", + "id": "1", + "metadata": {}, + "source": [ + "## Append SimOpt Path\n", + "\n", + "Since the notebook is stored in simopt/notebooks, we need to append the parent simopt directory to the system path to import the necessary modules later on." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2", + "metadata": {}, + "outputs": [], + "source": [ + "import sys\n", + "from pathlib import Path\n", + "\n", + "# Take the current directory, find the parent, and add it to the system path\n", + "sys.path.append(str(Path.cwd().parent))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3", + "metadata": {}, + "outputs": [], + "source": [ + "problem_name = \"CNTNEWS-1\"\n", + "model_name = \"CNTNEWS\"\n", + "solver_names = [\"ASTRODF\", \"RNDSRCH\"]\n", + "\n", + "# Specify the names of the model factors (in order) that will be varied.\n", + "model_factor_headers = [\"purchase_price\", \"sales_price\", \"order_quantity\"]\n", + "\n", + "# OPTIONAL: factors chosen for cross design\n", + "# factor name followed by list containing factor values to cross design over\n", + "# model_cross_design_factors = {}\n", + "\n", + "# OPTIONAL: Provide additional overrides for model default factors.\n", + "# If empty, default factor settings are used.\n", + "model_fixed_factors = {\"salvage_price\": 5, \"Burr_c\": 1}\n", + "\n", + "# OPTIONAL: Provide additional overrides for solver default factors.\n", + "# If empty, default factor settings are used.\n", + "# list of dictionaries that provide fixed factors for problems when you don't want\n", + "# to use the default values. if you want to use all default values use empty\n", + "# dictionary, order must match problem names\n", + "solver_fixed_factors = [{\"eta_1\": 0.5, \"eta_2\": 0.4}, {\"sample_size\": 15}]\n", + "\n", + "# uncomment this version to run w/ only default solver factors\n", + "# sp;ver_fixed_factors = [{},{}]\n", + "\n", + "# Provide the name of a file .txt locatated in the datafarming_experiments folder\n", + "# containing the following:\n", + "# - one row corresponding to each solver factor being varied\n", + "# - three columns:\n", + "# - first column: lower bound for factor value\n", + "# - second column: upper bound for factor value\n", + "# - third column: (integer) number of digits for discretizing values\n", + "# (e.g., 0 corresponds to integral values for the factor)\n", + "model_factor_settings_filename = \"testing_model_cntnews_1\"\n", + "\n", + "# Specify the number stacks to use for ruby design creation\n", + "problem_n_stacks = 1\n", + "\n", + "# Specify a common number of macroreplications of each unique solver/problem\n", + "# combination (i.e., the number of runs at each design point.)\n", + "n_macroreps = 3\n", + "\n", + "# Specify the number of postreplications to take at each recommended solution\n", + "# from each macroreplication at each design point.\n", + "n_postreps = 100\n", + "\n", + "# Specify the number of postreplications to take at x0 and x*.\n", + "n_postreps_init_opt = 200\n", + "\n", + "# Specify the CRN control for postreplications.\n", + "crn_across_budget = True # Default\n", + "crn_across_macroreps = False # Default\n", + "crn_across_init_opt = True # Default" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4", + "metadata": {}, + "outputs": [], + "source": [ + "from simopt.experiment_base import create_design\n", + "\n", + "# Create DataFarmingExperiment object for model design\n", + "model_design_list = create_design(\n", + " name=model_name,\n", + " factor_headers=model_factor_headers,\n", + " factor_settings_filename=model_factor_settings_filename,\n", + " class_type=\"model\",\n", + " n_stacks=problem_n_stacks,\n", + " fixed_factors=model_fixed_factors, # optional\n", + " # cross_design_factors=model_cross_design_factors, #optional\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5", + "metadata": {}, + "outputs": [], + "source": [ + "from simopt.experiment_base import ProblemsSolvers\n", + "\n", + "# create problem name list for ProblemsSolvers\n", + "problem_names = []\n", + "for _ in range(len(model_design_list)):\n", + " problem_names.append(problem_name)\n", + "\n", + "# Create ProblemsSolvers experiment with solver and model design\n", + "experiment = ProblemsSolvers(\n", + " solver_factors=solver_fixed_factors,\n", + " problem_factors=model_design_list,\n", + " solver_names=solver_names,\n", + " problem_names=problem_names,\n", + ")\n", + "\n", + "# check compatibility of selected solvers and problems\n", + "experiment.check_compatibility()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6", + "metadata": {}, + "outputs": [], + "source": [ + "# Run macroreplications at each design point.\n", + "experiment.run(n_macroreps)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7", + "metadata": {}, + "outputs": [], + "source": [ + "# Postprocess the experimental results from each design point.\n", + "experiment.post_replicate(\n", + " n_postreps=n_postreps,\n", + " crn_across_budget=crn_across_budget,\n", + " crn_across_macroreps=crn_across_macroreps,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8", + "metadata": {}, + "outputs": [], + "source": [ + "experiment.post_normalize(\n", + " n_postreps_init_opt=n_postreps_init_opt,\n", + " crn_across_init_opt=crn_across_init_opt,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9", + "metadata": {}, + "outputs": [], + "source": [ + "# Record and log results\n", + "experiment.record_group_experiment_results()\n", + "experiment.log_group_experiment_results()\n", + "experiment.report_group_statistics()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "simopt", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.13.3" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/notebooks/data_farming_solver.ipynb b/notebooks/data_farming_solver.ipynb new file mode 100644 index 000000000..4adc2cbfe --- /dev/null +++ b/notebooks/data_farming_solver.ipynb @@ -0,0 +1,222 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "0", + "metadata": {}, + "source": [ + "# Demo for Data Farming over Solvers.\n", + "\n", + "This script is intended to help with running a data-farming experiment on a solver.\n", + "\n", + "It creates a design of solver factors and runs multiple macroreplications at each version of the solver.\n", + "\n", + "Outputs are printed to a file." + ] + }, + { + "cell_type": "markdown", + "id": "1", + "metadata": {}, + "source": [ + "## Append SimOpt Path\n", + "\n", + "Since the notebook is stored in simopt/notebooks, we need to append the parent simopt directory to the system path to import the necessary modules later on." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2", + "metadata": {}, + "outputs": [], + "source": [ + "import sys\n", + "from pathlib import Path\n", + "\n", + "# Take the current directory, find the parent, and add it to the system path\n", + "sys.path.append(str(Path.cwd().parent))" + ] + }, + { + "cell_type": "markdown", + "id": "3", + "metadata": {}, + "source": [ + "## Configuration Parameters\n", + "\n", + "This section defines the core parameters for the data farming experiment.\n", + "\n", + "To query model/problem/solver names, run `python scripts/list_directories.py`" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4", + "metadata": {}, + "outputs": [], + "source": [ + "solver_name = \"ASTRODF\"\n", + "# list of problem names for solver design to be run on\n", + "# (if more than one version of same problem, repeat name)\n", + "problem_names = [\"SSCONT-1\", \"SAN-1\"]\n", + "\n", + "# Specify the names of the sovler factors (in order) that will be varied.\n", + "solver_factor_headers = [\"eta_1\", \"eta_2\", \"lambda_min\"]\n", + "\n", + "# OPTIONAL: factors chosen for cross design\n", + "# factor name followed by list containing factor values to cross design over\n", + "solver_cross_design_factors = {\"crn_across_solns\": [True, False]}\n", + "\n", + "# OPTIONAL: Provide additional overrides for solver default factors.\n", + "# If empty, default factor settings are used.\n", + "solver_fixed_factors = {}\n", + "# OPTIONAL: Provide additional overrides for problem default factors.\n", + "# If empty, default factor settings are used.\n", + "# list of dictionaries that provide fixed factors for problems when you don't want\n", + "# to use the default values. if you want to use all default values use empty\n", + "# dictionary, order must match problem names\n", + "problem_fixed_factors = [\n", + " {\"budget\": 2000, \"demand_mean\": 90.0, \"fixed_cost\": 25},\n", + " {\"budget\": 500},\n", + "]\n", + "\n", + "# Provide the name of a file .txt locatated in the datafarming_experiments\n", + "# folder containing\n", + "# the following:\n", + "# - one row corresponding to each solver factor being varied\n", + "# - three columns:\n", + "# - first column: lower bound for factor value\n", + "# - second column: upper bound for factor value\n", + "# - third column: (integer) number of digits for discretizing values\n", + "# (e.g., 0 corresponds to integral values for the factor)\n", + "solver_factor_settings_filename = \"astrodf_testing\"\n", + "\n", + "# Specify the number stacks to use for ruby design creation\n", + "solver_n_stacks = 1\n", + "\n", + "# Specify a common number of macroreplications of each unique solver/problem\n", + "# combination (i.e., the number of runs at each design point.)\n", + "n_macroreps = 3\n", + "\n", + "# Specify the number of postreplications to take at each recommended solution\n", + "# from each macroreplication at each design point.\n", + "n_postreps = 100\n", + "\n", + "# Specify the number of postreplications to take at x0 and x*.\n", + "n_postreps_init_opt = 200\n", + "\n", + "# Specify the CRN control for postreplications.\n", + "crn_across_budget = True # Default\n", + "crn_across_macroreps = False # Default\n", + "crn_across_init_opt = True # Default" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5", + "metadata": {}, + "outputs": [], + "source": [ + "from simopt.experiment_base import create_design\n", + "\n", + "# Create DataFarmingExperiment object for sovler design\n", + "solver_design_list = create_design(\n", + " name=solver_name,\n", + " factor_headers=solver_factor_headers,\n", + " factor_settings_filename=solver_factor_settings_filename,\n", + " n_stacks=solver_n_stacks,\n", + " fixed_factors=solver_fixed_factors, # optional\n", + " cross_design_factors=solver_cross_design_factors, # optional\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6", + "metadata": {}, + "outputs": [], + "source": [ + "from simopt.experiment_base import ProblemsSolvers\n", + "\n", + "# create solver name list for ProblemsSolvers\n", + "solver_names = []\n", + "for _ in range(len(solver_design_list)):\n", + " solver_names.append(solver_name)\n", + "\n", + "# Create ProblemsSolvers experiment with solver and model design\n", + "experiment = ProblemsSolvers(\n", + " solver_factors=solver_design_list,\n", + " problem_factors=problem_fixed_factors,\n", + " solver_names=solver_names,\n", + " problem_names=problem_names,\n", + ")\n", + "\n", + "# check compatibility of selected solvers and problems\n", + "experiment.check_compatibility()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7", + "metadata": {}, + "outputs": [], + "source": [ + "# Run macroreplications at each design point.\n", + "experiment.run(n_macroreps)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8", + "metadata": {}, + "outputs": [], + "source": [ + "# Postprocess the experimental results from each design point.\n", + "experiment.post_replicate(\n", + " n_postreps=n_postreps,\n", + " crn_across_budget=crn_across_budget,\n", + " crn_across_macroreps=crn_across_macroreps,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9", + "metadata": {}, + "outputs": [], + "source": [ + "experiment.post_normalize(\n", + " n_postreps_init_opt=n_postreps_init_opt,\n", + " crn_across_init_opt=crn_across_init_opt,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "10", + "metadata": {}, + "outputs": [], + "source": [ + "# Record and log results\n", + "experiment.record_group_experiment_results()\n", + "experiment.log_group_experiment_results()\n", + "experiment.report_group_statistics()" + ] + } + ], + "metadata": { + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/notebooks/demo_model.ipynb b/notebooks/demo_model.ipynb new file mode 100644 index 000000000..bae7c7c71 --- /dev/null +++ b/notebooks/demo_model.ipynb @@ -0,0 +1,146 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "0", + "metadata": {}, + "source": [ + "# Demo for Model Debugging.\n", + "\n", + "This script is intended to help with debugging a model.\n", + "\n", + "It imports a model, initializes a model object with given factors, sets up pseudorandom number generators, and runs one or more replications." + ] + }, + { + "cell_type": "markdown", + "id": "1", + "metadata": {}, + "source": [ + "## Append SimOpt Path\n", + "\n", + "Since the notebook is stored in simopt/notebooks, we need to append the parent simopt directory to the system path to import the necessary modules later on." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2", + "metadata": {}, + "outputs": [], + "source": [ + "import sys\n", + "from pathlib import Path\n", + "\n", + "# Take the current directory, find the parent, and add it to the system path\n", + "sys.path.append(str(Path.cwd().parent))" + ] + }, + { + "cell_type": "markdown", + "id": "3", + "metadata": {}, + "source": [ + "## Model Configuration Parameters\n", + "\n", + "This section defines the core parameters for the model.\n", + "\n", + "To query model names, run `python scripts/list_directories.py`" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4", + "metadata": {}, + "outputs": [], + "source": [ + "# Import the model from the models directory\n", + "from simopt.models.mm1queue import MM1Queue\n", + "\n", + "# Set fixed factors\n", + "# Setting to {} will resort to all default values.\n", + "fixed_factors = {\"lambda\": 3.0, \"mu\": 8.0}\n", + "\n", + "# Initialize model\n", + "mymodel = MM1Queue(fixed_factors=fixed_factors)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5", + "metadata": {}, + "outputs": [], + "source": [ + "# Check that all factors describe a simulatable model.\n", + "# Check fixed factors individually.\n", + "for key, value in mymodel.factors.items():\n", + " print(\n", + " f\"The factor {key} is set as {value}. \"\n", + " f\"Is this simulatable? {bool(mymodel.check_simulatable_factor(key))}.\"\n", + " )\n", + "# Check all factors collectively.\n", + "print(\n", + " f\"Is the specified model simulatable? {bool(mymodel.check_simulatable_factors())}.\"\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6", + "metadata": {}, + "outputs": [], + "source": [ + "# Create a list of RNG objects for the simulation model to use when\n", + "# running replications.\n", + "\n", + "from mrg32k3a.mrg32k3a import MRG32k3a\n", + "\n", + "rng_list = [MRG32k3a(s_ss_sss_index=[0, ss, 0]) for ss in range(mymodel.n_rngs)]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7", + "metadata": {}, + "outputs": [], + "source": [ + "# Run a single replication of the model.\n", + "responses, gradients = mymodel.replicate(rng_list)\n", + "print(\"\\nFor a single replication:\")\n", + "print(\"\\nResponses:\")\n", + "for key, value in responses.items():\n", + " print(f\"\\t {key} is {value}.\")\n", + "print(\"\\n Gradients:\")\n", + "for outerkey in gradients:\n", + " print(f\"\\tFor the response {outerkey}:\")\n", + " for innerkey, value in gradients[outerkey].items():\n", + " print(f\"\\t\\tThe gradient w.r.t. {innerkey} is {value}.\")" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "simopt", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.13.2" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/notebooks/demo_plots.ipynb b/notebooks/demo_plots.ipynb new file mode 100644 index 000000000..0f5956080 --- /dev/null +++ b/notebooks/demo_plots.ipynb @@ -0,0 +1,183 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "0", + "metadata": {}, + "source": [ + "# Demo Plotting Script.\n", + "\n", + "This script is intended to help with debugging problems and solvers.\n", + "\n", + "It create a problem-solver pairing (using the directory) and runs multiple macroreplications of the solver on the problem." + ] + }, + { + "cell_type": "markdown", + "id": "1", + "metadata": {}, + "source": [ + "## Append SimOpt Path\n", + "\n", + "Since the notebook is stored in simopt/notebooks, we need to append the parent simopt directory to the system path to import the necessary modules later on." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2", + "metadata": {}, + "outputs": [], + "source": [ + "import sys\n", + "from pathlib import Path\n", + "\n", + "# Take the current directory, find the parent, and add it to the system path\n", + "sys.path.append(str(Path.cwd().parent))" + ] + }, + { + "cell_type": "markdown", + "id": "3", + "metadata": {}, + "source": [ + "## Configuration Parameters\n", + "\n", + "This section defines the core parameters for the plotting demo.\n", + "\n", + "To query model/problem/solver names, run `python scripts/list_directories.py`" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4", + "metadata": {}, + "outputs": [], + "source": [ + "solver_names = {\"RNDSRCH\", \"ASTRODF\", \"NELDMD\"}\n", + "problem_names = {\"SAN-1\"}\n", + "\n", + "num_macroreps = 10\n", + "num_postreps = 200\n", + "num_postreps_init_opt = 200" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5", + "metadata": {}, + "outputs": [], + "source": [ + "# Import the ProblemSolver class and other useful functions\n", + "from simopt.experiment_base import (\n", + " ProblemSolver,\n", + " post_normalize,\n", + " read_experiment_results,\n", + ")\n", + "\n", + "myexperiments: list[ProblemSolver] = []\n", + "for problem_name in problem_names:\n", + " problem_experiments = []\n", + " for solver_name in solver_names:\n", + " print(f\"Testing solver {solver_name} on problem {problem_name}.\")\n", + " # Initialize an instance of the experiment class.\n", + " myexperiment = ProblemSolver(solver_name, problem_name)\n", + "\n", + " # Run a fixed number of macroreplications of the solver on the problem.\n", + " myexperiment.run(n_macroreps=num_macroreps)\n", + "\n", + " # If the solver runs have already been performed, uncomment the\n", + " # following pair of lines (and uncommmen the myexperiment.run(...)\n", + " # line above) to read in results from a .pickle file.\n", + " file_name = solver_name + \"_on_\" + problem_name + \".pickle\"\n", + " myexperiment = read_experiment_results(file_name)\n", + "\n", + " print(\"Post-processing results.\")\n", + " # Run a fixed number of postreplications at all recommended solutions.\n", + " myexperiment.post_replicate(n_postreps=num_postreps)\n", + "\n", + " problem_experiments.append(myexperiment)\n", + "\n", + " # Find an optimal solution x* for normalization.\n", + " post_normalize(problem_experiments, n_postreps_init_opt=num_postreps_init_opt)\n", + "\n", + " # Add the experiments for this problem to the main list.\n", + " myexperiments.extend(problem_experiments)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6", + "metadata": {}, + "outputs": [], + "source": [ + "# Produce basic plots.\n", + "\n", + "from simopt.experiment_base import (\n", + " PlotType,\n", + " plot_terminal_progress,\n", + ")\n", + "\n", + "print(\"Plotting results...\")\n", + "\n", + "\n", + "def _print_path(plot_path: list[Path]) -> None:\n", + " print(f\"Plot saved to {plot_path!s}\")\n", + "\n", + "\n", + "_print_path(\n", + " plot_terminal_progress(\n", + " experiments=myexperiments, plot_type=PlotType.BOX, normalize=False\n", + " )\n", + ")\n", + "\n", + "_print_path(\n", + " plot_terminal_progress(\n", + " experiments=myexperiments, plot_type=PlotType.BOX, normalize=True\n", + " )\n", + ")\n", + "\n", + "_print_path(\n", + " plot_terminal_progress(\n", + " experiments=myexperiments,\n", + " plot_type=PlotType.VIOLIN,\n", + " normalize=False,\n", + " all_in_one=False,\n", + " )\n", + ")\n", + "\n", + "_print_path(\n", + " plot_terminal_progress(\n", + " experiments=myexperiments, plot_type=PlotType.VIOLIN, normalize=True\n", + " )\n", + ")\n", + "\n", + "print(\"Plotting complete!\")" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "simopt", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.13.2" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/notebooks/demo_problem.ipynb b/notebooks/demo_problem.ipynb new file mode 100644 index 000000000..c02caf16d --- /dev/null +++ b/notebooks/demo_problem.ipynb @@ -0,0 +1,201 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "0", + "metadata": {}, + "source": [ + "# Demo script for the Problem class.\n", + "\n", + "This script is intended to help with debugging a problem.\n", + "\n", + "It imports a problem, initializes a problem object with given factors, sets up pseudorandom number generators, and runs multiple replications at a given solution." + ] + }, + { + "cell_type": "markdown", + "id": "1", + "metadata": {}, + "source": [ + "## Append SimOpt Path\n", + "\n", + "Since the notebook is stored in simopt/notebooks, we need to append the parent simopt directory to the system path to import the necessary modules later on." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2", + "metadata": {}, + "outputs": [], + "source": [ + "import sys\n", + "from pathlib import Path\n", + "\n", + "# Take the current directory, find the parent, and add it to the system path\n", + "sys.path.append(str(Path.cwd().parent))" + ] + }, + { + "cell_type": "markdown", + "id": "3", + "metadata": {}, + "source": [ + "## Configuration Parameters\n", + "\n", + "This section defines the core parameters for the demo.\n", + "\n", + "To query model/problem/solver names, run `python scripts/list_directories.py`" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4", + "metadata": {}, + "outputs": [], + "source": [ + "# Import problem.\n", + "# from models. import \n", + "# Replace with name of .py file containing problem class.\n", + "# Replace with name of problem class.\n", + "# Fix factors of problem. Specify a dictionary of factors.\n", + "# fixed_factors = {} # Resort to all default values.\n", + "# Look at Problem class definition to get names of factors.\n", + "# Initialize an instance of the specified problem class.\n", + "# myproblem = (fixed_factors=fixed_factors)\n", + "# Replace with name of problem class.\n", + "# Initialize a solution x corresponding to the problem.\n", + "# x = (,)\n", + "# Look at the Problem class definition to identify the decision variables.\n", + "# x will be a tuple consisting of the decision variables.\n", + "# The following line does not need to be changed.\n", + "# mysolution = Solution(x, myproblem)\n", + "# Working example for CntNVMaxProfit problem.\n", + "# -----------------------------------------------\n", + "from simopt.models.cntnv import CntNVMaxProfit\n", + "\n", + "fixed_factors = {\"initial_solution\": (2,), \"budget\": 500}\n", + "myproblem = CntNVMaxProfit(fixed_factors=fixed_factors)\n", + "x = (3,)\n", + "\n", + "# -----------------------------------------------\n", + "\n", + "# Another working example for FacilitySizingTotalCost problem. (Commented out)\n", + "# This example has stochastic constraints.\n", + "# -----------------------------------------------\n", + "# from models.facilitysizing import FacilitySizingTotalCost\n", + "# fixed_factors = {\"epsilon\": 0.1}\n", + "# myproblem = FacilitySizingTotalCost(fixed_factors=fixed_factors)\n", + "# x = (200, 200, 200)\n", + "# mysolution = Solution(x, myproblem)\n", + "# -----------------------------------------------\n", + "\n", + "num_macroreps = 10" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5", + "metadata": {}, + "outputs": [], + "source": [ + "# Create and attach rngs to solution\n", + "from mrg32k3a.mrg32k3a import MRG32k3a\n", + "from simopt.base import Solution\n", + "\n", + "rng_list = [MRG32k3a(s_ss_sss_index=[0, ss, 0]) for ss in range(myproblem.model.n_rngs)]\n", + "mysolution = Solution(x, myproblem)\n", + "\n", + "mysolution.attach_rngs(rng_list, copy=False)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6", + "metadata": {}, + "outputs": [], + "source": [ + "# Simulate a fixed number of replications at the solution x.\n", + "myproblem.simulate(mysolution, num_macroreps=num_macroreps)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7", + "metadata": {}, + "outputs": [], + "source": [ + "# Print results to console.\n", + "print(\n", + " f\"Ran {num_macroreps} replications of the {myproblem.name} problem \"\n", + " f\"at solution x = {x}.\\n\"\n", + ")\n", + "obj_mean = round(mysolution.objectives_mean[0], 4)\n", + "obj_stderr = round(mysolution.objectives_stderr[0], 4)\n", + "print(f\"The mean objective estimate was {obj_mean} with standard error {obj_stderr}.\")\n", + "print(\"The individual observations of the objective were:\")\n", + "for idx in range(num_macroreps):\n", + " print(f\"\\t {round(mysolution.objectives[idx][0], 4)}\")\n", + "if myproblem.gradient_available:\n", + " print(\"\\nThe individual observations of the gradients of the objective were:\")\n", + " for idx in range(num_macroreps):\n", + " grads = mysolution.objectives_gradients[idx][0]\n", + " print(f\"\\t {[round(g, 4) for g in grads]}\")\n", + "else:\n", + " print(\"\\nThis problem has no known gradients.\")\n", + "if myproblem.n_stochastic_constraints > 0:\n", + " print(\n", + " f\"\\nThis problem has {myproblem.n_stochastic_constraints} stochastic \"\n", + " \"constraints of the form E[LHS] <= 0.\"\n", + " )\n", + " for stc_idx in range(myproblem.n_stochastic_constraints):\n", + " stoch_const_mean = mysolution.stoch_constraints_mean[stc_idx]\n", + " stoch_const_mean_round = round(stoch_const_mean, 4)\n", + " stoch_const_stderr = mysolution.stoch_constraints_stderr[stc_idx]\n", + " stoch_const_stderr_round = round(stoch_const_stderr, 4)\n", + " print(\n", + " f\"\\tFor stochastic constraint #{stc_idx + 1}, \"\n", + " f\"the mean of the LHS was {stoch_const_mean_round} \"\n", + " f\"with standard error {stoch_const_stderr_round}.\"\n", + " )\n", + " print(\"\\tThe observations of the LHSs were:\")\n", + " for idx in range(num_macroreps):\n", + " # Quick check to make sure the stochastic constraints are not None.\n", + " if mysolution.stoch_constraints is None:\n", + " error_msg = \"Stochastic constraints should not be None.\"\n", + " raise ValueError(error_msg)\n", + " # Print out the current stochastic constraint value.\n", + " stoch_const = mysolution.stoch_constraints[stc_idx][idx]\n", + " stoch_const_round = round(stoch_const, 4)\n", + " print(f\"\\t\\t {stoch_const_round}\")\n", + "else:\n", + " print(\"\\nThis problem has no stochastic constraints.\")" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "simopt", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.13.2" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/notebooks/demo_problem_solver.ipynb b/notebooks/demo_problem_solver.ipynb new file mode 100644 index 000000000..705cb1ee3 --- /dev/null +++ b/notebooks/demo_problem_solver.ipynb @@ -0,0 +1,201 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "0", + "metadata": {}, + "source": [ + "# Demo script for the ProblemSolver class.\n", + "\n", + "This script is intended to help with debugging problems and solvers.\n", + "\n", + "It create a problem-solver pairing (using the directory) and runs multiple macroreplications of the solver on the problem." + ] + }, + { + "cell_type": "markdown", + "id": "1", + "metadata": {}, + "source": [ + "## Append SimOpt Path\n", + "\n", + "Since the notebook is stored in simopt/notebooks, we need to append the parent simopt directory to the system path to import the necessary modules later on." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2", + "metadata": {}, + "outputs": [], + "source": [ + "import sys\n", + "from pathlib import Path\n", + "\n", + "# Take the current directory, find the parent, and add it to the system path\n", + "sys.path.append(str(Path.cwd().parent))" + ] + }, + { + "cell_type": "markdown", + "id": "3", + "metadata": {}, + "source": [ + "## Configuration Parameters\n", + "\n", + "This section defines the core parameters for the demo.\n", + "\n", + "To query model/problem/solver names, run `python scripts/list_directories.py`" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4", + "metadata": {}, + "outputs": [], + "source": [ + "# Specify the names of the solver and problem to test.\n", + "# Example with random search solver on continuous newsvendor problem:\n", + "# solver_abbr_name = \"RNDSRCH\"\n", + "# problem_abbr_name = \"CNTNEWS-1\"\n", + "solver_abbr_name = \"RNDSRCH\"\n", + "problem_abbr_name = \"CNTNEWS-1\"\n", + "\n", + "num_macroreps = 10\n", + "num_postreps = 200\n", + "num_postreps_init_opt = 200" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5", + "metadata": {}, + "outputs": [], + "source": [ + "# If the solver runs have already been performed, specify the file name path.\n", + "# If no path is specified, a new run will be performed.\n", + "file_name_path = None\n", + "\n", + "if file_name_path is None:\n", + " # Import the ProblemSolver class and other useful functions\n", + " from simopt.experiment_base import ProblemSolver\n", + "\n", + " # Initialize an instance of the experiment class.\n", + " myexperiment = ProblemSolver(solver_abbr_name, problem_abbr_name)\n", + "\n", + " # Run a fixed number of macroreplications of the solver on the problem.\n", + " myexperiment.run(n_macroreps=num_macroreps)\n", + "else:\n", + " # following pair of lines (and uncomment the myexperiment.run(...)\n", + " # line above) to read in results from a .pickle file.\n", + " # file_name_path = \"\"\n", + " from simopt.experiment_base import read_experiment_results\n", + "\n", + " myexperiment = read_experiment_results(file_name_path)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6", + "metadata": {}, + "outputs": [], + "source": [ + "# Run a fixed number of postreplications at all recommended solutions.\n", + "myexperiment.post_replicate(n_postreps=num_postreps)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7", + "metadata": {}, + "outputs": [], + "source": [ + "# Find an optimal solution x* for normalization.\n", + "from simopt.experiment_base import post_normalize\n", + "\n", + "post_normalize([myexperiment], n_postreps_init_opt=num_postreps_init_opt)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8", + "metadata": {}, + "outputs": [], + "source": [ + "# Log results.\n", + "myexperiment.log_experiment_results()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9", + "metadata": {}, + "outputs": [], + "source": [ + "# Produce basic plots.\n", + "\n", + "from simopt.experiment_base import (\n", + " PlotType,\n", + " plot_progress_curves,\n", + " plot_solvability_cdfs,\n", + ")\n", + "\n", + "print(\"Plotting results...\")\n", + "\n", + "\n", + "def _print_path(plot_path: list[Path]) -> None:\n", + " print(f\"Plot saved to {plot_path!s}\")\n", + "\n", + "\n", + "_print_path(\n", + " plot_progress_curves(\n", + " experiments=[myexperiment], plot_type=PlotType.ALL, normalize=False\n", + " )\n", + ")\n", + "_print_path(\n", + " plot_progress_curves(\n", + " experiments=[myexperiment], plot_type=PlotType.MEAN, normalize=False\n", + " )\n", + ")\n", + "_print_path(\n", + " plot_progress_curves(\n", + " experiments=[myexperiment],\n", + " plot_type=PlotType.QUANTILE,\n", + " beta=0.90,\n", + " normalize=False,\n", + " )\n", + ")\n", + "_print_path(plot_solvability_cdfs(experiments=[myexperiment], solve_tol=0.1))\n", + "\n", + "print(\"Plotting complete!\")" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "simopt", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.13.2" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/notebooks/demo_problems_solvers.ipynb b/notebooks/demo_problems_solvers.ipynb new file mode 100644 index 000000000..cfab5f31c --- /dev/null +++ b/notebooks/demo_problems_solvers.ipynb @@ -0,0 +1,170 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "0", + "metadata": {}, + "source": [ + "# Demo for the ProblemsSolvers class.\n", + "\n", + "This script is intended to help with debugging problems and solvers.\n", + "\n", + "It create problem-solver groups (using the directory) and runs multiple macroreplications of each problem-solver pair." + ] + }, + { + "cell_type": "markdown", + "id": "1", + "metadata": {}, + "source": [ + "## Append SimOpt Path\n", + "\n", + "Since the notebook is stored in simopt/notebooks, we need to append the parent simopt directory to the system path to import the necessary modules later on." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2", + "metadata": {}, + "outputs": [], + "source": [ + "import sys\n", + "from pathlib import Path\n", + "\n", + "# Take the current directory, find the parent, and add it to the system path\n", + "sys.path.append(str(Path.cwd().parent))" + ] + }, + { + "cell_type": "markdown", + "id": "3", + "metadata": {}, + "source": [ + "## Configuration Parameters\n", + "\n", + "This section defines the core parameters for the demo.\n", + "\n", + "To query model/problem/solver names, run `python scripts/list_directories.py`" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4", + "metadata": {}, + "outputs": [], + "source": [ + "# Specify the names of the solver(s) and problem(s) to test.\n", + "solver_abbr_names = [\"RNDSRCH\", \"ASTRODF\", \"NELDMD\"]\n", + "problem_abbr_names = [\"CNTNEWS-1\", \"SAN-1\"]\n", + "\n", + "num_macroreps = 3\n", + "num_postreps = 50\n", + "num_postreps_init_opt = 50" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5", + "metadata": {}, + "outputs": [], + "source": [ + "# Initialize an instance of the experiment class.\n", + "from simopt.experiment_base import ProblemsSolvers\n", + "\n", + "mymetaexperiment = ProblemsSolvers(\n", + " solver_names=solver_abbr_names, problem_names=problem_abbr_names\n", + ")\n", + "\n", + "# Write to log file.\n", + "mymetaexperiment.log_group_experiment_results()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6", + "metadata": {}, + "outputs": [], + "source": [ + "# Run a fixed number of macroreplications of each solver on each problem.\n", + "mymetaexperiment.run(n_macroreps=num_macroreps)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7", + "metadata": {}, + "outputs": [], + "source": [ + "print(\"Post-processing results.\")\n", + "# Run a fixed number of postreplications at all recommended solutions.\n", + "mymetaexperiment.post_replicate(n_postreps=num_postreps)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8", + "metadata": {}, + "outputs": [], + "source": [ + "print(\"Post-normalizing results.\")\n", + "\n", + "# Find an optimal solution x* for normalization.\n", + "mymetaexperiment.post_normalize(n_postreps_init_opt=num_postreps_init_opt)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9", + "metadata": {}, + "outputs": [], + "source": [ + "# Produce basic plots.\n", + "\n", + "from simopt.experiment_base import PlotType, plot_solvability_profiles\n", + "\n", + "print(\"Plotting results...\")\n", + "\n", + "\n", + "def _print_path(plot_path: list[Path]) -> None:\n", + " print(f\"Plot saved to {plot_path!s}\")\n", + "\n", + "\n", + "_print_path(\n", + " plot_solvability_profiles(\n", + " experiments=mymetaexperiment.experiments, plot_type=PlotType.CDF_SOLVABILITY\n", + " )\n", + ")\n", + "\n", + "print(\"Plotting complete!\")" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "simopt", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.13.2" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/notebooks/demo_san-sscont-ironorecont_experiment.ipynb b/notebooks/demo_san-sscont-ironorecont_experiment.ipynb new file mode 100644 index 000000000..02fa58537 --- /dev/null +++ b/notebooks/demo_san-sscont-ironorecont_experiment.ipynb @@ -0,0 +1,582 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "0", + "metadata": {}, + "source": [ + "# Demo for an Experiment with SAN, SSCONT, and IRONORECONT problems.\n", + "\n", + "This script is intended to help with a large experiment with\n", + "5 solvers (two versions of random search, ASTRO-DF, STRONG, and Nelder-Mead) and 60 problems (20 unique instances of problems from\n", + "(s, S) inventory, iron ore, and stochastic activity network).\n", + "\n", + "Produces plots appearing in the INFORMS Journal on Computing submission." + ] + }, + { + "cell_type": "markdown", + "id": "1", + "metadata": {}, + "source": [ + "## Append SimOpt Path\n", + "\n", + "Since the notebook is stored in simopt/notebooks, we need to append the parent simopt directory to the system path to import the necessary modules later on." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2", + "metadata": {}, + "outputs": [], + "source": [ + "import sys\n", + "from pathlib import Path\n", + "\n", + "# Take the current directory, find the parent, and add it to the system path\n", + "sys.path.append(str(Path.cwd().parent))" + ] + }, + { + "cell_type": "markdown", + "id": "3", + "metadata": {}, + "source": [ + "## Experiment Configuration Parameters\n", + "\n", + "Define the options used in the experiments." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4", + "metadata": {}, + "outputs": [], + "source": [ + "# Experiment Configuration\n", + "num_macroreps = 10\n", + "num_postreps = 100\n", + "num_postnorms = 200" + ] + }, + { + "cell_type": "markdown", + "id": "5", + "metadata": {}, + "source": [ + "## Helper Functions\n", + "\n", + "Helper functions to streamline the experiment process." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6", + "metadata": {}, + "outputs": [], + "source": [ + "# Classes for Problem and Solver info\n", + "# These don't need modified, they're just used to organize the problem and solver info.\n", + "class Problem:\n", + " \"\"\"Problem class to hold problem information.\"\"\"\n", + "\n", + " def __init__(\n", + " self,\n", + " name: str,\n", + " rename: str | None = None,\n", + " fixed_factors: dict | None = None,\n", + " model_fixed_factors: dict | None = None,\n", + " ) -> None:\n", + " \"\"\"Initialize the Problem with name, rename, and problem/model fixed factors.\"\"\"\n", + " self.name = name\n", + " self.rename = rename if rename else name\n", + " self.fixed_factors = fixed_factors if fixed_factors else {}\n", + " self.model_fixed_factors = model_fixed_factors if model_fixed_factors else {}\n", + "\n", + "\n", + "class Solver:\n", + " \"\"\"Solver class to hold solver information.\"\"\"\n", + "\n", + " def __init__(\n", + " self, name: str, rename: str | None = None, fixed_factors: dict | None = None\n", + " ) -> None:\n", + " \"\"\"Initialize the Solver with name, rename, and solver fixed factors.\"\"\"\n", + " self.name = name\n", + " self.rename = rename if rename else name\n", + " self.fixed_factors = fixed_factors if fixed_factors else {}" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7", + "metadata": {}, + "outputs": [], + "source": [ + "from simopt.experiment_base import ProblemSolver, post_normalize\n", + "\n", + "\n", + "# Function to run an experiment with the given problems and solvers.\n", + "def run_experiment(\n", + " problems: list[Problem],\n", + " solvers: list[Solver],\n", + ") -> list[list[ProblemSolver]]:\n", + " \"\"\"Run the Experiment for the given problems and solvers.\n", + "\n", + " Args:\n", + " problems: List of Problem instances.\n", + " solvers: List of Solver instances.\n", + "\n", + " Returns:\n", + " List[list[ProblemSolver]]: A list of lists containing ProblemSolver instances,\n", + " grouped by problem.\n", + " \"\"\"\n", + " all_experiments = []\n", + " for problem_idx, problem in enumerate(problems):\n", + " print(\n", + " f\"Running Problem {problem_idx + 1}/{len(problems)}: {problem.rename}...\",\n", + " end=\"\",\n", + " flush=True,\n", + " )\n", + " # Keep track of experiments on the same problem for post-processing.\n", + " experiments_same_problem = []\n", + " # Create each ProblemSolver and run it.\n", + " for solver in solvers:\n", + " new_experiment = ProblemSolver(\n", + " solver_name=solver.name,\n", + " solver_rename=solver.rename,\n", + " solver_fixed_factors=solver.fixed_factors,\n", + " problem_name=problem.name,\n", + " problem_rename=problem.rename,\n", + " problem_fixed_factors=problem.fixed_factors,\n", + " model_fixed_factors=problem.model_fixed_factors,\n", + " )\n", + " # Run and post-replicate the experiment.\n", + " new_experiment.run(n_macroreps=num_macroreps)\n", + " new_experiment.post_replicate(n_postreps=num_postreps)\n", + " experiments_same_problem.append(new_experiment)\n", + "\n", + " # Post-normalize experiments with L.\n", + " # Provide NO proxies for f(x0), f(x*), or f(x).\n", + " post_normalize(\n", + " experiments=experiments_same_problem,\n", + " n_postreps_init_opt=num_postnorms,\n", + " )\n", + " all_experiments.append(experiments_same_problem)\n", + " print(\"Done.\")\n", + " print(\"All experiments completed.\")\n", + " return all_experiments" + ] + }, + { + "cell_type": "markdown", + "id": "8", + "metadata": {}, + "source": [ + "## Problem/Solver Configuration Parameters\n", + "\n", + "Define the problems and solvers used in the experiments." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9", + "metadata": {}, + "outputs": [], + "source": [ + "# Solvers to use in the experiment.\n", + "# Includes two versions of random search with varying sample sizes.\n", + "# The rename will be used in the plots to differentiate them.\n", + "solvers = [\n", + " Solver(name=\"RNDSRCH\", rename=\"RNDSRCH_ss=10\", fixed_factors={\"sample_size\": 10}),\n", + " Solver(name=\"RNDSRCH\", rename=\"RNDSRCH_ss=50\", fixed_factors={\"sample_size\": 50}),\n", + " Solver(name=\"ASTRODF\"),\n", + " Solver(name=\"NELDMD\"),\n", + " Solver(name=\"STRONG\"),\n", + "]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "10", + "metadata": {}, + "outputs": [], + "source": [ + "# Problem 1 - SAN\n", + "# Configure the problem\n", + "all_random_costs = [\n", + " (1, 2, 2, 7, 17, 7, 2, 13, 1, 9, 18, 16, 7),\n", + " (2, 1, 10, 13, 15, 13, 12, 9, 12, 15, 5, 8, 10),\n", + " (2, 6, 7, 11, 13, 5, 1, 2, 2, 3, 15, 16, 13),\n", + " (3, 4, 18, 8, 10, 17, 14, 19, 15, 15, 7, 10, 6),\n", + " (3, 6, 9, 15, 1, 19, 1, 13, 2, 19, 6, 7, 14),\n", + " (4, 4, 2, 4, 5, 3, 19, 4, 17, 5, 16, 8, 8),\n", + " (5, 14, 14, 7, 10, 14, 16, 16, 8, 7, 14, 11, 17),\n", + " (7, 9, 17, 19, 1, 7, 4, 3, 9, 9, 13, 17, 14),\n", + " (8, 14, 1, 10, 18, 10, 17, 1, 2, 11, 1, 16, 6),\n", + " (8, 17, 5, 17, 4, 14, 2, 5, 5, 5, 8, 8, 16),\n", + " (10, 3, 2, 7, 15, 12, 7, 9, 12, 17, 9, 1, 2),\n", + " (10, 5, 17, 12, 13, 14, 6, 5, 19, 17, 1, 7, 17),\n", + " (10, 16, 10, 13, 9, 1, 1, 16, 5, 7, 7, 12, 15),\n", + " (11, 5, 15, 13, 15, 17, 12, 12, 16, 11, 18, 19, 2),\n", + " (12, 11, 13, 4, 15, 11, 16, 2, 7, 7, 13, 8, 3),\n", + " (13, 3, 14, 2, 15, 18, 17, 13, 5, 17, 17, 5, 18),\n", + " (14, 8, 8, 14, 8, 8, 18, 16, 8, 18, 12, 6, 7),\n", + " (14, 18, 7, 8, 13, 17, 10, 17, 19, 1, 13, 6, 12),\n", + " (15, 1, 2, 6, 14, 18, 11, 19, 15, 18, 15, 1, 4),\n", + " (18, 4, 19, 2, 13, 11, 9, 2, 17, 18, 11, 7, 14),\n", + "]\n", + "num_problems = len(all_random_costs)\n", + "\n", + "# Create all the problem variants.\n", + "SAN_problems = [\n", + " Problem(\n", + " name=\"SAN-1\",\n", + " rename=f\"SAN-1_rc={costs}\",\n", + " fixed_factors={\n", + " \"budget\": 10000,\n", + " \"arc_costs\": costs,\n", + " },\n", + " )\n", + " for costs in all_random_costs\n", + "]\n", + "\n", + "# Run the experiment for SAN problems.\n", + "SAN_experiments = run_experiment(\n", + " problems=SAN_problems,\n", + " solvers=solvers,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "11", + "metadata": {}, + "outputs": [], + "source": [ + "# Problem 2 - SSCONT\n", + "# Configure the problem\n", + "demand_means = [25.0, 50.0, 100.0, 200.0, 400.0]\n", + "lead_means = [1.0, 3.0, 6.0, 9.0]\n", + "\n", + "# Create all the problem variants.\n", + "SSCONT_problems = [\n", + " Problem(\n", + " name=\"SSCONT-1\",\n", + " rename=f\"SSCONT-1_dm={dm}_lm={lm}\",\n", + " fixed_factors={\"budget\": 1000},\n", + " model_fixed_factors={\n", + " \"demand_mean\": dm,\n", + " \"lead_mean\": lm,\n", + " },\n", + " )\n", + " for dm in demand_means\n", + " for lm in lead_means\n", + "]\n", + "\n", + "# Run the experiment for SSCONT problems.\n", + "SSCONT_experiments = run_experiment(\n", + " problems=SSCONT_problems,\n", + " solvers=solvers,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "12", + "metadata": {}, + "outputs": [], + "source": [ + "# Problem 3 - IRONORECONT\n", + "# Configure the problem\n", + "st_devs = [1, 2, 3, 4, 5]\n", + "holding_costs = [1, 100]\n", + "inven_stops = [1000, 10000]\n", + "\n", + "# Create all the problem variants.\n", + "IRONORECONT_problems = [\n", + " Problem(\n", + " name=\"IRONORECONT-1\",\n", + " rename=f\"IRONORECONT-1_sd={sd}_hc={hc}_inv={inv}\",\n", + " fixed_factors={\"budget\": 1000},\n", + " model_fixed_factors={\n", + " \"st_dev\": sd,\n", + " \"holding_cost\": hc,\n", + " \"inven_stop\": inv,\n", + " },\n", + " )\n", + " for sd in st_devs\n", + " for hc in holding_costs\n", + " for inv in inven_stops\n", + "]\n", + "\n", + "# Run the experiment for IRONORECONT problems.\n", + "IRONORECONT_experiments = run_experiment(\n", + " problems=IRONORECONT_problems,\n", + " solvers=solvers,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "13", + "metadata": {}, + "outputs": [], + "source": [ + "# Combine the experiments into a list of lists, where the outer list\n", + "# contains all the experiments for a single solver, and the inner lists\n", + "# contain the ProblemSolver instances associated with that solver.\n", + "all_experiments = []\n", + "all_experiments.extend(SAN_experiments)\n", + "all_experiments.extend(SSCONT_experiments)\n", + "all_experiments.extend(IRONORECONT_experiments)\n", + "\n", + "experiment_dict = {}\n", + "for exp_problem_list in all_experiments:\n", + " for experiment in exp_problem_list:\n", + " # Use the solver name as the key and append the ProblemSolver instance.\n", + " key = experiment.solver.name\n", + " if key not in experiment_dict:\n", + " experiment_dict[key] = []\n", + " experiment_dict[key].append(experiment)\n", + "# Turn the dictionary into a list of lists.\n", + "experiments = list(experiment_dict.values())" + ] + }, + { + "cell_type": "markdown", + "id": "14", + "metadata": {}, + "source": [ + "## Plotting Settings\n", + "\n", + "Define the plotting settings for the experiments." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "15", + "metadata": {}, + "outputs": [], + "source": [ + "from simopt.experiment_base import (\n", + " PlotType,\n", + " plot_area_scatterplots,\n", + " plot_progress_curves,\n", + " plot_solvability_cdfs,\n", + " plot_solvability_profiles,\n", + " plot_terminal_progress,\n", + " plot_terminal_scatterplots,\n", + ")\n", + "\n", + "enable_confidence_intervals = True\n", + "alpha = 0.2" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "16", + "metadata": {}, + "outputs": [], + "source": [ + "plot_solvability_profiles(\n", + " experiments,\n", + " plot_type=PlotType.CDF_SOLVABILITY,\n", + " solve_tol=alpha,\n", + " all_in_one=True,\n", + " plot_conf_ints=enable_confidence_intervals,\n", + " print_max_hw=enable_confidence_intervals,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "17", + "metadata": {}, + "outputs": [], + "source": [ + "plot_solvability_profiles(\n", + " experiments,\n", + " plot_type=PlotType.QUANTILE_SOLVABILITY,\n", + " solve_tol=alpha,\n", + " beta=0.5,\n", + " all_in_one=True,\n", + " plot_conf_ints=enable_confidence_intervals,\n", + " print_max_hw=enable_confidence_intervals,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "18", + "metadata": {}, + "outputs": [], + "source": [ + "plot_solvability_profiles(\n", + " experiments=experiments,\n", + " plot_type=PlotType.DIFFERENCE_OF_CDF_SOLVABILITY,\n", + " solve_tol=alpha,\n", + " ref_solver=\"ASTRODF\",\n", + " all_in_one=True,\n", + " plot_conf_ints=enable_confidence_intervals,\n", + " print_max_hw=enable_confidence_intervals,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "19", + "metadata": {}, + "outputs": [], + "source": [ + "plot_solvability_profiles(\n", + " experiments=experiments,\n", + " plot_type=PlotType.DIFFERENCE_OF_QUANTILE_SOLVABILITY,\n", + " solve_tol=alpha,\n", + " beta=0.5,\n", + " ref_solver=\"ASTRODF\",\n", + " all_in_one=True,\n", + " plot_conf_ints=enable_confidence_intervals,\n", + " print_max_hw=enable_confidence_intervals,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "20", + "metadata": {}, + "outputs": [], + "source": [ + "plot_area_scatterplots(\n", + " experiments,\n", + " all_in_one=True,\n", + " plot_conf_ints=enable_confidence_intervals,\n", + " print_max_hw=enable_confidence_intervals,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "21", + "metadata": {}, + "outputs": [], + "source": [ + "plot_terminal_scatterplots(experiments, all_in_one=True)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "22", + "metadata": {}, + "outputs": [], + "source": [ + "n_solvers = len(solvers)\n", + "\n", + "for i in range(len(experiments[0])):\n", + " plot_progress_curves(\n", + " [experiments[solver_idx][i] for solver_idx in range(n_solvers)],\n", + " plot_type=PlotType.MEAN,\n", + " all_in_one=True,\n", + " plot_conf_ints=enable_confidence_intervals,\n", + " print_max_hw=True,\n", + " )\n", + " plot_terminal_progress(\n", + " [experiments[solver_idx][i] for solver_idx in range(n_solvers)],\n", + " plot_type=PlotType.VIOLIN,\n", + " normalize=True,\n", + " all_in_one=True,\n", + " )\n", + " # plot_solvability_cdfs(\n", + " # [experiments[solver_idx][i] for solver_idx in range(n_solvers)],\n", + " # solve_tol=0.2,\n", + " # all_in_one=True,\n", + " # plot_CIs=True,\n", + " # print_max_hw=True,\n", + " # )" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "23", + "metadata": {}, + "outputs": [], + "source": [ + "# Plots for mu_D = 400 and mu_L = 6 (appreared in the paper)\n", + "plot_progress_curves(\n", + " [experiments[solver_idx][0] for solver_idx in range(n_solvers)],\n", + " plot_type=PlotType.ALL,\n", + " all_in_one=True,\n", + ")\n", + "\n", + "plot_progress_curves(\n", + " [experiments[solver_idx][0] for solver_idx in range(3, 4)],\n", + " plot_type=PlotType.ALL,\n", + " all_in_one=True,\n", + " normalize=False,\n", + ")\n", + "\n", + "plot_progress_curves(\n", + " [experiments[solver_idx][0] for solver_idx in range(n_solvers)],\n", + " plot_type=PlotType.ALL,\n", + " all_in_one=True,\n", + " plot_conf_ints=True,\n", + " print_max_hw=False,\n", + " normalize=True,\n", + ")\n", + "\n", + "plot_solvability_cdfs(\n", + " experiments=[experiments[solver_idx][0] for solver_idx in range(n_solvers)],\n", + " solve_tol=0.2,\n", + " all_in_one=True,\n", + " plot_conf_ints=True,\n", + " print_max_hw=False,\n", + ")\n", + "\n", + "plot_terminal_progress(\n", + " [experiments[solver_idx][0] for solver_idx in range(n_solvers)],\n", + " plot_type=PlotType.VIOLIN,\n", + " normalize=False,\n", + " all_in_one=True,\n", + ")" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "simopt", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.13.3" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/notebooks/demo_sscont_experiment.ipynb b/notebooks/demo_sscont_experiment.ipynb new file mode 100644 index 000000000..149dd568e --- /dev/null +++ b/notebooks/demo_sscont_experiment.ipynb @@ -0,0 +1,428 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "0", + "metadata": {}, + "source": [ + "# Run five solvers on 20 versions of the (s, S) inventory problem.\n", + "\n", + "Produces plots appearing in the INFORMS Journal on Computing submission." + ] + }, + { + "cell_type": "markdown", + "id": "1", + "metadata": {}, + "source": [ + "## Append SimOpt Path\n", + "\n", + "Since the notebook is stored in simopt/notebooks, we need to append the parent simopt directory to the system path to import the necessary modules later on." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2", + "metadata": {}, + "outputs": [], + "source": [ + "import sys\n", + "from pathlib import Path\n", + "\n", + "# Take the current directory, find the parent, and add it to the system path\n", + "sys.path.append(str(Path.cwd().parent))" + ] + }, + { + "cell_type": "markdown", + "id": "3", + "metadata": {}, + "source": [ + "## Experiment Configuration Parameters\n", + "\n", + "Define the options used in the experiments." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4", + "metadata": {}, + "outputs": [], + "source": [ + "# Experiment Configuration\n", + "num_macroreps = 10\n", + "num_postreps = 100\n", + "num_postnorms = 200" + ] + }, + { + "cell_type": "markdown", + "id": "5", + "metadata": {}, + "source": [ + "## Helper Functions\n", + "\n", + "Helper functions to streamline the experiment process." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6", + "metadata": {}, + "outputs": [], + "source": [ + "# Classes for Problem and Solver info\n", + "# These don't need modified, they're just used to organize the problem and solver info.\n", + "class Problem:\n", + " \"\"\"Problem class to hold problem information.\"\"\"\n", + "\n", + " def __init__(\n", + " self,\n", + " name: str,\n", + " rename: str | None = None,\n", + " fixed_factors: dict | None = None,\n", + " model_fixed_factors: dict | None = None,\n", + " ) -> None:\n", + " \"\"\"Initialize the Problem with name, rename, and problem/model fixed factors.\"\"\"\n", + " self.name = name\n", + " self.rename = rename if rename else name\n", + " self.fixed_factors = fixed_factors if fixed_factors else {}\n", + " self.model_fixed_factors = model_fixed_factors if model_fixed_factors else {}\n", + "\n", + "\n", + "class Solver:\n", + " \"\"\"Solver class to hold solver information.\"\"\"\n", + "\n", + " def __init__(\n", + " self, name: str, rename: str | None = None, fixed_factors: dict | None = None\n", + " ) -> None:\n", + " \"\"\"Initialize the Solver with name, rename, and solver fixed factors.\"\"\"\n", + " self.name = name\n", + " self.rename = rename if rename else name\n", + " self.fixed_factors = fixed_factors if fixed_factors else {}" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7", + "metadata": {}, + "outputs": [], + "source": [ + "from simopt.experiment_base import ProblemSolver, post_normalize\n", + "\n", + "\n", + "# Function to run an experiment with the given problems and solvers.\n", + "def run_experiment(\n", + " problems: list[Problem],\n", + " solvers: list[Solver],\n", + ") -> list[list[ProblemSolver]]:\n", + " \"\"\"Run the Experiment for the given problems and solvers.\n", + "\n", + " Args:\n", + " problems: List of Problem instances.\n", + " solvers: List of Solver instances.\n", + "\n", + " Returns:\n", + " List[list[ProblemSolver]]: A list of lists containing ProblemSolver instances,\n", + " grouped by problem.\n", + " \"\"\"\n", + " all_experiments = []\n", + " for problem_idx, problem in enumerate(problems):\n", + " print(\n", + " f\"Running Problem {problem_idx + 1}/{len(problems)}: {problem.rename}...\",\n", + " end=\"\",\n", + " flush=True,\n", + " )\n", + " # Keep track of experiments on the same problem for post-processing.\n", + " experiments_same_problem = []\n", + " # Create each ProblemSolver and run it.\n", + " for solver in solvers:\n", + " new_experiment = ProblemSolver(\n", + " solver_name=solver.name,\n", + " solver_rename=solver.rename,\n", + " solver_fixed_factors=solver.fixed_factors,\n", + " problem_name=problem.name,\n", + " problem_rename=problem.rename,\n", + " problem_fixed_factors=problem.fixed_factors,\n", + " model_fixed_factors=problem.model_fixed_factors,\n", + " )\n", + " # Run and post-replicate the experiment.\n", + " new_experiment.run(n_macroreps=num_macroreps)\n", + " new_experiment.post_replicate(n_postreps=num_postreps)\n", + " experiments_same_problem.append(new_experiment)\n", + "\n", + " # Post-normalize experiments with L.\n", + " # Provide NO proxies for f(x0), f(x*), or f(x).\n", + " post_normalize(\n", + " experiments=experiments_same_problem,\n", + " n_postreps_init_opt=num_postnorms,\n", + " )\n", + " all_experiments.append(experiments_same_problem)\n", + " print(\"Done.\")\n", + " print(\"All experiments completed.\")\n", + " return all_experiments" + ] + }, + { + "cell_type": "markdown", + "id": "8", + "metadata": {}, + "source": [ + "## Problem/Solver Configuration Parameters\n", + "\n", + "Define the problems and solvers used in the experiments." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9", + "metadata": {}, + "outputs": [], + "source": [ + "# Solvers to use in the experiment.\n", + "# Includes two versions of random search with varying sample sizes.\n", + "# The rename will be used in the plots to differentiate them.\n", + "solvers = [\n", + " Solver(name=\"RNDSRCH\", rename=\"RNDSRCH_ss=10\", fixed_factors={\"sample_size\": 10}),\n", + " Solver(name=\"RNDSRCH\", rename=\"RNDSRCH_ss=50\", fixed_factors={\"sample_size\": 50}),\n", + " Solver(name=\"ASTRODF\"),\n", + " Solver(name=\"NELDMD\"),\n", + " Solver(name=\"STRONG\"),\n", + "]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "10", + "metadata": {}, + "outputs": [], + "source": [ + "# Configure the problem\n", + "demand_means = [25.0, 50.0, 100.0, 200.0, 400.0]\n", + "lead_means = [1.0, 3.0, 6.0, 9.0]\n", + "\n", + "# Create all the problem variants.\n", + "SSCONT_problems = [\n", + " Problem(\n", + " name=\"SSCONT-1\",\n", + " rename=f\"SSCONT-1_dm={dm}_lm={lm}\",\n", + " fixed_factors={\"budget\": 1000},\n", + " model_fixed_factors={\n", + " \"demand_mean\": dm,\n", + " \"lead_mean\": lm,\n", + " },\n", + " )\n", + " for dm in demand_means\n", + " for lm in lead_means\n", + "]\n", + "\n", + "# Run the experiment for SSCONT problems.\n", + "SSCONT_experiments = run_experiment(\n", + " problems=SSCONT_problems,\n", + " solvers=solvers,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "11", + "metadata": {}, + "outputs": [], + "source": [ + "# Combine the experiments into a list of lists, where the outer list\n", + "# contains all the experiments for a single solver, and the inner lists\n", + "# contain the ProblemSolver instances associated with that solver.\n", + "\n", + "experiment_dict = {}\n", + "for exp_problem_list in SSCONT_experiments:\n", + " for experiment in exp_problem_list:\n", + " # Use the solver name as the key and append the ProblemSolver instance.\n", + " key = experiment.solver.name\n", + " if key not in experiment_dict:\n", + " experiment_dict[key] = []\n", + " experiment_dict[key].append(experiment)\n", + "# Turn the dictionary into a list of lists.\n", + "experiments = list(experiment_dict.values())" + ] + }, + { + "cell_type": "markdown", + "id": "12", + "metadata": {}, + "source": [ + "## Plotting Settings\n", + "\n", + "Define the plotting settings for the experiments." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "13", + "metadata": {}, + "outputs": [], + "source": [ + "from simopt.experiment_base import (\n", + " PlotType,\n", + " plot_area_scatterplots,\n", + " plot_progress_curves,\n", + " plot_solvability_profiles,\n", + " plot_terminal_progress,\n", + " plot_terminal_scatterplots,\n", + ")\n", + "\n", + "enable_confidence_intervals = True\n", + "alpha = 0.2" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "14", + "metadata": {}, + "outputs": [], + "source": [ + "plot_area_scatterplots(\n", + " experiments, all_in_one=True, plot_conf_ints=True, print_max_hw=True\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "15", + "metadata": {}, + "outputs": [], + "source": [ + "plot_solvability_profiles(\n", + " experiments,\n", + " plot_type=PlotType.CDF_SOLVABILITY,\n", + " solve_tol=0.1,\n", + " all_in_one=True,\n", + " plot_conf_ints=True,\n", + " print_max_hw=True,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "16", + "metadata": {}, + "outputs": [], + "source": [ + "plot_solvability_profiles(\n", + " experiments,\n", + " plot_type=PlotType.QUANTILE_SOLVABILITY,\n", + " solve_tol=0.1,\n", + " beta=0.5,\n", + " all_in_one=True,\n", + " plot_conf_ints=True,\n", + " print_max_hw=True,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "17", + "metadata": {}, + "outputs": [], + "source": [ + "plot_solvability_profiles(\n", + " experiments=experiments,\n", + " plot_type=PlotType.DIFFERENCE_OF_CDF_SOLVABILITY,\n", + " solve_tol=0.1,\n", + " ref_solver=\"ASTRODF\",\n", + " all_in_one=True,\n", + " plot_conf_ints=True,\n", + " print_max_hw=True,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "18", + "metadata": {}, + "outputs": [], + "source": [ + "plot_solvability_profiles(\n", + " experiments=experiments,\n", + " plot_type=PlotType.DIFFERENCE_OF_QUANTILE_SOLVABILITY,\n", + " solve_tol=0.1,\n", + " beta=0.5,\n", + " ref_solver=\"ASTRODF\",\n", + " all_in_one=True,\n", + " plot_conf_ints=True,\n", + " print_max_hw=True,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "19", + "metadata": {}, + "outputs": [], + "source": [ + "plot_terminal_scatterplots(experiments, all_in_one=True)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "20", + "metadata": {}, + "outputs": [], + "source": [ + "n_problems = len(SSCONT_problems)\n", + "n_solvers = len(experiments)\n", + "\n", + "for i in range(n_problems):\n", + " plot_progress_curves(\n", + " [experiments[solver_idx][i] for solver_idx in range(n_solvers)],\n", + " plot_type=PlotType.MEAN,\n", + " all_in_one=True,\n", + " plot_conf_ints=True,\n", + " print_max_hw=True,\n", + " normalize=False,\n", + " )\n", + " plot_terminal_progress(\n", + " [experiments[solver_idx][i] for solver_idx in range(n_solvers)],\n", + " plot_type=PlotType.VIOLIN,\n", + " normalize=True,\n", + " all_in_one=True,\n", + " )" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "simopt", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.13.3" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/notebooks/exp_base_testing.ipynb b/notebooks/exp_base_testing.ipynb new file mode 100644 index 000000000..8cb2bfe62 --- /dev/null +++ b/notebooks/exp_base_testing.ipynb @@ -0,0 +1,173 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "0", + "metadata": {}, + "source": [ + "# Demo for the ProblemsSolvers class.\n", + "\n", + "This script is intended to help with running a data-farming experiment on a solver.\n", + "\n", + "It creates a design of solver factors and runs multiple macroreplications at each version of the solver.\n", + "\n", + "Outputs are printed to a file." + ] + }, + { + "cell_type": "markdown", + "id": "1", + "metadata": {}, + "source": [ + "## Append SimOpt Path\n", + "\n", + "Since the notebook is stored in simopt/notebooks, we need to append the parent simopt directory to the system path to import the necessary modules later on." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2", + "metadata": {}, + "outputs": [], + "source": [ + "import sys\n", + "from pathlib import Path\n", + "\n", + "# Take the current directory, find the parent, and add it to the system path\n", + "sys.path.append(str(Path.cwd().parent))" + ] + }, + { + "cell_type": "markdown", + "id": "3", + "metadata": {}, + "source": [ + "## Configuration Parameters\n", + "\n", + "This section defines the core parameters for the demo.\n", + "\n", + "To query model/problem/solver names, run `python scripts/list_directories.py`" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4", + "metadata": {}, + "outputs": [], + "source": [ + "# Specify the name of the solver as it appears in directory.py\n", + "\n", + "solver_names = [\"RNDSRCH\", \"RNDSRCH\", \"ASTRODF\"]\n", + "\n", + "solver_renames = [\"RND_test1\", \"RND_test2\", \"AST_test\"]\n", + "\n", + "problem_names = [\"EXAMPLE-1\", \"CNTNEWS-1\"]\n", + "\n", + "problem_renames = [\"EX_test\", \"NEWS_test\"]\n", + "\n", + "experiment_name = \"test_exp\"\n", + "\n", + "solver_factors = [{}, {\"sample_size\": 2}, {}]\n", + "\n", + "problem_factors = [{}, {}]\n", + "\n", + "num_macroreps = 2\n", + "num_postreps = 10\n", + "num_postreps_init_opt = 10" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5", + "metadata": {}, + "outputs": [], + "source": [ + "# Create ProblemsSolvers experiment with solver and model design\n", + "from simopt.experiment_base import ProblemsSolvers\n", + "\n", + "experiment = ProblemsSolvers(\n", + " solver_factors=solver_factors,\n", + " problem_factors=problem_factors,\n", + " solver_names=solver_names,\n", + " problem_names=problem_names,\n", + " solver_renames=solver_renames,\n", + " problem_renames=problem_renames,\n", + " experiment_name=experiment_name,\n", + " create_pair_pickles=True,\n", + ")\n", + "\n", + "# check compatibility of selected solvers and problems\n", + "experiment.check_compatibility()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6", + "metadata": {}, + "outputs": [], + "source": [ + "# Run macroreplications at each design point.\n", + "experiment.run(n_macroreps=num_macroreps)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7", + "metadata": {}, + "outputs": [], + "source": [ + "# Postprocess the experimental results from each design point.\n", + "experiment.post_replicate(n_postreps=num_postreps)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8", + "metadata": {}, + "outputs": [], + "source": [ + "experiment.post_normalize(n_postreps_init_opt=num_postreps_init_opt)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9", + "metadata": {}, + "outputs": [], + "source": [ + "# Record and log results\n", + "experiment.record_group_experiment_results()\n", + "experiment.log_group_experiment_results()\n", + "experiment.report_group_statistics()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "simopt", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.13.2" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/notebooks/hello_simopt.ipynb b/notebooks/hello_simopt.ipynb index 8871a4ea8..821dd1084 100644 --- a/notebooks/hello_simopt.ipynb +++ b/notebooks/hello_simopt.ipynb @@ -1,70 +1,136 @@ { - "cells": [ - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# ruff: noqa: E402\n", - "import os\n", - "\n", - "os.chdir(\"../\") # Move one level up to import simopt\n", - "\n", - "# Import experiment_base module, which contains functions for experimentation.\n", - "import simopt.experiment_base" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Run 10 macroreplications of ASTRO-DF on the continuous newsvendor problem.\n", - "myexperiment = simopt.experiment_base.ProblemSolver(\"ASTRODF\", \"CNTNEWS-1\")\n", - "myexperiment.run(n_macroreps=10)\n", - "\n", - "# Post-process the results.\n", - "myexperiment.post_replicate(n_postreps=200)\n", - "simopt.experiment_base.post_normalize(\n", - " experiments=[myexperiment], n_postreps_init_opt=200\n", - ")\n", - "\n", - "# Record the results and plot the mean progress curve.\n", - "myexperiment.log_experiment_results()\n", - "simopt.experiment_base.plot_progress_curves(\n", - " experiments=[myexperiment],\n", - " plot_type=simopt.experiment_base.PlotType.MEAN,\n", - " normalize=False,\n", - ")" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Hello SimOpt!\n", + "\n", + "This script is intended to be an introduction to the world of SimOpt.\n", + "\n", + "The script creates a simple experiment and goes through the process of running, post-processing, post-normalizing, and recording the results." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Append SimOpt Path\n", + "\n", + "Since the notebook is stored in simopt/notebooks, we need to append the parent simopt directory to the system path to import the necessary modules later on." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import sys\n", + "from pathlib import Path\n", + "\n", + "# Take the current directory, find the parent, and add it to the system path\n", + "sys.path.append(str(Path.cwd().parent))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Configuration Parameters\n", + "\n", + "This section defines the core parameters for the demo.\n", + "\n", + "To query model/problem/solver names, run `python scripts/list_directories.py`" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Create an instance of the ProblemSolver class for the given solver and problem\n", + "solver_name = \"ASTRODF\"\n", + "problem_name = \"CNTNEWS-1\"\n", + "\n", + "num_macroreps = 10\n", + "num_postreps = 200\n", + "num_postreps_init_opt = 200" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from simopt.experiment_base import ProblemSolver\n", + "\n", + "myexperiment = ProblemSolver(solver_name, problem_name)\n", + "# Run the experiment with a specified number of macro-repetitions\n", + "myexperiment.run(n_macroreps=num_macroreps)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Post-process the results\n", + "myexperiment.post_replicate(n_postreps=num_postreps)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Normalize the results\n", + "from simopt.experiment_base import post_normalize\n", + "\n", + "post_normalize(experiments=[myexperiment], n_postreps_init_opt=num_postreps_init_opt)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Record the results and plot the mean progress curve.\n", + "from simopt.experiment_base import PlotType, plot_progress_curves\n", + "\n", + "myexperiment.log_experiment_results()\n", + "plot_progress_curves(\n", + " experiments=[myexperiment],\n", + " plot_type=PlotType.MEAN,\n", + " normalize=False,\n", + ")" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "simopt", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.13.3" + } }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.13.5" - }, - "vscode": { - "interpreter": { - "hash": "efc4a4ec95e6d859810f241e65d87235b82d6a4c5c4a60fd317e05e0763f2d90" - } - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} + "nbformat": 4, + "nbformat_minor": 4 +} \ No newline at end of file diff --git a/notebooks/load_solver_design.ipynb b/notebooks/load_solver_design.ipynb new file mode 100644 index 000000000..714b1986d --- /dev/null +++ b/notebooks/load_solver_design.ipynb @@ -0,0 +1,228 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "0", + "metadata": {}, + "source": [ + "# Demo for Loading Solver Design Points.\n", + "\n", + "This script is intended to help with loading a design of solver factors to run on one or more problems.\n", + "\n", + "If more than one problem are used a cross-design of problems will be run.\n", + "\n", + "The design file should be a txt or csv file with headers as\n", + "factor names and each row representing an individual design point.\n", + "\n", + "Design files generated by the simopt GUI can also be run using this script.\n", + "\n", + "Outputs are printed to the experiments folder in simopt." + ] + }, + { + "cell_type": "markdown", + "id": "1", + "metadata": {}, + "source": [ + "## Append SimOpt Path\n", + "\n", + "Since the notebook is stored in simopt/notebooks, we need to append the parent simopt directory to the system path to import the necessary modules later on." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2", + "metadata": {}, + "outputs": [], + "source": [ + "import sys\n", + "from pathlib import Path\n", + "\n", + "# Take the current directory, find the parent, and add it to the system path\n", + "sys.path.append(str(Path.cwd().parent))" + ] + }, + { + "cell_type": "markdown", + "id": "3", + "metadata": {}, + "source": [ + "## Configuration Parameters\n", + "\n", + "This section defines the core parameters for the demo.\n", + "\n", + "To query model/problem/solver names, run `python scripts/list_directories.py`" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4", + "metadata": {}, + "outputs": [], + "source": [ + "from pathlib import Path\n", + "\n", + "# run this script in the terminal from the simopt directory\n", + "# name of solver that design was created on\n", + "solver_name = \"RNDSRCH\"\n", + "# list of problem names for solver design to be run on\n", + "# (if more than one version of same problem, repeat name)\n", + "problem_names = [\n", + " \"SSCONT-1\",\n", + " \"SAN-1\",\n", + "]\n", + "\n", + "# name of file containing design points (csv or excel): column headers must exactly\n", + "# match names of solver factors w/ each row representing a design point\n", + "# (can also use csv's generated by GUI)\n", + "\n", + "design_filename = Path(\"data_farming_experiments\", \"RNDSRCH_design.csv\")\n", + "\n", + "# list of dictionaries that provide fixed factors for problems when you don't want\n", + "# to use the default values, if you want to use all default values use empty\n", + "# dictionary, order must match problem names\n", + "problem_fixed_factors = [\n", + " {\"budget\": 2000, \"demand_mean\": 90.0, \"fixed_cost\": 25},\n", + " {\"budget\": 500},\n", + "]\n", + "\n", + "# uncomment this version to run w/ only default problem factors\n", + "# problem_fixed_factors = [{},{}]\n", + "\n", + "# use this dictionary to change any default solver factors that were not included\n", + "# in the design\n", + "solver_fixed_factors = {}\n", + "\n", + "# number of macroreplication to run and each solver design point\n", + "n_macroreps = 2\n", + "# number of post replications to run on each macro replication\n", + "n_postreps = 100\n", + "# number of normalization postreplications to run at initial solution and optimal\n", + "# solution\n", + "n_postreps_init_opt = 200" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5", + "metadata": {}, + "outputs": [], + "source": [ + "# turn design file into df & retrive dp information\n", + "import pandas as pd\n", + "\n", + "design_table = pd.read_csv(design_filename)\n", + "\n", + "design_factor_names = design_table.columns.tolist()\n", + "\n", + "# remove GUI columns from list if present (ignore if not using design files\n", + "# generated by GUI)\n", + "design_factor_names.remove(\"Design #\")\n", + "design_factor_names.remove(\"Solver Name\")\n", + "design_factor_names.remove(\"Design Type\")\n", + "design_factor_names.remove(\"Number Stacks\")\n", + "\n", + "dp_list = [] # list of all design points\n", + "\n", + "for _, row in design_table.iterrows():\n", + " dp = {} # dictionary of current dp\n", + " for factor in design_factor_names:\n", + " dp[factor] = row[factor]\n", + " dp_list.append(dp)\n", + "\n", + "# add fixed solver factors to dps\n", + "for fixed_factor in solver_fixed_factors:\n", + " for dp in dp_list:\n", + " dp[fixed_factor] = solver_fixed_factors[fixed_factor]\n", + "\n", + "n_dp = len(dp_list)\n", + "solver_names = []\n", + "for _ in range(n_dp):\n", + " solver_names.append(solver_name)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6", + "metadata": {}, + "outputs": [], + "source": [ + "from simopt.experiment_base import ProblemsSolvers\n", + "\n", + "experiment = ProblemsSolvers(\n", + " solver_factors=dp_list,\n", + " problem_factors=problem_fixed_factors,\n", + " solver_names=solver_names,\n", + " problem_names=problem_names,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7", + "metadata": {}, + "outputs": [], + "source": [ + "experiment.run(n_macroreps)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8", + "metadata": {}, + "outputs": [], + "source": [ + "experiment.post_replicate(n_postreps)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9", + "metadata": {}, + "outputs": [], + "source": [ + "experiment.post_normalize(n_postreps_init_opt)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "10", + "metadata": {}, + "outputs": [], + "source": [ + "experiment.record_group_experiment_results()\n", + "experiment.log_group_experiment_results()\n", + "experiment.report_group_statistics()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "simopt", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.13.3" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/scripts/list_directories.py b/scripts/list_directories.py new file mode 100644 index 000000000..e9301c763 --- /dev/null +++ b/scripts/list_directories.py @@ -0,0 +1,54 @@ +"""Print the names and classes of all models, problems, and solvers. + +Since the directory is dynamically generated, this script can be used to quickly +check its contents and lookup the abbreviated names of classes. This is especially +useful when it is unclear if the class's name has been overridden from the default. +""" + +# Import standard libraries +import sys +from pathlib import Path + +# Append the parent directory (simopt package) to the system path +sys.path.append(str(Path(__file__).resolve().parent.parent)) + +# Import simopt modules +import simopt.directory as directory +from simopt.utils import print_table + + +def main() -> None: + """Print the contents of the problem and solver directories.""" + + def invert_dict(d: dict) -> dict: + """Invert a dictionary.""" + return {v: k for k, v in d.items()} + + directory_types = { + "Model": directory.model_directory, + "Problem": directory.problem_directory, + "Solver": directory.solver_directory, + } + + for dir_type, dir_dict in directory_types.items(): + # Class Name (ABBR) - Class Name (Full) - Class Type + # Invert the dictionaries to get a common key + dir_dict_inv = invert_dict(dir_dict) + dir_dict_full = directory.generate_unabbreviated_mapping(dir_dict) + dir_dict_full_inv = invert_dict(dir_dict_full) + entries = [] + for class_module, name_abbr in dir_dict_inv.items(): + # Remove the first two parts of the module name (simopt.directory) + shortened_module = ".".join(class_module.__module__.split(".")[2:]) + module_str = shortened_module + "." + class_module.__name__ + entry_tuple = (name_abbr, dir_dict_full_inv[class_module], module_str) + entries.append(entry_tuple) + print_table( + f"{dir_type} Directory", + ["Name (Abbr)", "Name (Full)", "Module.Class"], + entries, + ) + + +if __name__ == "__main__": + main() diff --git a/simopt/experiment_base.py b/simopt/experiment_base.py index d8ec25c6b..6a3aeb9fb 100644 --- a/simopt/experiment_base.py +++ b/simopt/experiment_base.py @@ -310,6 +310,9 @@ def __init__( self.has_postreplicated = False self.has_postnormalized = False self.xstar = () + self.x0 = () + self.objective_curves = [] + self.progress_curves = [] # Initialize solver. if isinstance(solver, Solver): # Method 2 @@ -1341,8 +1344,8 @@ class PlotType(Enum): SOLVE_TIME_CDF = "solve_time_cdf" CDF_SOLVABILITY = "cdf_solvability" QUANTILE_SOLVABILITY = "quantile_solvability" - DIFF_CDF_SOLVABILITY = "difference_of_cdf_solvability" - DIFF_QUANTILE_SOLVABILITY = "difference_of_quantile_solvability" + DIFFERENCE_OF_CDF_SOLVABILITY = "difference_of_cdf_solvability" + DIFFERENCE_OF_QUANTILE_SOLVABILITY = "difference_of_quantile_solvability" AREA = "area" BOX = "box" VIOLIN = "violin" @@ -1414,8 +1417,8 @@ def bootstrap_procedure( PlotType.SOLVE_TIME_CDF, PlotType.CDF_SOLVABILITY, PlotType.QUANTILE_SOLVABILITY, - PlotType.DIFF_CDF_SOLVABILITY, - PlotType.DIFF_QUANTILE_SOLVABILITY, + PlotType.DIFFERENCE_OF_CDF_SOLVABILITY, + PlotType.DIFFERENCE_OF_QUANTILE_SOLVABILITY, ] if plot_type not in acceptable_plot_types: error_msg = ( @@ -1537,8 +1540,8 @@ def functional_of_curves( - PlotType.SOLVE_TIME_CDF - PlotType.CDF_SOLVABILITY - PlotType.QUANTILE_SOLVABILITY - - PlotType.DIFF_CDF_SOLVABILITY - - PlotType.DIFF_QUANTILE_SOLVABILITY + - PlotType.DIFFERENCE_OF_CDF_SOLVABILITY + - PlotType.DIFFERENCE_OF_QUANTILE_SOLVABILITY beta (float, optional): Quantile level (0 < beta < 1). Defaults to 0.5. solve_tol (float, optional): Optimality gap for defining a solved instance (0 < solve_tol ≤ 1). Defaults to 0.1. @@ -1598,7 +1601,7 @@ def functional_of_curves( for curves in solver_1_curves ] ), - PlotType.DIFF_CDF_SOLVABILITY: lambda: curve_utils.difference_of_curves( + PlotType.DIFFERENCE_OF_CDF_SOLVABILITY: lambda: curve_utils.difference_of_curves( # noqa: E501 curve_utils.mean_of_curves( [ curve_utils.cdf_of_curves_crossing_times( @@ -1616,7 +1619,7 @@ def functional_of_curves( ] ), ), - PlotType.DIFF_QUANTILE_SOLVABILITY: lambda: curve_utils.difference_of_curves( + PlotType.DIFFERENCE_OF_QUANTILE_SOLVABILITY: lambda: curve_utils.difference_of_curves( # noqa: E501 curve_utils.mean_of_curves( [ curve_utils.quantile_cross_jump( @@ -2390,10 +2393,31 @@ def plot_area_scatterplots( [mean_estimator - mean_bs_conf_int_lb], [mean_bs_conf_int_ub - mean_estimator], ] - y_err = [ - [std_dev_estimator - std_dev_bs_conf_int_lb], - [std_dev_bs_conf_int_ub - std_dev_estimator], - ] + y_err_x = std_dev_estimator - std_dev_bs_conf_int_lb + y_err_y = std_dev_bs_conf_int_ub - std_dev_estimator + # If y_err_x or y_err_y is negative, set it to zero and warn. + if y_err_x < 0 or y_err_y < 0: + old_coords = (y_err_x, y_err_y) + y_err_x = max(0, y_err_x) + y_err_y = max(0, y_err_y) + new_coords = (y_err_x, y_err_y) + logging.warning( + "Warning: Negative error values detected in " + "area scatterplot. " + f"Old coordinates: {old_coords}, " + "Negative error values detected in area scatterplot " + "error bars. " + "This can occur due to statistical fluctuations in " + "bootstrap confidence interval estimation, especially with " + "small sample sizes or high variance. " + "Negative error bars are set to zero, which may affect the " + "visual interpretation of uncertainty. " + f"Old coordinates: {old_coords}, " + f"new coordinates: {new_coords}. " + "If this occurs frequently, consider reviewing your data " + "or increasing the number of replications." + ) + y_err = [[y_err_x], [y_err_y]] handle = plt.errorbar( x=mean_estimator, y=std_dev_estimator, @@ -2540,8 +2564,8 @@ def plot_solvability_profiles( plot_type (PlotType): Type of solvability plot to produce: - PlotType.CDF_SOLVABILITY - PlotType.QUANTILE_SOLVABILITY - - PlotType.DIFF_CDF_SOLVABILITY - - PlotType.DIFF_QUANTILE_SOLVABILITY + - PlotType.DIFFERENCE_OF_CDF_SOLVABILITY + - PlotType.DIFFERENCE_OF_QUANTILE_SOLVABILITY all_in_one (bool, optional): If True, plot all curves together. Defaults to True. n_bootstraps (int, optional): Number of bootstrap samples. Defaults to 100. @@ -2614,7 +2638,7 @@ def plot_solvability_profiles( solve_tol=solve_tol, plot_title=plot_title, ) - elif plot_type == PlotType.DIFF_CDF_SOLVABILITY: + elif plot_type == PlotType.DIFFERENCE_OF_CDF_SOLVABILITY: setup_plot( plot_type=plot_type, solver_name=solver_set_name, @@ -2622,7 +2646,7 @@ def plot_solvability_profiles( solve_tol=solve_tol, plot_title=plot_title, ) - elif plot_type == PlotType.DIFF_QUANTILE_SOLVABILITY: + elif plot_type == PlotType.DIFFERENCE_OF_QUANTILE_SOLVABILITY: setup_plot( plot_type=plot_type, solver_name=solver_set_name, @@ -2649,14 +2673,14 @@ def plot_solvability_profiles( sub_curve = None if plot_type in [ PlotType.CDF_SOLVABILITY, - PlotType.DIFF_CDF_SOLVABILITY, + PlotType.DIFFERENCE_OF_CDF_SOLVABILITY, ]: sub_curve = curve_utils.cdf_of_curves_crossing_times( curves=experiment.progress_curves, threshold=solve_tol ) elif plot_type in [ PlotType.QUANTILE_SOLVABILITY, - PlotType.DIFF_QUANTILE_SOLVABILITY, + PlotType.DIFFERENCE_OF_QUANTILE_SOLVABILITY, ]: sub_curve = curve_utils.quantile_cross_jump( curves=experiment.progress_curves, @@ -2754,8 +2778,8 @@ def plot_solvability_profiles( ) ) elif plot_type in [ - PlotType.DIFF_CDF_SOLVABILITY, - PlotType.DIFF_QUANTILE_SOLVABILITY, + PlotType.DIFFERENCE_OF_CDF_SOLVABILITY, + PlotType.DIFFERENCE_OF_QUANTILE_SOLVABILITY, ]: if ref_solver is None: error_msg = ( @@ -2823,7 +2847,7 @@ def plot_solvability_profiles( conf_level=conf_level, difference=True, ) - if plot_type == PlotType.DIFF_CDF_SOLVABILITY: + if plot_type == PlotType.DIFFERENCE_OF_CDF_SOLVABILITY: file_list.append( save_plot( solver_name=solver_set_name, @@ -2836,7 +2860,7 @@ def plot_solvability_profiles( save_as_pickle=save_as_pickle, ) ) - elif plot_type == PlotType.DIFF_QUANTILE_SOLVABILITY: + elif plot_type == PlotType.DIFFERENCE_OF_QUANTILE_SOLVABILITY: file_list.append( save_plot( solver_name=solver_set_name, @@ -2865,14 +2889,14 @@ def plot_solvability_profiles( sub_curve = None if plot_type in [ PlotType.CDF_SOLVABILITY, - PlotType.DIFF_CDF_SOLVABILITY, + PlotType.DIFFERENCE_OF_CDF_SOLVABILITY, ]: sub_curve = curve_utils.cdf_of_curves_crossing_times( curves=experiment.progress_curves, threshold=solve_tol ) elif plot_type in [ PlotType.QUANTILE_SOLVABILITY, - PlotType.DIFF_QUANTILE_SOLVABILITY, + PlotType.DIFFERENCE_OF_QUANTILE_SOLVABILITY, ]: sub_curve = curve_utils.quantile_cross_jump( curves=experiment.progress_curves, @@ -2972,8 +2996,8 @@ def plot_solvability_profiles( ) ) if plot_type in [ - PlotType.DIFF_CDF_SOLVABILITY, - PlotType.DIFF_QUANTILE_SOLVABILITY, + PlotType.DIFFERENCE_OF_CDF_SOLVABILITY, + PlotType.DIFFERENCE_OF_QUANTILE_SOLVABILITY, ]: if ref_solver is None: error_msg = ( @@ -2986,7 +3010,7 @@ def plot_solvability_profiles( ref_solver_idx = solver_names.index(ref_solver) for solver_idx in range(n_solvers): if solver_idx is not ref_solver_idx: - if plot_type == PlotType.DIFF_CDF_SOLVABILITY: + if plot_type == PlotType.DIFFERENCE_OF_CDF_SOLVABILITY: file_list.append( setup_plot( plot_type=plot_type, @@ -2995,7 +3019,7 @@ def plot_solvability_profiles( solve_tol=solve_tol, ) ) - elif plot_type == PlotType.DIFF_QUANTILE_SOLVABILITY: + elif plot_type == PlotType.DIFFERENCE_OF_QUANTILE_SOLVABILITY: file_list.append( setup_plot( plot_type=plot_type, @@ -3054,7 +3078,7 @@ def plot_solvability_profiles( conf_level=conf_level, difference=True, ) - if plot_type == PlotType.DIFF_CDF_SOLVABILITY: + if plot_type == PlotType.DIFFERENCE_OF_CDF_SOLVABILITY: file_list.append( save_plot( solver_name=experiments[solver_idx][0].solver.name, @@ -3066,7 +3090,7 @@ def plot_solvability_profiles( save_as_pickle=save_as_pickle, ) ) - elif plot_type == PlotType.DIFF_QUANTILE_SOLVABILITY: + elif plot_type == PlotType.DIFFERENCE_OF_QUANTILE_SOLVABILITY: file_list.append( save_plot( solver_name=experiments[solver_idx][0].solver.name, @@ -3386,8 +3410,9 @@ def setup_plot( - SOLVE_TIME_CDF: CDF of solve time. - CDF_SOLVABILITY: CDF solvability profile. - QUANTILE_SOLVABILITY: Quantile solvability profile. - - DIFF_CDF_SOLVABILITY: Difference of CDF solvability profiles. - - DIFF_QUANTILE_SOLVABILITY: Difference of quantile solvability profiles. + - DIFFERENCE_OF_CDF_SOLVABILITY: Difference of CDF solvability profiles. + - DIFFERENCE_OF_QUANTILE_SOLVABILITY: Difference of quantile solvability + profiles. - AREA: Area scatterplot. - BOX: Box plot of terminal progress. - VIOLIN: Violin plot of terminal progress. @@ -3474,7 +3499,7 @@ def setup_plot( f"Profile of {round(beta, 2)}-Quantiles " f"of {round(solve_tol, 2)}-Solve Times" ) - elif plot_type == PlotType.DIFF_CDF_SOLVABILITY: + elif plot_type == PlotType.DIFFERENCE_OF_CDF_SOLVABILITY: if solve_tol is None: error_msg = "Solve tolerance must be specified for cdf solvability plot." raise ValueError(error_msg) @@ -3485,7 +3510,7 @@ def setup_plot( ) plt.plot([0, 1], [0, 0], color="black", linestyle="--") plt.ylim((-1, 1)) - elif plot_type == PlotType.DIFF_QUANTILE_SOLVABILITY: + elif plot_type == PlotType.DIFFERENCE_OF_QUANTILE_SOLVABILITY: if beta is None: error_msg = "Beta must be specified for quantile solvability plot." raise ValueError(error_msg) @@ -3553,8 +3578,9 @@ def save_plot( - SOLVE_TIME_CDF: CDF of solve time. - CDF_SOLVABILITY: CDF solvability profile. - QUANTILE_SOLVABILITY: Quantile solvability profile. - - DIFF_CDF_SOLVABILITY: Difference of CDF solvability profiles. - - DIFF_QUANTILE_SOLVABILITY: Difference of quantile solvability profiles. + - DIFFERENCE_OF_CDF_SOLVABILITY: Difference of CDF solvability profiles. + - DIFFERENCE_OF_QUANTILE_SOLVABILITY: Difference of quantile solvability + profiles. - AREA: Area scatterplot. - TERMINAL_SCATTER: Scatterplot of mean and std dev of terminal progress. normalize (bool): Whether to normalize with respect to optimality gaps. @@ -3590,9 +3616,9 @@ def save_plot( extra_0 = float(extra[0]) extra_1 = float(extra[1]) plot_name = f"profile_{extra_1}_quantile_{extra_0}_solve_times" - elif plot_type == PlotType.DIFF_CDF_SOLVABILITY: + elif plot_type == PlotType.DIFFERENCE_OF_CDF_SOLVABILITY: plot_name = f"diff_profile_cdf_{extra}_solve_times" - elif plot_type == PlotType.DIFF_QUANTILE_SOLVABILITY: + elif plot_type == PlotType.DIFFERENCE_OF_QUANTILE_SOLVABILITY: if not (isinstance(extra, list) and len(extra) == 2): error_msg = ( "Extra must be a list of two floats for " diff --git a/simopt/gui/plot_window.py b/simopt/gui/plot_window.py index 7925a78e2..3c6e46c40 100644 --- a/simopt/gui/plot_window.py +++ b/simopt/gui/plot_window.py @@ -496,7 +496,7 @@ def add_plot(self) -> None: elif self.plot_type_list[-1] == "CDF Difference Plot": path_name = plot_solvability_profiles( list_exp_list, - plot_type=PlotType.DIFF_CDF_SOLVABILITY, + plot_type=PlotType.DIFFERENCE_OF_CDF_SOLVABILITY, plot_conf_ints=bool(param_value_list[0]), print_max_hw=bool(param_value_list[1]), solve_tol=param_value_list[2], @@ -515,7 +515,7 @@ def add_plot(self) -> None: elif self.plot_type_list[-1] == "Quantile Difference Plot": path_name = plot_solvability_profiles( list_exp_list, - plot_type=PlotType.DIFF_QUANTILE_SOLVABILITY, + plot_type=PlotType.DIFFERENCE_OF_QUANTILE_SOLVABILITY, plot_conf_ints=bool(param_value_list[0]), print_max_hw=bool(param_value_list[1]), solve_tol=param_value_list[2], diff --git a/simopt/utils.py b/simopt/utils.py index 83d8d6225..626ff224e 100644 --- a/simopt/utils.py +++ b/simopt/utils.py @@ -1,5 +1,6 @@ """Utility functions for simopt.""" +import sys from pathlib import Path from typing import Callable, Generic, Optional, TypeVar @@ -109,3 +110,102 @@ def resolve_file_path(target: str | Path, directory: str | Path) -> Path: return Path(target).resolve() # If it's a relative path, resolve it against the directory return (Path(directory) / target).resolve() + + +def print_table(name: str, headers: list[str], data: list[tuple] | dict) -> None: + """Print a table with headers and data. + + Args: + name (str): Name of the table. + headers (list[str]): List of column headers. + data (list[tuple]): List of rows, each row is a tuple of values. + """ + # Convert data out of dict (if necessary) + if isinstance(data, dict): + data = list(data.items()) + # Calculate the maximum length of each column + data_widths = [max(len(str(item)) for item in col) for col in zip(*data)] + header_widths = [len(header) for header in headers] + max_widths = [ + max(header_width, col_width) + for header_width, col_width in zip(header_widths, data_widths) + ] + + # Compute total width of the table + # There's 3 separator characters between each column + separator_lengths = 3 * (len(headers) - 1) + total_width = sum(max_widths) + separator_lengths + # If table is shorter than name, expand last column + if total_width < len(name): + shortfall = len(name) - total_width + max_widths[-1] += shortfall + total_width = len(name) + + # Center title in the table + title_indent_count = (total_width - len(name)) // 2 + title_lead = " " * title_indent_count + title_follow = " " * (total_width - title_indent_count - len(name)) + title = f"{title_lead}{name}{title_follow}" + + if sys.stdout.isatty(): + # Unicode box-drawing characters + corner_tl = "┌" + corner_tr = "┐" + corner_bl = "└" + corner_br = "┘" + tee_left = "├" + tee_right = "┤" + dash = "─" + plus = f"{dash}┼{dash}" + pipe = "│" + + reset = "\033[0m" + bg_grey = "\033[48;5;235m" + bg_black = "\033[48;5;0m" + fg_white = "\033[38;5;252m" + else: + # ASCII fallback + corner_tl = "+" + corner_tr = "+" + corner_bl = "+" + corner_br = "+" + tee_left = "+" + tee_right = "+" + dash = "-" + plus = "-+-" + pipe = "|" + + reset = "" + bg_grey = "" + bg_black = "" + fg_white = "" + + underline_row = dash * (total_width + 2) # Extend to the tees + header_row = f" {pipe} ".join( + f"{header:<{width}}" for header, width in zip(headers, max_widths) + ) + sep_row = plus.join(dash * width for width in max_widths) + rows = [] + for row in data: + row_str = f" {pipe} ".join( + f"{item!s:>{width}}" + if isinstance(item, (int, float)) + else f"{item!s:<{width}}" + for item, width in zip(row, max_widths) + ) + rows.append(row_str) + + border_width = total_width + 2 # Extend 2 extra spaces + + # Print the table + print(f"{bg_black}{fg_white}", end="") # Override background and foreground colors + print(f"{corner_tl}{dash * border_width}{corner_tr}") + print(f"{pipe} {title} {pipe}") + print(f"{tee_left}{underline_row}{tee_right}") + print(f"{pipe} {header_row} {pipe}") + print(f"{pipe} {sep_row} {pipe}") + for i, row in enumerate(rows): + row_bg = bg_grey if i % 2 else bg_black + print(f"{bg_black}{fg_white}{pipe}{row_bg} {row} {bg_black}{pipe}{reset}") + print(f"{corner_bl}{dash * border_width}{corner_br}") + print(reset, end="") # Reset colors diff --git a/simopt/utils.pyi b/simopt/utils.pyi index 1828c6746..9f328d4ff 100644 --- a/simopt/utils.pyi +++ b/simopt/utils.pyi @@ -7,3 +7,4 @@ def classproperty(func: Any) -> Any: ... # noqa: ANN401 def make_nonzero(value: float, name: str, epsilon: float = 1e-15) -> float: ... def override(func: T) -> T: ... def resolve_file_path(target: str | Path, directory: str | Path) -> Path: ... +def print_table(name: str, headers: list[str], data: list[tuple] | dict) -> None: ...