output result to a custom dir

Author: mayinghan

eval_protocol/pytest/handle_persist_flow.py

@@ -16,7 +16,8 @@
 def handle_persist_flow(all_results: list[list[EvaluationRow]], test_func_name: str):
     try:
         # Default is to save and upload experiment JSONL files, unless explicitly disabled
-        should_save = os.getenv("EP_NO_PERSIST_RESULTS_JSONL") != "1"
+        custom_output_dir = os.getenv("EP_OUTPUT_DIR")
+        should_save = os.getenv("EP_NO_UPLOAD") != "1" or custom_output_dir is not None
 
         if should_save:
             current_run_rows = [item for sublist in all_results for item in sublist]
@@ -27,6 +28,8 @@ def handle_persist_flow(all_results: list[list[EvaluationRow]], test_func_name:
                         experiments[row.execution_metadata.experiment_id].append(row)
 
                 eval_protocol_dir = find_eval_protocol_dir()
+                if custom_output_dir:
+                    eval_protocol_dir = custom_output_dir
                 exp_dir = pathlib.Path(eval_protocol_dir) / "experiment_results"
                 exp_dir.mkdir(parents=True, exist_ok=True)
 

eval_protocol/pytest/plugin.py

@@ -134,10 +134,9 @@ def pytest_addoption(parser) -> None:
         help=("If set, use this base URL for remote rollout processing. Example: http://localhost:8000"),
     )
     group.addoption(
-        "--ep-no-persist-results-jsonl",
-        action="store_true",
-        default=False,
-        help=("Disable persisting results as jsonl files. Default: false (results are persisted by default)."),
+        "--ep-output-dir",
+        default=None,
+        help=("If set, save evaluation results to this directory in jsonl format."),
     )
 
 
@@ -264,9 +263,9 @@ def pytest_configure(config) -> None:
     if threshold_env is not None:
         os.environ["EP_PASSED_THRESHOLD"] = threshold_env
 
-    if config.getoption("--ep-no-save-results-in-jsonl"):
-        # flag to turn off persisting results as jsonl files
-        os.environ["EP_NO_PERSIST_RESULTS_JSONL"] = "1"
+    if config.getoption("--ep-output-dir"):
+        # set this to save eval results to the target dir in jsonl format
+        os.environ["EP_OUTPUT_DIR"] = config.getoption("--ep-output-dir")
 
     if config.getoption("--ep-no-upload"):
         os.environ["EP_NO_UPLOAD"] = "1"