From a75eafff8eaa2ecea9a5764d4a01c61b597caa5a Mon Sep 17 00:00:00 2001 From: RohitP2005 Date: Mon, 10 Feb 2025 19:32:18 +0530 Subject: [PATCH 1/8] Inital template --- src/agentlab/llm/traces/config.py | 5 +++ src/agentlab/llm/traces/query.py | 29 +++++++++++++++ src/agentlab/llm/traces/uploads.py | 57 ++++++++++++++++++++++++++++++ 3 files changed, 91 insertions(+) create mode 100644 src/agentlab/llm/traces/config.py create mode 100644 src/agentlab/llm/traces/query.py create mode 100644 src/agentlab/llm/traces/uploads.py diff --git a/src/agentlab/llm/traces/config.py b/src/agentlab/llm/traces/config.py new file mode 100644 index 00000000..985dfcf6 --- /dev/null +++ b/src/agentlab/llm/traces/config.py @@ -0,0 +1,5 @@ +HF_USERNAME = "your_username" +HF_INDEX_DATASET = "your_username/agent_traces_index" +HF_TRACE_DATASET = "your_username/agent_traces_data" +WHITELISTED_BENCHMARKS = ["benchmark1", "benchmark2"] + diff --git a/src/agentlab/llm/traces/query.py b/src/agentlab/llm/traces/query.py new file mode 100644 index 00000000..9792b9f2 --- /dev/null +++ b/src/agentlab/llm/traces/query.py @@ -0,0 +1,29 @@ +from datasets import load_dataset +import requests + +# Hugging Face dataset name for the index +INDEX_DATASET = "your_username/agent_traces_index" + +# Function to query traces based on LLM and benchmark +def query_traces(llm=None, benchmark=None): + dataset = load_dataset(INDEX_DATASET, split="train") + df = dataset.to_pandas() + + if llm: + df = df[df["llm"] == llm] + if benchmark: + df = df[df["benchmark"] == benchmark] + + return df[["exp_id", "study_name", "trace_pointer"]].to_dict(orient="records") + +# Function to download a trace based on exp_id +def download_trace(exp_id: str, save_path: str): + dataset = load_dataset(INDEX_DATASET, split="train") + df = dataset.to_pandas() + trace_url = df[df["exp_id"] == exp_id]["trace_pointer"].values[0] + + response = requests.get(trace_url) + with open(save_path, "wb") as f: + f.write(response.content) + print(f"Downloaded trace {exp_id} to {save_path}") + diff --git a/src/agentlab/llm/traces/uploads.py b/src/agentlab/llm/traces/uploads.py new file mode 100644 index 00000000..451e54a3 --- /dev/null +++ b/src/agentlab/llm/traces/uploads.py @@ -0,0 +1,57 @@ +from datasets import Dataset, load_dataset +from huggingface_hub import HfApi +import pandas as pd + +# Hugging Face dataset names +INDEX_DATASET = "/agent_traces_index" +TRACE_DATASET = "/agent_traces_data" + +# Hugging Face API instance +api = HfApi() + +def upload_index_data(index_df: pd.DataFrame): + dataset = Dataset.from_pandas(index_df) + dataset.push_to_hub(INDEX_DATASET, split="train") + +def upload_trace(trace_file: str, exp_id: str): + api.upload_file( + path_or_fileobj=trace_file, + path_in_repo=f"{exp_id}.zip", + repo_id=TRACE_DATASET, + repo_type="dataset", + ) + +def add_study(exp_id: str, study_name: str, llm: str, benchmark: str, trace_file: str): + # Check if the benchmark is whitelisted + WHITELISTED_BENCHMARKS = ["benchmark1", "benchmark2"] + if benchmark not in WHITELISTED_BENCHMARKS: + raise ValueError("Benchmark not whitelisted") + + # Assign a license based on LLM and benchmark + LICENSES = { + ("GPT-4", "benchmark1"): "MIT", + ("Llama2", "benchmark2"): "Apache-2.0", + } + license_type = LICENSES.get((llm, benchmark), "Unknown") + + # Upload trace file + upload_trace(trace_file, exp_id) + + # Create metadata entry + index_entry = { + "exp_id": exp_id, + "study_name": study_name, + "llm": llm, + "benchmark": benchmark, + "license": license_type, + "trace_pointer": f"https://huggingface.co/datasets/{TRACE_DATASET}/resolve/main/{exp_id}.zip", + } + + # Load the existing index dataset and add new entry + dataset = load_dataset(INDEX_DATASET, split="train") + df = dataset.to_pandas() + df = df.append(index_entry, ignore_index=True) + upload_index_data(df) + + print(f"Study {exp_id} added successfully!") + From a6034f47099f21460d71e98ad3835ce052f1f55e Mon Sep 17 00:00:00 2001 From: RohitP2005 Date: Fri, 28 Feb 2025 21:26:08 +0530 Subject: [PATCH 2/8] Compress traces before uploding --- src/agentlab/llm/traces/uploads.py | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/src/agentlab/llm/traces/uploads.py b/src/agentlab/llm/traces/uploads.py index 451e54a3..ab152a6a 100644 --- a/src/agentlab/llm/traces/uploads.py +++ b/src/agentlab/llm/traces/uploads.py @@ -1,6 +1,9 @@ from datasets import Dataset, load_dataset from huggingface_hub import HfApi import pandas as pd +import zipfile +import os +from pathlib import Path # Hugging Face dataset names INDEX_DATASET = "/agent_traces_index" @@ -14,13 +17,33 @@ def upload_index_data(index_df: pd.DataFrame): dataset.push_to_hub(INDEX_DATASET, split="train") def upload_trace(trace_file: str, exp_id: str): + # Define the target zip file path + zip_file_path = f"{exp_id}.zip" + + # Check if the file is already zipped + if not trace_file.lower().endswith(".zip"): + # Compress the file into a zip archive + with zipfile.ZipFile(zip_file_path, 'w', zipfile.ZIP_DEFLATED) as zipf: + trace_file_name = Path(trace_file).name + zipf.write(trace_file, trace_file_name) + print(f"Compressed {trace_file} to {zip_file_path}") + else: + zip_file_path = trace_file + print(f"File {trace_file} is already a zip archive.") + + # Upload the (possibly newly created) zip file api.upload_file( - path_or_fileobj=trace_file, + path_or_fileobj=zip_file_path, path_in_repo=f"{exp_id}.zip", repo_id=TRACE_DATASET, repo_type="dataset", ) + # Optionally remove the temporary zip file if it was created during this process + if zip_file_path != trace_file: + os.remove(zip_file_path) + print(f"Temporary zip file {zip_file_path} removed.") + def add_study(exp_id: str, study_name: str, llm: str, benchmark: str, trace_file: str): # Check if the benchmark is whitelisted WHITELISTED_BENCHMARKS = ["benchmark1", "benchmark2"] From 1e5c4ccfc3310909503efd3b0d8ebe1fcd126455 Mon Sep 17 00:00:00 2001 From: RohitP2005 Date: Sat, 1 Mar 2025 02:35:28 +0530 Subject: [PATCH 3/8] Restructuring traces fucntionality --- src/agentlab/traces/__init__.py | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 src/agentlab/traces/__init__.py diff --git a/src/agentlab/traces/__init__.py b/src/agentlab/traces/__init__.py new file mode 100644 index 00000000..487886d9 --- /dev/null +++ b/src/agentlab/traces/__init__.py @@ -0,0 +1,3 @@ +from .trace_utils import * +from .uploads import * +from .query import * From 7923fdde5a77cf68b74fc1eba29db8000cdbfd83 Mon Sep 17 00:00:00 2001 From: RohitP2005 Date: Sat, 1 Mar 2025 02:36:26 +0530 Subject: [PATCH 4/8] Defining classes for Study and Experiments --- src/agentlab/traces/uploads.py | 87 ++++++++++++++++++++++++++++++++++ 1 file changed, 87 insertions(+) create mode 100644 src/agentlab/traces/uploads.py diff --git a/src/agentlab/traces/uploads.py b/src/agentlab/traces/uploads.py new file mode 100644 index 00000000..276cf17f --- /dev/null +++ b/src/agentlab/traces/uploads.py @@ -0,0 +1,87 @@ +from typing import List, Optional +from datetime import datetime +from typing import Optional, List +from datetime import datetime +from agentlab.traces.trace_utils import update_index +from traces import upload_trace,upload_study + +class Experiment: + """Represents a single experiment with relevant metadata.""" + + def __init__( + self, + exp_id: str, + study_id: str, + name: str, + llm: str, + benchmark: str, + license: str, + dir: str, + ): + + self.exp_id = exp_id + self.study_id = study_id + self.name = name + self.llm = llm + self.benchmark = benchmark + self.license = license + self.dir = dir + self.timestamp = datetime.now().isoformat() + + def __repr__(self): + return ( + f"Experiment(exp_id={self.exp_id}, study_id={self.study_id}, " + f"name={self.name}, llm={self.llm}, benchmark={self.benchmark}, " + + ) + + +class Study: + """Represents a study containing multiple experiments.""" + + + def __init__(self, study_id: str, study_name: str, description: str, experiments: List[Experiment]): + self.study_id = study_id + self.study_name = study_name + self.description = description + self.experiments = experiments + + def add_experiment(self, experiment: Experiment) -> None: + """Add an experiment to the study.""" + self.experiments.append(experiment) + + def remove_experiment(self, experiment_id: str) -> None: + """Remove an experiment from the study by ID.""" + self.experiments = [exp for exp in self.experiments if exp.exp_id != experiment_id] + + def get_experiment(self, experiment_id: str) -> Optional[Experiment]: + """Retrieve an experiment by ID.""" + for experiment in self.experiments: + if experiment.exp_id == experiment_id: + return experiment + return None + + def upload(self) -> None: + """Upload all experiment traces in the study to Hugging Face.""" + self.upload_Study() + for exp in self.experiments: + trace_pointer = upload_trace(exp.exp_id,exp.dir,exp.benchmark) + # Assign a license based on LLM and benchmark + LICENSES = { + ("GPT-4", "benchmark1"): "MIT", + ("Llama2", "benchmark2"): "Apache-2.0", + } + license_type = LICENSES.get((exp.exp_llm, exp.exp_benchmark), "Unknown") + update_index(exp.exp_id,self.study_id,exp.llm,exp.benchmark,license_type,trace_pointer) + + def upload_Study(self): + """Upload study to the study dataset""" + study_data = { + "study_id": [self.study_id], + "study_name": [self.study_name], + "description": [self.description], + } + upload_study(study_data) + + def __repr__(self): + return f"Study(id={self.study_id}, name={self.study_name}, experiments={len(self.experiments)})" From 8c1b880bc903114640d62cff9c7969d49337856a Mon Sep 17 00:00:00 2001 From: RohitP2005 Date: Sat, 1 Mar 2025 02:37:02 +0530 Subject: [PATCH 5/8] Addind trace utility fucntions --- src/agentlab/traces/trace_utils.py | 175 +++++++++++++++++++++++++++++ 1 file changed, 175 insertions(+) create mode 100644 src/agentlab/traces/trace_utils.py diff --git a/src/agentlab/traces/trace_utils.py b/src/agentlab/traces/trace_utils.py new file mode 100644 index 00000000..7fcf70f4 --- /dev/null +++ b/src/agentlab/traces/trace_utils.py @@ -0,0 +1,175 @@ +import os +from datasets import Dataset, load_dataset, concatenate_datasets +import zipfile + +# Retrieve environment variables +hf_repo_name = os.getenv('HF_REPO_NAME') +hf_token = os.getenv('HF_TOKEN') + +def upload_study(study_details: dict): + """ + Uploads study details to the Hugging Face `STUDY_DATASET`. + + Args: + study_details (dict): A dictionary containing study_id, study_name, and description. + """ + + if not hf_repo_name or not hf_token: + raise ValueError("HF_REPO_NAME and HF_TOKEN environment variables must be set.") + + study_dataset = f"{hf_repo_name}/agent_traces_study" + + # Load existing dataset or create a new one + try: + dataset = load_dataset(study_dataset, split='train', token=hf_token) + existing_data = dataset.to_dict() + except Exception as e: + print(f"Could not load existing dataset: {e}. Creating a new dataset.") + existing_data = None + + # Create a new dataset with the new study details + new_data = Dataset.from_dict({ + 'study_id': [study_details.get('study_id')], + 'study_name': [study_details.get('study_name')], + 'description': [study_details.get('description')] + }) + + # Concatenate with existing data if available + if existing_data: + existing_dataset = Dataset.from_dict(existing_data) + combined_data = concatenate_datasets([existing_dataset, new_data]) + else: + combined_data = new_data + + # Push updated dataset to the Hugging Face Hub + try: + combined_data.push_to_hub(study_dataset, token=hf_token) + print("Study details uploaded successfully!") + except Exception as e: + print(f"Failed to upload study details: {e}") + +def upload_trace(exp_id: str, directory: str, benchmark: str) -> str: + """ + Compresses a directory into a zip file, uploads it to the TRACE_DATASET on Hugging Face, + and returns the URL of the uploaded file. + + Args: + exp_id (str): The experiment ID associated with this trace. + directory (str): The path to the directory to compress. + benchmark (str): The benchmark name, which must be whitelisted. + + Returns: + str: The URL of the uploaded zip file in the dataset. + """ + # Check if the benchmark is whitelisted + WHITELISTED_BENCHMARKS = ["benchmark1", "benchmark2"] + if benchmark not in WHITELISTED_BENCHMARKS: + raise ValueError("Benchmark not whitelisted") + + if not hf_repo_name or not hf_token: + raise ValueError("HF_REPO_NAME and HF_TOKEN environment variables must be set.") + + trace_dataset = f"{hf_repo_name}/agent_traces_data" + + # Create a zip file from the directory + zip_filename = f"{exp_id}.zip" + with zipfile.ZipFile(zip_filename, 'w', zipfile.ZIP_DEFLATED) as zipf: + for root, _, files in os.walk(directory): + for file in files: + file_path = os.path.join(root, file) + zipf.write(file_path, os.path.relpath(file_path, directory)) + + print(f"Directory '{directory}' compressed into '{zip_filename}'.") + + # Load existing dataset or create a new one + try: + dataset = load_dataset(trace_dataset, use_auth_token=hf_token, split='train') + existing_data = { + 'exp_id': dataset['exp_id'], + 'zip_file': dataset['zip_file'] + } + except Exception as e: + print(f"Could not load existing dataset: {e}. Creating a new dataset.") + existing_data = None + + # Create a new dataset with the new experiment trace + new_data = Dataset.from_dict({ + 'exp_id': [exp_id], + 'zip_file': [zip_filename] + }) + + # Concatenate with existing data if available + if existing_data: + existing_dataset = Dataset.from_dict(existing_data) + combined_data = concatenate_datasets([existing_dataset, new_data]) + else: + combined_data = new_data + + # Push updated dataset to the Hugging Face Hub + combined_data.push_to_hub(trace_dataset, token=hf_token) + print("Experiment trace uploaded successfully!") + + # Clean up the local zip file + os.remove(zip_filename) + print(f"Temporary zip file '{zip_filename}' removed.") + + # Construct and return the file URL on the Hugging Face Hub + file_url = f"https://huggingface.co/datasets/{trace_dataset}/resolve/main/{zip_filename}" + print(f"File URL: {file_url}") + + return file_url + +def update_index(exp_id: str, study_id: str, llm: str, benchmark: str, license: str, trace_pointer: str): + """ + Adds a record to the INDEX_DATASET on Hugging Face with the given experiment details. + + Args: + exp_id (str): The experiment ID. + study_id (str): The study ID. + llm (str): The name of the large language model used. + benchmark (str): The benchmark used for evaluation. + license (str): The license type for the experiment. + trace_pointer (str): The URL pointer to the trace data (must be a valid HTTPS URL). + """ + + if not hf_repo_name or not hf_token: + raise ValueError("HF_REPO_NAME and HF_TOKEN environment variables must be set.") + + index_dataset = f"{hf_repo_name}/agent_traces_index" + + # Load existing dataset or create a new one + try: + dataset = load_dataset(index_dataset, use_auth_token=hf_token, split='train') + existing_data = { + 'exp_id': dataset['exp_id'], + 'study_id': dataset['study_id'], + 'llm': dataset['llm'], + 'benchmark': dataset['benchmark'], + 'license': dataset['license'], + 'trace_pointer': dataset['trace_pointer'] + } + except Exception as e: + print(f"Could not load existing dataset: {e}. Creating a new dataset.") + existing_data = None + + # Create a new dataset with the provided index details + new_data = Dataset.from_dict({ + 'exp_id': [exp_id], + 'study_id': [study_id], + 'llm': [llm], + 'benchmark': [benchmark], + 'license': [license], + 'trace_pointer': [trace_pointer] + }) + + # Concatenate with existing data if available + if existing_data: + existing_dataset = Dataset.from_dict(existing_data) + combined_data = concatenate_datasets([existing_dataset, new_data]) + else: + combined_data = new_data + + # Push updated dataset to the Hugging Face Hub + combined_data.push_to_hub(index_dataset, token=hf_token) + print("Index dataset updated successfully!") + From e58dd98b36cd11803c97f895d1296afbe6a90569 Mon Sep 17 00:00:00 2001 From: RohitP2005 Date: Sat, 1 Mar 2025 02:37:37 +0530 Subject: [PATCH 6/8] Addind quering fucntions --- src/agentlab/traces/query.py | 58 ++++++++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) create mode 100644 src/agentlab/traces/query.py diff --git a/src/agentlab/traces/query.py b/src/agentlab/traces/query.py new file mode 100644 index 00000000..09c3f92c --- /dev/null +++ b/src/agentlab/traces/query.py @@ -0,0 +1,58 @@ +from datasets import load_dataset +from typing import List +import os + +HF_TOKEN = os.getenv('HF_TOKEN') +HF_REPO_NAME = os.getenv('HF_REPO_NAME') + + +def query_traces_by_llm_and_benchmark(llm: str, benchmark: str) -> List[dict]: + """ + Query traces based on the provided LLM and benchmark. + :param llm: The name of the LLM (e.g., 'GPT-4'). + :param benchmark: The benchmark name (e.g., 'benchmark1'). + :return: A list of trace metadata dictionaries. + """ + INDEX_DATASET = f"{HF_REPO_NAME}/agent_traces_index" + + try: + dataset = load_dataset(INDEX_DATASET, use_auth_token=HF_TOKEN, split='train') + results = [ + { + 'exp_id': row['exp_id'], + 'study_id': row['study_id'], + 'llm': row['llm'], + 'benchmark': row['benchmark'], + 'trace_pointer': row['trace_pointer'] + } + for row in dataset + if row['llm'] == llm and row['benchmark'] == benchmark + ] + return results + except Exception as e: + print(f"Error querying traces for LLM '{llm}' and benchmark '{benchmark}': {e}") + return [] + + +def download_trace_by_experiment_id(exp_id: str, output_dir: str) -> None: + """ + Download the trace file based on the experiment ID. + :param exp_id: The ID of the experiment whose trace file needs to be downloaded. + :param output_dir: The directory where the trace file will be saved. + """ + TRACE_DATASET = f"{HF_REPO_NAME}/agent_traces_data" + + try: + dataset = load_dataset(TRACE_DATASET, use_auth_token=HF_TOKEN, split='train') + for row in dataset: + if row['exp_id'] == exp_id: + trace_file = row['zip_file'] + output_path = os.path.join(output_dir, trace_file) + dataset.download_and_prepare() + dataset.to_csv(output_path) + print(f"Trace file for experiment '{exp_id}' downloaded to {output_path}.") + return + print(f"Experiment ID '{exp_id}' not found in the dataset.") + except Exception as e: + print(f"Error downloading trace file for experiment '{exp_id}': {e}") + From bcf9988b3321d53ff50931dadea00ac333595a6d Mon Sep 17 00:00:00 2001 From: RohitP2005 Date: Sat, 1 Mar 2025 02:38:13 +0530 Subject: [PATCH 7/8] removing old implementations --- src/agentlab/llm/traces/config.py | 5 -- src/agentlab/llm/traces/query.py | 29 ----------- src/agentlab/llm/traces/uploads.py | 80 ------------------------------ 3 files changed, 114 deletions(-) delete mode 100644 src/agentlab/llm/traces/config.py delete mode 100644 src/agentlab/llm/traces/query.py delete mode 100644 src/agentlab/llm/traces/uploads.py diff --git a/src/agentlab/llm/traces/config.py b/src/agentlab/llm/traces/config.py deleted file mode 100644 index 985dfcf6..00000000 --- a/src/agentlab/llm/traces/config.py +++ /dev/null @@ -1,5 +0,0 @@ -HF_USERNAME = "your_username" -HF_INDEX_DATASET = "your_username/agent_traces_index" -HF_TRACE_DATASET = "your_username/agent_traces_data" -WHITELISTED_BENCHMARKS = ["benchmark1", "benchmark2"] - diff --git a/src/agentlab/llm/traces/query.py b/src/agentlab/llm/traces/query.py deleted file mode 100644 index 9792b9f2..00000000 --- a/src/agentlab/llm/traces/query.py +++ /dev/null @@ -1,29 +0,0 @@ -from datasets import load_dataset -import requests - -# Hugging Face dataset name for the index -INDEX_DATASET = "your_username/agent_traces_index" - -# Function to query traces based on LLM and benchmark -def query_traces(llm=None, benchmark=None): - dataset = load_dataset(INDEX_DATASET, split="train") - df = dataset.to_pandas() - - if llm: - df = df[df["llm"] == llm] - if benchmark: - df = df[df["benchmark"] == benchmark] - - return df[["exp_id", "study_name", "trace_pointer"]].to_dict(orient="records") - -# Function to download a trace based on exp_id -def download_trace(exp_id: str, save_path: str): - dataset = load_dataset(INDEX_DATASET, split="train") - df = dataset.to_pandas() - trace_url = df[df["exp_id"] == exp_id]["trace_pointer"].values[0] - - response = requests.get(trace_url) - with open(save_path, "wb") as f: - f.write(response.content) - print(f"Downloaded trace {exp_id} to {save_path}") - diff --git a/src/agentlab/llm/traces/uploads.py b/src/agentlab/llm/traces/uploads.py deleted file mode 100644 index ab152a6a..00000000 --- a/src/agentlab/llm/traces/uploads.py +++ /dev/null @@ -1,80 +0,0 @@ -from datasets import Dataset, load_dataset -from huggingface_hub import HfApi -import pandas as pd -import zipfile -import os -from pathlib import Path - -# Hugging Face dataset names -INDEX_DATASET = "/agent_traces_index" -TRACE_DATASET = "/agent_traces_data" - -# Hugging Face API instance -api = HfApi() - -def upload_index_data(index_df: pd.DataFrame): - dataset = Dataset.from_pandas(index_df) - dataset.push_to_hub(INDEX_DATASET, split="train") - -def upload_trace(trace_file: str, exp_id: str): - # Define the target zip file path - zip_file_path = f"{exp_id}.zip" - - # Check if the file is already zipped - if not trace_file.lower().endswith(".zip"): - # Compress the file into a zip archive - with zipfile.ZipFile(zip_file_path, 'w', zipfile.ZIP_DEFLATED) as zipf: - trace_file_name = Path(trace_file).name - zipf.write(trace_file, trace_file_name) - print(f"Compressed {trace_file} to {zip_file_path}") - else: - zip_file_path = trace_file - print(f"File {trace_file} is already a zip archive.") - - # Upload the (possibly newly created) zip file - api.upload_file( - path_or_fileobj=zip_file_path, - path_in_repo=f"{exp_id}.zip", - repo_id=TRACE_DATASET, - repo_type="dataset", - ) - - # Optionally remove the temporary zip file if it was created during this process - if zip_file_path != trace_file: - os.remove(zip_file_path) - print(f"Temporary zip file {zip_file_path} removed.") - -def add_study(exp_id: str, study_name: str, llm: str, benchmark: str, trace_file: str): - # Check if the benchmark is whitelisted - WHITELISTED_BENCHMARKS = ["benchmark1", "benchmark2"] - if benchmark not in WHITELISTED_BENCHMARKS: - raise ValueError("Benchmark not whitelisted") - - # Assign a license based on LLM and benchmark - LICENSES = { - ("GPT-4", "benchmark1"): "MIT", - ("Llama2", "benchmark2"): "Apache-2.0", - } - license_type = LICENSES.get((llm, benchmark), "Unknown") - - # Upload trace file - upload_trace(trace_file, exp_id) - - # Create metadata entry - index_entry = { - "exp_id": exp_id, - "study_name": study_name, - "llm": llm, - "benchmark": benchmark, - "license": license_type, - "trace_pointer": f"https://huggingface.co/datasets/{TRACE_DATASET}/resolve/main/{exp_id}.zip", - } - - # Load the existing index dataset and add new entry - dataset = load_dataset(INDEX_DATASET, split="train") - df = dataset.to_pandas() - df = df.append(index_entry, ignore_index=True) - upload_index_data(df) - - print(f"Study {exp_id} added successfully!") - From ec5b1b16ee5dabf781eab9507effd2b0341732f1 Mon Sep 17 00:00:00 2001 From: ThibaultLSDC Date: Tue, 11 Mar 2025 17:24:23 -0400 Subject: [PATCH 8/8] updating trace upload functions to AgentLab/BGym objects --- .../experiments/reproducibility_util.py | 33 ++- src/agentlab/experiments/study.py | 16 +- src/agentlab/traces/trace_utils.py | 191 ++++++++++-------- 3 files changed, 138 insertions(+), 102 deletions(-) diff --git a/src/agentlab/experiments/reproducibility_util.py b/src/agentlab/experiments/reproducibility_util.py index 01f3fdc9..fd5af408 100644 --- a/src/agentlab/experiments/reproducibility_util.py +++ b/src/agentlab/experiments/reproducibility_util.py @@ -324,6 +324,21 @@ def append_to_journal( info, report_df: pd.DataFrame, journal_path=None, strict_reproducibility=True ): """Append the info and results to the reproducibility journal.""" + journal_path, headers = get_headers_from_journal(journal_path) + + rows = create_journal_entries(info, report_df, strict_reproducibility, headers) + + write_entries_to_journal(journal_path, rows) + + +def write_entries_to_journal(journal_path, rows): + with open(journal_path, "a", newline="") as file: + writer = csv.writer(file) + for row in rows: + writer.writerow(row) + + +def get_headers_from_journal(journal_path=None): if journal_path is None: try: _get_repo(agentlab) # if not based on git clone, this will raise an error @@ -339,6 +354,13 @@ def append_to_journal( logging.info(f"Appending to journal {journal_path}") + headers = None + if journal_path.exists(): + headers = _get_csv_headers(journal_path) + return journal_path, headers + + +def create_journal_entries(info, report_df, strict_reproducibility=True, headers=None): if len(report_df) != len(info["agent_names"]): raise ValueError( "Mismatch between the number of agents in reproducibility info and the summary report." @@ -347,12 +369,7 @@ def append_to_journal( report_df = _verify_report( report_df, info["agent_names"], strict_reproducibility=strict_reproducibility ) - rows = [] - headers = None - if journal_path.exists(): - headers = _get_csv_headers(journal_path) - if headers is None: # first creation headers = list(info.keys()) headers[headers.index("agent_names")] = "agent_name" @@ -366,8 +383,4 @@ def append_to_journal( _add_result_to_info(info_copy, report_df) rows.append([str(info_copy[key]) for key in headers]) - - with open(journal_path, "a", newline="") as file: - writer = csv.writer(file) - for row in rows: - writer.writerow(row) + return rows diff --git a/src/agentlab/experiments/study.py b/src/agentlab/experiments/study.py index b93b3ae2..4f3d2cb7 100644 --- a/src/agentlab/experiments/study.py +++ b/src/agentlab/experiments/study.py @@ -373,6 +373,13 @@ def _run(self, n_jobs=1, parallel_backend="joblib", strict_reproducibility=False avg_step_timeout=self.avg_step_timeout, ) + def get_journal_entries(self, strict_reproducibility=True, headers=None): + """Get the journal entries for the study.""" + _, summary_df, _ = self.get_results() + return repro.create_journal_entries( + self.reproducibility_info, summary_df, strict_reproducibility, headers + ) + def append_to_journal(self, strict_reproducibility=True): """Append the study to the journal. @@ -380,12 +387,11 @@ def append_to_journal(self, strict_reproducibility=True): strict_reproducibility: bool If True, incomplete experiments will raise an error. """ - _, summary_df, _ = self.get_results() - repro.append_to_journal( - self.reproducibility_info, - summary_df, - strict_reproducibility=strict_reproducibility, + journal_path, headers = repro.get_headers_from_journal() + rows = self.get_journal_entries( + strict_reproducibility=strict_reproducibility, headers=headers ) + repro.write_entries_to_journal(journal_path, rows) @property def name(self): diff --git a/src/agentlab/traces/trace_utils.py b/src/agentlab/traces/trace_utils.py index 7fcf70f4..98ebe612 100644 --- a/src/agentlab/traces/trace_utils.py +++ b/src/agentlab/traces/trace_utils.py @@ -1,175 +1,192 @@ import os -from datasets import Dataset, load_dataset, concatenate_datasets import zipfile +from bgym import ExpArgs, ExpResult +from datasets import Dataset, concatenate_datasets, load_dataset + +from agentlab.experiments.study import Study + # Retrieve environment variables -hf_repo_name = os.getenv('HF_REPO_NAME') -hf_token = os.getenv('HF_TOKEN') +hf_user_name = os.getenv("HF_REPO_NAME") +hf_token = os.getenv("HF_TOKEN") + -def upload_study(study_details: dict): +def upload_study(study: Study): """ Uploads study details to the Hugging Face `STUDY_DATASET`. Args: - study_details (dict): A dictionary containing study_id, study_name, and description. + study (Study): The study object containing the experiment details. """ - - if not hf_repo_name or not hf_token: + + if not hf_user_name or not hf_token: raise ValueError("HF_REPO_NAME and HF_TOKEN environment variables must be set.") - - study_dataset = f"{hf_repo_name}/agent_traces_study" - + + study_dataset = f"{hf_user_name}/agent_traces_study" + # Load existing dataset or create a new one try: - dataset = load_dataset(study_dataset, split='train', token=hf_token) + dataset = load_dataset(study_dataset, split="train", token=hf_token) existing_data = dataset.to_dict() + headers = dataset.column_names except Exception as e: print(f"Could not load existing dataset: {e}. Creating a new dataset.") existing_data = None - + headers = None + # Create a new dataset with the new study details - new_data = Dataset.from_dict({ - 'study_id': [study_details.get('study_id')], - 'study_name': [study_details.get('study_name')], - 'description': [study_details.get('description')] - }) - + entries = study.get_journal_entries( + strict_reproducibility=False, headers=headers + ) # type: list[list] + if headers is None: + headers = entries[0] + + entries = entries[1:] + entries = list(zip(*entries)) + new_data = Dataset.from_dict({header: entries[i] for i, header in enumerate(headers)}) + # Concatenate with existing data if available if existing_data: existing_dataset = Dataset.from_dict(existing_data) combined_data = concatenate_datasets([existing_dataset, new_data]) else: combined_data = new_data - + # Push updated dataset to the Hugging Face Hub try: - combined_data.push_to_hub(study_dataset, token=hf_token) + combined_data.push_to_hub(study_dataset, token=hf_token, create_pr=True) print("Study details uploaded successfully!") except Exception as e: print(f"Failed to upload study details: {e}") -def upload_trace(exp_id: str, directory: str, benchmark: str) -> str: + +def upload_trace(exp_args: ExpArgs) -> str: """ Compresses a directory into a zip file, uploads it to the TRACE_DATASET on Hugging Face, and returns the URL of the uploaded file. Args: - exp_id (str): The experiment ID associated with this trace. - directory (str): The path to the directory to compress. - benchmark (str): The benchmark name, which must be whitelisted. + exp_args (ExpArgs): The experiment arguments. Returns: str: The URL of the uploaded zip file in the dataset. """ - # Check if the benchmark is whitelisted - WHITELISTED_BENCHMARKS = ["benchmark1", "benchmark2"] - if benchmark not in WHITELISTED_BENCHMARKS: - raise ValueError("Benchmark not whitelisted") + # # Check if the benchmark is whitelisted + # WHITELISTED_BENCHMARKS = ["benchmark1", "benchmark2"] + # if benchmark not in WHITELISTED_BENCHMARKS: + # raise ValueError("Benchmark not whitelisted") - if not hf_repo_name or not hf_token: + if not hf_user_name or not hf_token: raise ValueError("HF_REPO_NAME and HF_TOKEN environment variables must be set.") - - trace_dataset = f"{hf_repo_name}/agent_traces_data" - + + trace_dataset = f"{hf_user_name}/agent_traces_data" + # Create a zip file from the directory - zip_filename = f"{exp_id}.zip" - with zipfile.ZipFile(zip_filename, 'w', zipfile.ZIP_DEFLATED) as zipf: - for root, _, files in os.walk(directory): + zip_filename = f"{exp_args.exp_id}.zip" + with zipfile.ZipFile(zip_filename, "w", zipfile.ZIP_DEFLATED) as zipf: + for root, _, files in os.walk(exp_args.exp_dir): for file in files: file_path = os.path.join(root, file) - zipf.write(file_path, os.path.relpath(file_path, directory)) - - print(f"Directory '{directory}' compressed into '{zip_filename}'.") - + zipf.write(file_path, os.path.relpath(file_path, exp_args.exp_dir)) + + print(f"Directory '{exp_args}' compressed into '{zip_filename}'.") + # Load existing dataset or create a new one try: - dataset = load_dataset(trace_dataset, use_auth_token=hf_token, split='train') - existing_data = { - 'exp_id': dataset['exp_id'], - 'zip_file': dataset['zip_file'] - } + dataset = load_dataset(trace_dataset, use_auth_token=hf_token, split="train") + existing_data = {"exp_id": dataset["exp_id"], "zip_file": dataset["zip_file"]} except Exception as e: print(f"Could not load existing dataset: {e}. Creating a new dataset.") existing_data = None - + # Create a new dataset with the new experiment trace - new_data = Dataset.from_dict({ - 'exp_id': [exp_id], - 'zip_file': [zip_filename] - }) - + new_data = Dataset.from_dict({"exp_id": [exp_args.exp_id], "zip_file": [zip_filename]}) + # Concatenate with existing data if available if existing_data: existing_dataset = Dataset.from_dict(existing_data) combined_data = concatenate_datasets([existing_dataset, new_data]) else: combined_data = new_data - + # Push updated dataset to the Hugging Face Hub combined_data.push_to_hub(trace_dataset, token=hf_token) print("Experiment trace uploaded successfully!") - + # Clean up the local zip file os.remove(zip_filename) print(f"Temporary zip file '{zip_filename}' removed.") - + # Construct and return the file URL on the Hugging Face Hub file_url = f"https://huggingface.co/datasets/{trace_dataset}/resolve/main/{zip_filename}" print(f"File URL: {file_url}") - + return file_url -def update_index(exp_id: str, study_id: str, llm: str, benchmark: str, license: str, trace_pointer: str): + +def update_index( + exp_results: list[ExpResult], study_id: str, license: str, trace_pointers: list[str] +): """ Adds a record to the INDEX_DATASET on Hugging Face with the given experiment details. Args: - exp_id (str): The experiment ID. - study_id (str): The study ID. - llm (str): The name of the large language model used. - benchmark (str): The benchmark used for evaluation. - license (str): The license type for the experiment. - trace_pointer (str): The URL pointer to the trace data (must be a valid HTTPS URL). + exp_args (ExpArgs): The experiment arguments. + study_id (str): The study ID associated with this experiment. + license (str): The license type for the experiment trace. + trace_pointer (str): The URL of the experiment trace file. """ - - if not hf_repo_name or not hf_token: + + if not hf_user_name or not hf_token: raise ValueError("HF_REPO_NAME and HF_TOKEN environment variables must be set.") - - index_dataset = f"{hf_repo_name}/agent_traces_index" - + + index_dataset = f"{hf_user_name}/agent_traces_index" + # Load existing dataset or create a new one try: - dataset = load_dataset(index_dataset, use_auth_token=hf_token, split='train') - existing_data = { - 'exp_id': dataset['exp_id'], - 'study_id': dataset['study_id'], - 'llm': dataset['llm'], - 'benchmark': dataset['benchmark'], - 'license': dataset['license'], - 'trace_pointer': dataset['trace_pointer'] - } + dataset = load_dataset(index_dataset, use_auth_token=hf_token, split="train") + existing_data = dataset.to_dict() except Exception as e: print(f"Could not load existing dataset: {e}. Creating a new dataset.") existing_data = None - + # Create a new dataset with the provided index details - new_data = Dataset.from_dict({ - 'exp_id': [exp_id], - 'study_id': [study_id], - 'llm': [llm], - 'benchmark': [benchmark], - 'license': [license], - 'trace_pointer': [trace_pointer] - }) - + + dataset = [exp_res.get_exp_record() for exp_res in exp_results] + for el in dataset: + el.pop("exp_dir") + + # list[dict] -> dict[list] + new_data = {key: [d[key] for d in dataset] for key in dataset[0].keys()} + new_data["study_id"] = [study_id.hex] * len(exp_results) + new_data["license"] = [license] * len(exp_results) + new_data["trace_pointer"] = trace_pointers + + new_data = Dataset.from_dict(new_data) + # Concatenate with existing data if available if existing_data: existing_dataset = Dataset.from_dict(existing_data) combined_data = concatenate_datasets([existing_dataset, new_data]) else: combined_data = new_data - + # Push updated dataset to the Hugging Face Hub - combined_data.push_to_hub(index_dataset, token=hf_token) + combined_data.push_to_hub(index_dataset, token=hf_token, create_pr=True) print("Index dataset updated successfully!") + +if __name__ == "__main__": + import os + import pathlib + + from agentlab.experiments.study import Study + from agentlab.traces.trace_utils import update_index, upload_study + + path = pathlib.Path("/path/to/study") + + study = Study.load(path) + study.load_exp_args_list() + + upload_study(study) + update_index(study.exp_args_list, study.uuid, "open", ["w/e"] * len(study.exp_args_list))