diff --git a/benchmark_utils/__init__.py b/benchmark_utils/__init__.py index eaeac2f..40f92b6 100644 --- a/benchmark_utils/__init__.py +++ b/benchmark_utils/__init__.py @@ -92,10 +92,8 @@ def check_data(data_path, dataset, data_type): } raise ImportError( f"{data_type.capitalize()} data not found for {dataset}. " - "Please download the data " - "from the official repository " - f"{official_repo[dataset]}" - f"and place it in {data_path}" + "Please download the data from the official repository " + f"{official_repo[dataset]} and place it in {data_path}" ) return required_files[0] diff --git a/config.yml b/config.yml new file mode 100644 index 0000000..94a2869 --- /dev/null +++ b/config.yml @@ -0,0 +1,4 @@ +plots: + - per_dataset_delta + - head_to_head + - table diff --git a/datasets/daphnet.py b/datasets/daphnet.py index 1ca6c48..72e8210 100644 --- a/datasets/daphnet.py +++ b/datasets/daphnet.py @@ -3,7 +3,6 @@ from pathlib import Path import numpy as np import pandas as pd -import matplotlib.pyplot as plt from benchmark_utils.download import fetch_tsb_uad @@ -104,7 +103,7 @@ def load_data(db_path, record_ids=None, verbose=False, number=-1): class Dataset(BaseDataset): name = "DAPHNET" - requirements = ["pip:pooch"] + requirements = ["pip::pooch"] parameters = { # "recordings_id": [["S01R02E0"]], @@ -144,17 +143,6 @@ def get_data(self): X_test = X_test.reshape(n_recordings, 1, -1) y_test = y_test.reshape(n_recordings, -1) - plt.figure(figsize=(6, 3)) - plt.plot(X_train[0, 0, :500], linewidth=1.2) - plt.plot(range(297, 305), - X_train[0, 0, 297:305], color="orange", linewidth=3) - plt.title("Daphnet dataset") - plt.tight_layout() - plt.savefig("daphnet_example.png") - plt.close() - - print("PLOT SAVED") - return dict( X_train=X_train, y_test=y_test, diff --git a/datasets/dodgers.py b/datasets/dodgers.py index 8d3b7a2..4f41e37 100644 --- a/datasets/dodgers.py +++ b/datasets/dodgers.py @@ -90,7 +90,7 @@ def load_data(db_path, record_ids=None, verbose=False): class Dataset(BaseDataset): name = "DODGERS" - requirements = ["pip:pooch"] + requirements = ["pip::pooch"] parameters = { # "recordings_id": [["101"]], diff --git a/datasets/ecg.py b/datasets/ecg.py index 4db6e3b..d3ab30d 100644 --- a/datasets/ecg.py +++ b/datasets/ecg.py @@ -95,16 +95,19 @@ def load_data(db_path, record_ids=None, verbose=False, number=-1): class Dataset(BaseDataset): name = "ECG" - requirements = ["pip:pooch"] + requirements = ["pip::pooch"] parameters = { - "recordings_id": [["1", "2"]], + "recordings_id": [ + ["MBA_ECG14046_data_1", "MBA_ECG14046_data_2"], + "all", + ], "debug": [False], "number": [-1], } def get_data(self): - """Load the MITDB dataset.""" + """Load the ECG dataset.""" path = fetch_tsb_uad("ECG") # X shape (n_recordings, n_samples) diff --git a/datasets/genesis.py b/datasets/genesis.py index f6b62b1..b49a996 100644 --- a/datasets/genesis.py +++ b/datasets/genesis.py @@ -91,7 +91,7 @@ def load_data(db_path, record_ids=None, verbose=False): class Dataset(BaseDataset): name = "GENESIS" - requirements = ["pip:pooch"] + requirements = ["pip::pooch"] parameters = { "recordings_id": [["1", "2"]], diff --git a/datasets/ghl.py b/datasets/ghl.py index 2e29fe8..abff366 100644 --- a/datasets/ghl.py +++ b/datasets/ghl.py @@ -95,7 +95,7 @@ def load_data(db_path, record_ids=None, verbose=False): class Dataset(BaseDataset): name = "GHL" - requirements = ["pip:pooch"] + requirements = ["pip::pooch"] parameters = { "recordings_id": [["1", "2"]], diff --git a/datasets/iops.py b/datasets/iops.py index d6909ba..7003eb4 100644 --- a/datasets/iops.py +++ b/datasets/iops.py @@ -109,7 +109,7 @@ def load_data(db_path, verbose=False): class Dataset(BaseDataset): name = "IOPS" - requirements = ["pip:pooch"] + requirements = ["pip::pooch"] parameters = { "debug": [False], diff --git a/datasets/kdd21.py b/datasets/kdd21.py index 1ed8581..8fadd20 100644 --- a/datasets/kdd21.py +++ b/datasets/kdd21.py @@ -89,7 +89,7 @@ def load_data(db_path, record_ids=None, verbose=False): class Dataset(BaseDataset): name = "KDD21" - requirements = ["pip:pooch"] + requirements = ["pip::pooch"] parameters = { "recordings_id": [["1", "2"]], diff --git a/datasets/mgab.py b/datasets/mgab.py index f01978b..f854399 100644 --- a/datasets/mgab.py +++ b/datasets/mgab.py @@ -86,7 +86,7 @@ def load_data(db_path, record_ids=None, verbose=False): class Dataset(BaseDataset): name = "MGAB" - requirements = ["pip:pooch"] + requirements = ["pip::pooch"] parameters = { "recordings_id": [["1", "2"]], diff --git a/datasets/mitdb.py b/datasets/mitdb.py index 5cf8668..b7abebd 100644 --- a/datasets/mitdb.py +++ b/datasets/mitdb.py @@ -102,7 +102,7 @@ def load_mitdb_data(db_path, record_ids=None, verbose=False): class Dataset(BaseDataset): name = "MITDB" - requirements = ["pip:pooch"] + requirements = ["pip::pooch"] parameters = { "recordings_id": [["100", "201", "109", "105", "111", "221"]], diff --git a/datasets/nab.py b/datasets/nab.py index 88b1d0f..dc1e970 100644 --- a/datasets/nab.py +++ b/datasets/nab.py @@ -88,7 +88,7 @@ def load_data(db_path, record_ids=None, verbose=False): class Dataset(BaseDataset): name = "NAB" - requirements = ["pip:pooch"] + requirements = ["pip::pooch"] parameters = { "recordings_id": [["art0"], ["art1"], ["CloudWatch"]], diff --git a/datasets/occupancy.py b/datasets/occupancy.py index 3a466af..61a141b 100644 --- a/datasets/occupancy.py +++ b/datasets/occupancy.py @@ -111,7 +111,7 @@ def load_data(db_path, record_ids=None, verbose=False): class Dataset(BaseDataset): name = "OCCUPANCY" - requirements = ["pip:pooch"] + requirements = ["pip::pooch"] parameters = { "recordings_id": [None], diff --git a/datasets/opportunity.py b/datasets/opportunity.py index e8e0e62..25fbee0 100644 --- a/datasets/opportunity.py +++ b/datasets/opportunity.py @@ -91,7 +91,7 @@ def load_data(db_path, record_ids=None, verbose=False): class Dataset(BaseDataset): name = "OPPORTUNITY" - requirements = ["pip:pooch"] + requirements = ["pip::pooch"] parameters = { "recordings_id": [["1", "2"]], diff --git a/datasets/psm.py b/datasets/psm.py index b5ce22c..6b4b3d7 100644 --- a/datasets/psm.py +++ b/datasets/psm.py @@ -55,6 +55,9 @@ def get_data(self): y_test = pd.read_csv(path / "PSM_test_label.csv").to_numpy()[:, 1] + # Make sure the data has shape (n_samples, n_features, n_times) + X_train, X_test = X_train[:, None], X_test[:, None] + # Limiting the size of the dataset for testing purposes if self.debug: X_train = X_train[:1000] diff --git a/datasets/sensorscope.py b/datasets/sensorscope.py index 4ea5fdb..53f8d39 100644 --- a/datasets/sensorscope.py +++ b/datasets/sensorscope.py @@ -90,7 +90,7 @@ def load_data(db_path, record_ids=None, verbose=False): class Dataset(BaseDataset): name = "SENSORSCOPE" - requirements = ["pip:pooch"] + requirements = ["pip::pooch"] parameters = { "recordings_id": [["10", "11"]], diff --git a/datasets/smd.py b/datasets/smd.py index 25d6a0e..18060fe 100644 --- a/datasets/smd.py +++ b/datasets/smd.py @@ -94,7 +94,7 @@ def load_data(db_path, record_ids=None): class Dataset(BaseDataset): name = "SMD" - requirements = ["pip:pooch"] + requirements = ["pip::pooch"] parameters = { "recordings_id": [["1", "2"]], @@ -122,7 +122,7 @@ def get_data(self): # Reshaping data to (n_recordings, n_features, n_samples) # For SMD, treat as single recording - n_features = X_train.shape[1] + n_features = X_train.shape[0] X_train = X_train.T.reshape(1, n_features, -1) X_test = X_test.T.reshape(1, n_features, -1) y_test = y_test.reshape(1, -1) diff --git a/datasets/svdb.py b/datasets/svdb.py index 9abdd0a..03cce1b 100644 --- a/datasets/svdb.py +++ b/datasets/svdb.py @@ -3,7 +3,6 @@ from pathlib import Path import numpy as np import pandas as pd -import matplotlib.pyplot as plt from benchmark_utils.download import fetch_tsb_uad @@ -102,7 +101,7 @@ def load_data(db_path, record_ids=None, verbose=False, number=-1): class Dataset(BaseDataset): name = "SVDB" - requirements = ["pip:pooch"] + requirements = ["pip::pooch"] parameters = { "recordings_id": [["801"]], @@ -137,17 +136,6 @@ def get_data(self): X_test = X_test.reshape(n_recordings, 1, -1) y_test = y_test.reshape(n_recordings, -1) - plt.figure(figsize=(6, 3)) - plt.plot(X_train[0, 0, :500], linewidth=1.2) - plt.plot(range(350, 360), - X_train[0, 0, 350:360], color="orange", linewidth=3) - plt.title("SVDB dataset") - plt.tight_layout() - plt.savefig("svdb_example.png") - plt.close() - - print("PLOT SAVED") - return dict( X_train=X_train, y_test=y_test, diff --git a/datasets/trend.py b/datasets/trend.py index 44db101..eca3e91 100644 --- a/datasets/trend.py +++ b/datasets/trend.py @@ -62,18 +62,4 @@ def get_data(self): y_test = info_contam["outliers_mask"][self.n_samples:] y_test = np.any(y_test, axis=1) - import matplotlib.pyplot as plt - # Plot example time series with trend - plt.figure(figsize=(10, 4)) - plt.plot(X_train[0, 0, :]) - plt.title('Example Time Series with Added Trend') - plt.xlabel('Time') - plt.ylabel('Value') - plt.legend() - plt.show() - - print(f"X_train shape: {X_train.shape}") - print(f"X_test shape: {X_test.shape}") - print(f"y_test shape: {y_test.shape}") - return dict(X_train=X_train, y_test=y_test, X_test=X_test) diff --git a/datasets/yahoo.py b/datasets/yahoo.py index bbdee90..c040c8a 100644 --- a/datasets/yahoo.py +++ b/datasets/yahoo.py @@ -105,7 +105,7 @@ def load_data(db_path, record_ids=None, verbose=False): class Dataset(BaseDataset): name = "YAHOO" - requirements = ["pip:pooch"] + requirements = ["pip::pooch"] parameters = { "recordings_id": [["1"]], diff --git a/objective.py b/objective.py index 04dbde5..feaf89b 100644 --- a/objective.py +++ b/objective.py @@ -27,6 +27,8 @@ class Objective(BaseObjective): install_cmd = "conda" requirements = ["scikit-learn"] + # Do not track multiple results per config + sampling_strategy = "run_once" parameters = { "score_metrics": [("auc_pr", "auc_roc")], @@ -121,8 +123,6 @@ def evaluate_result( ) ) - # Setting value to 0. The actual value is not used for ranking. - result["value"] = 0.0 return result def get_objective(self): diff --git a/plots/delta_bar.py b/plots/delta_bar.py new file mode 100644 index 0000000..79ca343 --- /dev/null +++ b/plots/delta_bar.py @@ -0,0 +1,77 @@ +import matplotlib.colors as mcolors +import matplotlib.pyplot as plt + +from benchopt import BasePlot + + +def _short_name(name): + """Strip parameters from a benchopt component name.""" + return name.split("[")[0] + + +class Plot(BasePlot): + """Bar chart showing per-dataset score difference between two solvers. + + Bars represent ``score(solver_b) − score(solver_a)`` for each dataset, + sorted from most negative to most positive. Green bars indicate that + solver B wins on that dataset; red bars indicate that solver A wins. + """ + + name = "Per-dataset delta" + type = "bar_chart" + options = { + "metric": ["objective_auc_pr", "objective_auc_roc"], + } + + def plot(self, df, metric): + solvers = sorted(df["solver_name"].unique()) + if len(solvers) < 2: + return [] + + solver_a, solver_b = solvers[0], solvers[1] + pivot = ( + df.pivot_table( + index="dataset_name", + columns="solver_name", + values=metric, + aggfunc="median", + )[[solver_a, solver_b]] + .dropna() + ) + + delta = (pivot[solver_b] - pivot[solver_a]) + delta.index = delta.index.map(_short_name) + grouped = delta.groupby(level=0).apply(list) + grouped = grouped.reindex( + grouped.map(lambda v: sum(v) / len(v)).sort_values().index + ) + + medians = {ds: sorted(v)[len(v) // 2] for ds, v in grouped.items()} + all_med = list(medians.values()) + vmin = min(min(all_med), -1e-9) + vmax = max(max(all_med), 1e-9) + norm = mcolors.TwoSlopeNorm(vmin=vmin, vcenter=0.0, vmax=vmax) + cmap = plt.get_cmap("RdYlGn") + + bars = [] + for short_ds, vals in grouped.items(): + median = medians[short_ds] + color = mcolors.to_hex(cmap(norm(median))) + bars.append( + { + "y": [float(v) for v in vals], + "label": short_ds, + "color": color, + } + ) + return bars + + def get_metadata(self, df, metric): + solvers = sorted(df["solver_name"].unique()) + a = _short_name(solvers[0]) if solvers else "Solver A" + b = _short_name(solvers[1]) if len(solvers) > 1 else "Solver B" + m = metric.replace("objective_", "").upper().replace("_", "-") + return { + "title": f"Per-dataset delta: {b} − {a} ({m})", + "ylabel": f"Δ {m} ({b} − {a})", + } diff --git a/plots/head_to_head.py b/plots/head_to_head.py new file mode 100644 index 0000000..62b3ab8 --- /dev/null +++ b/plots/head_to_head.py @@ -0,0 +1,72 @@ +import numpy as np +from benchopt import BasePlot + + +def _short_name(name): + """Strip parameters from a benchopt component name.""" + return name.split("[")[0] + + +class Plot(BasePlot): + """Scatter plot comparing two solvers head-to-head across all datasets. + + Each point represents one dataset. Points above the diagonal indicate that + the solver on the y-axis outperforms the one on the x-axis. + """ + + name = "Head-to-head" + type = "scatter" + options = { + "metric": ["objective_auc_pr", "objective_auc_roc"], + } + + def plot(self, df, metric): + solvers = sorted(df["solver_name"].unique()) + if len(solvers) < 2: + return [] + + solver_a, solver_b = solvers[0], solvers[1] + pivot = ( + df.pivot_table( + index="dataset_name", + columns="solver_name", + values=metric, + aggfunc="mean", + )[[solver_a, solver_b]] + .dropna() + ) + + traces = [] + for dataset in pivot.index: + x = float(pivot.loc[dataset, solver_a]) + y = float(pivot.loc[dataset, solver_b]) + label = _short_name(dataset) + traces.append( + {"x": [x], "y": [y], "label": label, **self.get_style(label)} + ) + + vals = pivot.values.flatten() + lo, hi = float(np.nanmin(vals)), float(np.nanmax(vals)) + pad = (hi - lo) * 0.05 + traces.append( + { + "x": [lo - pad, hi + pad], + "y": [lo - pad, hi + pad], + "label": "Equal performance", + "color": "gray", + "marker": "", + } + ) + return traces + + def get_metadata(self, df, metric): + solvers = sorted(df["solver_name"].unique()) + a = _short_name(solvers[0]) if solvers else "Solver A" + b = _short_name(solvers[1]) if len(solvers) > 1 else "Solver B" + m = metric.replace("objective_", "").upper().replace("_", "-") + return { + "title": f"Head-to-head: {m}", + "xlabel": f"{a} ({m})", + "ylabel": f"{b} ({m})", + "scale": "linear", + } diff --git a/solvers/AR.py b/solvers/AR.py index 50eb908..eb1e2fa 100644 --- a/solvers/AR.py +++ b/solvers/AR.py @@ -17,8 +17,6 @@ class Solver(BaseSolver): install_cmd = "conda" requirements = ["pytorch", "tqdm"] - sampling_strategy = "run_once" - parameters = { "batch_size": [128], "n_epochs": [50], diff --git a/solvers/anomalybert.py b/solvers/anomalybert.py index 8c9d1b4..7d028b7 100644 --- a/solvers/anomalybert.py +++ b/solvers/anomalybert.py @@ -17,7 +17,6 @@ class Solver(BaseSolver): name = "AnomalyBERT" - sampling_strategy = "run_once" requirements = ["pip::timm", "pytorch", "numpy", "tqdm"] @@ -35,8 +34,6 @@ class Solver(BaseSolver): "cutoff": [None], } - sampling_strategy = "run_once" - def set_objective(self, X_train, X_test): # X_train shape: (n_series, n_features, n_samples) if X_train.ndim == 3: diff --git a/solvers/autoencoder.py b/solvers/autoencoder.py index a36fad0..19d4320 100644 --- a/solvers/autoencoder.py +++ b/solvers/autoencoder.py @@ -29,8 +29,6 @@ class Solver(BaseSolver): "batch_size": 8, } - sampling_strategy = "run_once" - def set_objective(self, X_train, X_test): if self.window_size == "auto": self.window_size = find_period_length(X_train.reshape(-1)) diff --git a/solvers/dagmm.py b/solvers/dagmm.py index 148fbc3..45491b5 100644 --- a/solvers/dagmm.py +++ b/solvers/dagmm.py @@ -26,8 +26,6 @@ class Solver(BaseSolver): # "device": ["cuda:3"] } - sampling_strategy = "run_once" - def set_objective(self, X_train, X_test): n_features = X_train.shape[1] self.X_train = X_train.transpose(0, 2, 1).reshape(-1, n_features) diff --git a/solvers/lstm.py b/solvers/lstm.py index ff4a975..8a9a009 100644 --- a/solvers/lstm.py +++ b/solvers/lstm.py @@ -19,8 +19,6 @@ class Solver(BaseSolver): install_cmd = "conda" requirements = ["pytorch", "tqdm"] - sampling_strategy = "run_once" - parameters = { "embedding_dim": [64], "batch_size": [32], diff --git a/solvers/matrixprofile.py b/solvers/matrixprofile.py index 7b91d8d..e7a2179 100644 --- a/solvers/matrixprofile.py +++ b/solvers/matrixprofile.py @@ -24,8 +24,6 @@ class Solver(BaseSolver): "window_size": 8, } - sampling_strategy = "run_once" - def set_objective(self, X_train, X_test): # Shapes received: (n_recordings, n_features, n_samples) self.X_train = X_train diff --git a/solvers/rosecdl.py b/solvers/rosecdl.py index 6ccf54c..bb235f4 100644 --- a/solvers/rosecdl.py +++ b/solvers/rosecdl.py @@ -39,8 +39,6 @@ class Solver(BaseSolver): "cutoff": [None], } - sampling_strategy = "run_once" - def set_objective(self, X_train, X_test): self.device = torch.device( "cuda" if torch.cuda.is_available() else "cpu") diff --git a/solvers/tsb_chronos.py b/solvers/tsb_chronos.py index 855c842..9f93fa5 100644 --- a/solvers/tsb_chronos.py +++ b/solvers/tsb_chronos.py @@ -22,8 +22,6 @@ class Solver(BaseSolver): "cutoff": [None], } - sampling_strategy = "run_once" - def set_objective(self, X_train, X_test): _, n_features, _ = X_train.shape self.data = np.append(X_train, X_test, axis=2) diff --git a/solvers/tsb_timesfm.py b/solvers/tsb_timesfm.py index cfa591b..b2fbbb2 100644 --- a/solvers/tsb_timesfm.py +++ b/solvers/tsb_timesfm.py @@ -20,8 +20,6 @@ class Solver(BaseSolver): "cutoff": [None], } - sampling_strategy = "run_once" - def set_objective(self, X_train, X_test): _, n_features, _ = X_train.shape self.data = np.append(X_train, X_test, axis=2) diff --git a/solvers/tsb_timesnet.py b/solvers/tsb_timesnet.py index ed431ae..355a588 100644 --- a/solvers/tsb_timesnet.py +++ b/solvers/tsb_timesnet.py @@ -31,8 +31,6 @@ class Solver(BaseSolver): "batch_size": 16, } - sampling_strategy = "run_once" - def set_objective(self, X_train, X_test): _, n_features, _ = X_train.shape self.X_train = X_train.reshape(-1, n_features) diff --git a/solvers/vae.py b/solvers/vae.py index 9dfd400..a9a5c53 100644 --- a/solvers/vae.py +++ b/solvers/vae.py @@ -13,8 +13,6 @@ class Solver(BaseSolver): install_cmd = "conda" requirements = ["pyod", "pytorch"] - sampling_strategy = "run_once" - parameters = { "contamination": [0.005, 0.05, 0.1, 0.2], "n_epochs": [50], diff --git a/solvers/vanilla-transformer.py b/solvers/vanilla-transformer.py index 11f91dd..c09f96d 100644 --- a/solvers/vanilla-transformer.py +++ b/solvers/vanilla-transformer.py @@ -20,8 +20,6 @@ class Solver(BaseSolver): install_cmd = "conda" requirements = ["pytorch", "tqdm"] - sampling_strategy = "run_once" - parameters = { "num_layers": [1], "num_heads": [2],