From 972adf74185c675f7d73b30c348f85096b43c105 Mon Sep 17 00:00:00 2001 From: yemeen Date: Tue, 14 Jan 2025 18:19:46 -0500 Subject: [PATCH 1/5] Add benchmark workflow for ECT package and benchmark scripts --- .github/workflows/benchmark.yml | 33 +++++++++++++++++++++ benchmarks/run_benchmarks.py | 51 +++++++++++++++++++++++++++++++++ 2 files changed, 84 insertions(+) create mode 100644 .github/workflows/benchmark.yml create mode 100644 benchmarks/run_benchmarks.py diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml new file mode 100644 index 0000000..0e9353a --- /dev/null +++ b/.github/workflows/benchmark.yml @@ -0,0 +1,33 @@ +name: Benchmark +on: + push: + branches: [ "main" ] + paths: + - '**.py' + - 'benchmarks/**' + +jobs: + benchmark: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.10' + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -e . + pip install numpy matplotlib + + - name: Run benchmarks + run: python benchmarks/run_benchmarks.py + + - name: Store benchmark results + uses: actions/upload-artifact@v3 + with: + name: benchmark-results + path: benchmark_results/ diff --git a/benchmarks/run_benchmarks.py b/benchmarks/run_benchmarks.py new file mode 100644 index 0000000..d96e87f --- /dev/null +++ b/benchmarks/run_benchmarks.py @@ -0,0 +1,51 @@ +"""Main benchmark runner for ECT package""" +import numpy as np +import time +from ect import ECT, EmbeddedGraph +import json +from pathlib import Path + + +def create_test_shape(num_points=1000): + t = np.linspace(0, 2*np.pi, num_points) + x = np.cos(t) + 0.5 * np.cos(3*t) + y = np.sin(t) + 0.5 * np.sin(3*t) + return np.column_stack([x, y]) + + +def run_benchmarks(): + results = {} + + sizes = [100, 500, 1000] + for size in sizes: + shape = create_test_shape(size) + G = EmbeddedGraph() + G.add_cycle(shape) + + start_time = time.time() + myect = ECT(num_dirs=360, num_thresh=360) + myect.calculateECT(G) + ect_time = time.time() - start_time + + start_time = time.time() + for theta in np.linspace(0, 2*np.pi, 360): + G.g_omega(theta) + g_omega_time = time.time() - start_time + + results[f'shape_size_{size}'] = { + 'ect_time': ect_time, + 'g_omega_time': g_omega_time + } + + return results + + +if __name__ == "__main__": + + results = run_benchmarks() + + output_dir = Path("benchmark_results") + output_dir.mkdir(exist_ok=True) + + with open(output_dir / "results.json", "w") as f: + json.dump(results, f, indent=2) From b3cfe69327180bb1205ed5489523e91ac150f6c6 Mon Sep 17 00:00:00 2001 From: yemeen Date: Wed, 15 Jan 2025 15:08:24 -0500 Subject: [PATCH 2/5] Update to uv installation and venv setup in benchmark workflow --- .github/workflows/benchmark.yml | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 0e9353a..35d2bf7 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -17,14 +17,21 @@ jobs: with: python-version: '3.10' - - name: Install dependencies + - name: Install uv run: | - python -m pip install --upgrade pip - pip install -e . - pip install numpy matplotlib + curl -LsSf https://astral.sh/uv/install.sh | sh + + - name: Create venv and install dependencies + run: | + uv venv + source .venv/bin/activate + uv pip install -e . + uv pip install numpy matplotlib - name: Run benchmarks - run: python benchmarks/run_benchmarks.py + run: | + source .venv/bin/activate + python benchmarks/run_benchmarks.py - name: Store benchmark results uses: actions/upload-artifact@v3 From 357e6260145d886118dd8c8ed4c9da004e1bd72f Mon Sep 17 00:00:00 2001 From: yemeen Date: Wed, 15 Jan 2025 15:44:46 -0500 Subject: [PATCH 3/5] Benchmarks for graph and cw computations. --- .gitignore | 2 + benchmarks/benchmark_cw.py | 53 ++++++++++++++++++++++++ benchmarks/benchmark_graph.py | 78 +++++++++++++++++++++++++++++++++++ benchmarks/run_benchmarks.py | 67 ++++++++++++++---------------- 4 files changed, 165 insertions(+), 35 deletions(-) create mode 100644 benchmarks/benchmark_cw.py create mode 100644 benchmarks/benchmark_graph.py diff --git a/.gitignore b/.gitignore index c01e0f5..89bd6d2 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,8 @@ doc_source/notebooks/Matisse/outlines/* *.DS_Store +benchmarks/results/* + # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] diff --git a/benchmarks/benchmark_cw.py b/benchmarks/benchmark_cw.py new file mode 100644 index 0000000..78a2b53 --- /dev/null +++ b/benchmarks/benchmark_cw.py @@ -0,0 +1,53 @@ +"""Benchmarks for CW complex computations""" +import numpy as np +import time +from ect import ECT, EmbeddedCW, create_example_cw +import json +from pathlib import Path + + +def benchmark_cw_ect(num_runs=5): + """Benchmark ECT computation on CW complexes""" + results = {} + + configs = [ + (8, 10), # Small + (36, 36), # Medium + (360, 360), # Large + ] + + for num_dir, num_thresh in configs: + times = [] + K = create_example_cw() + + print( + f"\nTesting ECT with {num_dir} directions, {num_thresh} thresholds") + for _ in range(num_runs): + start_time = time.time() + + myect = ECT(num_dirs=num_dir, num_thresh=num_thresh) + myect.calculateECT(K) + + execution_time = time.time() - start_time + times.append(execution_time) + + results[f'dirs_{num_dir}_thresh_{num_thresh}'] = { + 'mean_time': float(np.mean(times)), + 'std_time': float(np.std(times)), + 'min_time': float(np.min(times)), + 'max_time': float(np.max(times)) + } + + return results + + +if __name__ == "__main__": + print("Running CW complex benchmarks...") + results = benchmark_cw_ect() + + # Save results + output_dir = Path("benchmark_results") + output_dir.mkdir(exist_ok=True) + + with open(output_dir / "cw_results.json", "w") as f: + json.dump(results, f, indent=2) diff --git a/benchmarks/benchmark_graph.py b/benchmarks/benchmark_graph.py new file mode 100644 index 0000000..96ee6c3 --- /dev/null +++ b/benchmarks/benchmark_graph.py @@ -0,0 +1,78 @@ +"""Benchmarks for graph-based ECT computations""" +import numpy as np +import time +from ect import ECT, EmbeddedGraph + + +def create_test_shape(num_points=1000, complexity=1): + """Create test shape with varying complexity""" + t = np.linspace(0, 2*np.pi, num_points) + x = np.cos(t) + y = np.sin(t) + + for i in range(2, complexity + 2): + x += (1/i) * np.cos(i*t) + y += (1/i) * np.sin(i*t) + + return np.column_stack([x, y]) + + +def benchmark_graph_ect(num_runs=5): + """Benchmark ECT computation on graphs""" + results = {} + + configs = [ + (100, 1), # Simple shape, few points + (1000, 1), # Simple shape, many points + (100, 3), # Complex shape, few points + (1000, 3), # Complex shape, many points + ] + + for points, complexity in configs: + shape = create_test_shape(points, complexity) + G = EmbeddedGraph() + G.add_cycle(shape) + + times = [] + print( + f"\nTesting shape with {points} points and complexity {complexity}") + + for _ in range(num_runs): + start_time = time.time() + myect = ECT(num_dirs=360, num_thresh=360) + myect.calculateECT(G) + times.append(time.time() - start_time) + + results[f'points_{points}_complexity_{complexity}'] = { + 'mean_time': float(np.mean(times)), + 'std_time': float(np.std(times)), + 'min_time': float(np.min(times)), + 'max_time': float(np.max(times)) + } + + return results + + +def benchmark_g_omega(num_runs=5): + """Benchmark g_omega computation""" + results = {} + + sizes = [100, 500, 1000] + for size in sizes: + shape = create_test_shape(size) + G = EmbeddedGraph() + G.add_cycle(shape) + + times = [] + for _ in range(num_runs): + start_time = time.time() + for theta in np.linspace(0, 2*np.pi, 360): + G.g_omega(theta) + times.append(time.time() - start_time) + + results[f'size_{size}'] = { + 'mean_time': float(np.mean(times)), + 'std_time': float(np.std(times)) + } + + return results diff --git a/benchmarks/run_benchmarks.py b/benchmarks/run_benchmarks.py index d96e87f..6f30bbb 100644 --- a/benchmarks/run_benchmarks.py +++ b/benchmarks/run_benchmarks.py @@ -1,51 +1,48 @@ """Main benchmark runner for ECT package""" import numpy as np import time -from ect import ECT, EmbeddedGraph -import json from pathlib import Path +import json +from benchmark_graph import benchmark_graph_ect, benchmark_g_omega +from benchmark_cw import benchmark_cw_ect +import platform -def create_test_shape(num_points=1000): - t = np.linspace(0, 2*np.pi, num_points) - x = np.cos(t) + 0.5 * np.cos(3*t) - y = np.sin(t) + 0.5 * np.sin(3*t) - return np.column_stack([x, y]) - - -def run_benchmarks(): - results = {} - - sizes = [100, 500, 1000] - for size in sizes: - shape = create_test_shape(size) - G = EmbeddedGraph() - G.add_cycle(shape) +def run_all_benchmarks(num_runs=5): + """Run all benchmarks and collect results""" + results = { + 'metadata': { + 'num_runs': num_runs, + 'timestamp': time.strftime('%Y-%m-%d %H:%M:%S'), + 'platform': platform.platform(), + 'python_version': platform.python_version() + }, + 'benchmarks': {} + } - start_time = time.time() - myect = ECT(num_dirs=360, num_thresh=360) - myect.calculateECT(G) - ect_time = time.time() - start_time + print("\nRunning graph ECT benchmarks...") + results['benchmarks']['graph_ect'] = benchmark_graph_ect(num_runs=num_runs) - start_time = time.time() - for theta in np.linspace(0, 2*np.pi, 360): - G.g_omega(theta) - g_omega_time = time.time() - start_time + print("\nRunning CW complex benchmarks...") + results['benchmarks']['cw_ect'] = benchmark_cw_ect(num_runs=num_runs) - results[f'shape_size_{size}'] = { - 'ect_time': ect_time, - 'g_omega_time': g_omega_time - } + print("\nRunning g_omega benchmarks...") + results['benchmarks']['g_omega'] = benchmark_g_omega(num_runs=num_runs) return results -if __name__ == "__main__": - - results = run_benchmarks() - - output_dir = Path("benchmark_results") +def save_results(results, output_dir="benchmarks/results"): + """Save benchmark results to JSON file""" + output_dir = Path(output_dir) output_dir.mkdir(exist_ok=True) - with open(output_dir / "results.json", "w") as f: + with open(output_dir / "benchmark_results.json", "w") as f: json.dump(results, f, indent=2) + + print(f"\nResults saved to {output_dir}/benchmark_results.json") + + +if __name__ == "__main__": + results = run_all_benchmarks() + save_results(results) From a1facacb626b6a135526e657ce70ae77d7dd67a0 Mon Sep 17 00:00:00 2001 From: yemeen Date: Wed, 15 Jan 2025 15:49:53 -0500 Subject: [PATCH 4/5] Add benchmark configuration for 10000 points --- benchmarks/benchmark_graph.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/benchmarks/benchmark_graph.py b/benchmarks/benchmark_graph.py index 96ee6c3..cdfad0d 100644 --- a/benchmarks/benchmark_graph.py +++ b/benchmarks/benchmark_graph.py @@ -22,10 +22,11 @@ def benchmark_graph_ect(num_runs=5): results = {} configs = [ - (100, 1), # Simple shape, few points - (1000, 1), # Simple shape, many points - (100, 3), # Complex shape, few points - (1000, 3), # Complex shape, many points + (100, 1), + (1000, 1), + (100, 3), + (1000, 3), + (10000, 3), ] for points, complexity in configs: From 6b4afa485d2d4e866602537418dc4f094442e17d Mon Sep 17 00:00:00 2001 From: yemeen Date: Wed, 15 Jan 2025 16:01:18 -0500 Subject: [PATCH 5/5] Add benchmark target to run performance benchmarks --- Makefile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Makefile b/Makefile index b512834..9f159fe 100644 --- a/Makefile +++ b/Makefile @@ -29,6 +29,9 @@ html: sphinx-build -M html doc_source docs rsync -a docs/html/ docs/ rm -r docs/html + +benchmark: + python benchmarks/run_benchmarks.py all: # Running autopep8