Add CodSpeed integration for continuous performance testing

.github/workflows/codspeed.yml

@@ -0,0 +1,54 @@
+name: CodSpeed
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+  workflow_dispatch:
+
+permissions:
+  contents: read
+  id-token: write
+
+jobs:
+  codspeed:
+    name: Run benchmarks
+    runs-on: codspeed-macro
+    defaults:
+      run:
+        shell: bash -el {0}
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: true
+          fetch-depth: 0
+
+      - name: Setup conda environment
+        uses: mamba-org/setup-micromamba@v2
+        with:
+          micromamba-version: 2.5.0-1
+          environment-file: .buildconfig/ci-linux.yml
+          cache-environment: true
+          create-args: python=3.11
+
+      - name: ccache
+        uses: hendrikmuhs/ccache-action@v1.2.20
+        with:
+          key: ubuntu-24.04
+
+      - name: Build scipp and setup deps
+        run: |
+          python -m pip install -r requirements/test.in
+          python -m pip install pytest-codspeed
+          cmake --preset ci-linux
+          cmake --build --preset build
+
+      - name: Run benchmarks
+        uses: CodSpeedHQ/action@v4
+        with:
+          mode: walltime
+          # Custom shells aren't supported
+          # https://github.com/CodSpeedHQ/action/issues/65
+          run: bash -el -c "PYTHONPATH=${{ github.workspace }}/install LD_LIBRARY_PATH=${{ github.workspace }}/install/lib:$LD_LIBRARY_PATH pytest bench/ --codspeed"

README.md

@@ -4,6 +4,7 @@
 [![License: BSD 3-Clause](https://img.shields.io/badge/License-BSD%203--Clause-blue.svg)](LICENSE)
 [![CII Best Practices](https://bestpractices.coreinfrastructure.org/projects/4019/badge)](https://bestpractices.coreinfrastructure.org/projects/4019)
 [![DOI](https://zenodo.org/badge/147631466.svg)](https://zenodo.org/badge/latestdoi/147631466)
+[![CodSpeed](https://img.shields.io/endpoint?url=https://codspeed.io/badge.json)](https://codspeed.io/MridulS/scipp?utm_source=badge)
 
 # Scipp
 

bench/__init__.py

@@ -0,0 +1,2 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright (c) 2023 Scipp contributors (https://github.com/scipp)

bench/test_bin_bench.py

@@ -0,0 +1,86 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright (c) 2023 Scipp contributors (https://github.com/scipp)
+import numpy as np
+import pytest
+
+import scipp as sc
+from scipp.binning import make_binned
+
+
+@pytest.mark.parametrize("nbin", [1024, 2048, 4096, 8192])
+def test_bin2d_rebin_outer(benchmark, nbin):
+    binned = sc.data.binned_x(nevent=2 * nbin, nbin=nbin)
+    y = sc.linspace(dim='y', start=0, stop=1, num=2, unit='m')
+    da = make_binned(binned, edges=[y])
+    x = sc.linspace(dim='x', start=0, stop=1, num=nbin - 1, unit='m')
+
+    benchmark(make_binned, da, edges=[x])
+
+
+@pytest.mark.parametrize("nbin", [1024, 2048, 4096, 8192])
+def test_bin2d_rebin_outer_transposed(benchmark, nbin):
+    binned = sc.data.binned_x(nevent=2 * nbin, nbin=nbin)
+    y = sc.linspace(dim='y', start=0, stop=1, num=2, unit='m')
+    da = make_binned(binned, edges=[y])
+    x = sc.linspace(dim='x', start=0, stop=1, num=nbin - 1, unit='m')
+
+    benchmark(make_binned, da.transpose(), edges=[x])
+
+
+@pytest.mark.parametrize("nbin", [1024, 2048, 4096, 8192])
+def test_bin2d_rebin_outer_transposed_copied(benchmark, nbin):
+    binned = sc.data.binned_x(nevent=2 * nbin, nbin=nbin)
+    y = sc.linspace(dim='y', start=0, stop=1, num=2, unit='m')
+    da = make_binned(binned, edges=[y])
+    da_transposed = da.transpose().copy()
+    x = sc.linspace(dim='x', start=0, stop=1, num=nbin - 1, unit='m')
+
+    benchmark(make_binned, da_transposed, edges=[x])
+
+
+@pytest.mark.parametrize("nbin", [1, 2, 4, 8, 16, 32, 64, 128, 256, 512])
+def test_bin1d_table(benchmark, nbin):
+    table = sc.data.table_xyz(50_000_000)
+    x = sc.linspace(dim='x', start=0, stop=1, num=nbin + 1, unit='m')
+
+    benchmark(make_binned, table, edges=[x])
+
+
+@pytest.mark.parametrize("nevent,ngroup", [
+    (1_000, 4),
+    (1_000, 64),
+    (1_000, 1024),
+    (1_000_000, 4),
+    (1_000_000, 64),
+    (1_000_000, 1024),
+    (10_000_000, 4),
+    (10_000_000, 64),
+])
+def test_group_contiguous(benchmark, nevent, ngroup):
+    table = sc.data.table_xyz(nevent)
+    table.coords['group'] = (ngroup * table.coords['x']).to(dtype='int64')
+    del table.coords['x']
+    contiguous_groups = sc.arange('group', ngroup, unit='m')
+
+    benchmark(table.group, contiguous_groups)
+
+
+@pytest.mark.parametrize("nevent,ngroup", [
+    (1_000, 4),
+    (1_000, 64),
+    (1_000, 1024),
+    (1_000_000, 4),
+    (1_000_000, 64),
+    (1_000_000, 1024),
+    (10_000_000, 4),
+    (10_000_000, 64),
+])
+def test_group(benchmark, nevent, ngroup):
+    table = sc.data.table_xyz(nevent)
+    table.coords['group'] = (ngroup * table.coords['x']).to(dtype='int64')
+    del table.coords['x']
+    groups = sc.arange('group', ngroup, unit='m')
+    groups.values[0] = 1
+    groups.values[1] = 0
+
+    benchmark(table.group, groups)

bench/test_binned_bench.py

@@ -0,0 +1,166 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright (c) 2023 Scipp contributors (https://github.com/scipp)
+import numpy as np
+import pytest
+
+import scipp as sc
+from scipp.binning import make_binned
+
+
+@pytest.mark.parametrize("nevent,nbin", [
+    (1, 1),
+    (10, 1),
+    (100, 1),
+    (1000, 10),
+    (10000, 100),
+    (100000, 1000),
+])
+def test_bins_constituents(benchmark, nevent, nbin):
+    da = sc.data.binned_x(nevent, nbin)
+    benchmark(lambda: da.bins.constituents)
+
+
+@pytest.mark.parametrize("nevent,nbin", [
+    (1, 1),
+    (10, 1),
+    (100, 1),
+    (1000, 10),
+    (10000, 100),
+    (100000, 1000),
+])
+def test_bins_size(benchmark, nevent, nbin):
+    da = sc.data.binned_x(nevent, nbin)
+    benchmark(da.bins.size)
+
+
+@pytest.mark.parametrize("nevent,nbin", [
+    (1, 1),
+    (10, 1),
+    (100, 1),
+    (1000, 10),
+    (10000, 100),
+    (100000, 1000),
+])
+def test_bins_sum(benchmark, nevent, nbin):
+    da = sc.data.binned_x(nevent, nbin)
+    benchmark(da.bins.sum)
+
+
+@pytest.mark.parametrize("nevent,nbin", [
+    (1, 1),
+    (10, 1),
+    (100, 1),
+    (1000, 10),
+    (10000, 100),
+    (100000, 1000),
+])
+def test_bins_mean(benchmark, nevent, nbin):
+    da = sc.data.binned_x(nevent, nbin)
+    benchmark(da.bins.mean)
+
+
+@pytest.mark.parametrize("nevent,nbin", [
+    (1, 1),
+    (10, 1),
+    (100, 1),
+    (1000, 10),
+    (10000, 100),
+    (100000, 1000),
+])
+def test_bins_concat(benchmark, nevent, nbin):
+    da = sc.data.binned_x(nevent, nbin)
+    benchmark(da.bins.concat, 'x')
+
+
+@pytest.mark.parametrize("nbin", [1, 2, 4, 8, 16, 32, 64, 128])
+def test_binned2d_concat(benchmark, nbin):
+    nx = 100000
+    binned = sc.data.binned_x(nevent=2 * nx, nbin=nx)
+    y = sc.linspace(dim='y', start=0, stop=1, num=nbin + 1, unit='m')
+    da = make_binned(binned, edges=[y])
+
+    benchmark(da.bins.concat, 'x')
+
+
+@pytest.mark.parametrize("nbin", [1024, 2048, 4096, 8192])
+def test_binned2d_concat_inner(benchmark, nbin):
+    binned = sc.data.binned_x(nevent=2 * nbin, nbin=nbin)
+    y = sc.linspace(dim='y', start=0, stop=1, num=2, unit='m')
+    da = make_binned(binned, edges=[y])
+
+    benchmark(da.bins.concat, 'y')
+
+
+def test_lookup_create_bool(benchmark):
+    x = sc.linspace(dim='x', start=0.0, stop=1.0, num=1_000_001, unit='m')
+    groups = sc.arange(dim='x', start=0, stop=1_000_000) // 1000 % 5
+    hist_bool = sc.DataArray(data=groups.astype('bool'), coords={'x': x})
+
+    def create():
+        hist_bool.coords['x'].values[-1] *= 1.1
+        return sc.lookup(hist_bool, 'x')
+
+    benchmark(create)
+
+
+def test_lookup_create_float64(benchmark):
+    x = sc.linspace(dim='x', start=0.0, stop=1.0, num=1_000_001, unit='m')
+    groups = sc.arange(dim='x', start=0, stop=1_000_000) // 1000 % 5
+    hist_float = sc.DataArray(data=groups.astype('float64'), coords={'x': x})
+
+    def create():
+        hist_float.coords['x'].values[-1] *= 1.1
+        return sc.lookup(hist_float, 'x')
+
+    benchmark(create)
+
+
+def test_lookup_create_int64(benchmark):
+    x = sc.linspace(dim='x', start=0.0, stop=1.0, num=1_000_001, unit='m')
+    groups = sc.arange(dim='x', start=0, stop=1_000_000) // 1000 % 5
+    hist_int = sc.DataArray(data=groups, coords={'x': x})
+
+    def create():
+        hist_int.coords['x'].values[-1] *= 1.1
+        return sc.lookup(hist_int, 'x')
+
+    benchmark(create)
+
+
+def test_lookup_map_bool(benchmark):
+    binned_x = sc.data.binned_x(100_000_000, 10000).bins.coords['x']
+    x = sc.linspace(dim='x', start=0.0, stop=1.0, num=1_000_001, unit='m')
+    groups = sc.arange(dim='x', start=0, stop=1_000_000) // 1000 % 5
+    hist_bool = sc.DataArray(data=groups.astype('bool'), coords={'x': x})
+
+    def map_lookup():
+        hist_bool.coords['x'].values[-1] *= 1.1
+        return sc.lookup(hist_bool, 'x')[binned_x]
+
+    benchmark(map_lookup)
+
+
+def test_lookup_map_float64(benchmark):
+    binned_x = sc.data.binned_x(100_000_000, 10000).bins.coords['x']
+    x = sc.linspace(dim='x', start=0.0, stop=1.0, num=1_000_001, unit='m')
+    groups = sc.arange(dim='x', start=0, stop=1_000_000) // 1000 % 5
+    hist_float = sc.DataArray(data=groups.astype('float64'), coords={'x': x})
+
+    def map_lookup():
+        hist_float.coords['x'].values[-1] *= 1.1
+        return sc.lookup(hist_float, 'x')[binned_x]
+
+    benchmark(map_lookup)
+
+
+def test_lookup_map_int64(benchmark):
+    binned_x = sc.data.binned_x(100_000_000, 10000).bins.coords['x']
+    x = sc.linspace(dim='x', start=0.0, stop=1.0, num=1_000_001, unit='m')
+    groups = sc.arange(dim='x', start=0, stop=1_000_000) // 1000 % 5
+    hist_int = sc.DataArray(data=groups, coords={'x': x})
+
+    def map_lookup():
+        hist_int.coords['x'].values[-1] *= 1.1
+        return sc.lookup(hist_int, 'x')[binned_x]
+
+    benchmark(map_lookup)

bench/test_variable_bench.py

@@ -0,0 +1,48 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright (c) 2023 Scipp contributors (https://github.com/scipp)
+import numpy as np
+import pytest
+
+import scipp as sc
+
+
+@pytest.fixture
+def var1():
+    return sc.array(dims=['x'], values=[1, 2, 3, 4, 5])
+
+
+@pytest.fixture
+def var2():
+    return sc.array(dims=['x'], values=[1, 2, 3, 4, 5])
+
+
+def test_shallow_copy(benchmark, var1):
+    benchmark(lambda: var1.copy(deep=False))
+
+
+def test_deep_copy(benchmark, var1):
+    benchmark(lambda: var1.copy())
+
+
+def test_variable_inplace_operation(benchmark, var1, var2):
+    def operation():
+        v = var1.copy()
+        v += var2
+        return v
+
+    benchmark(operation)
+
+
+def test_variable_non_inplace_operation(benchmark, var1, var2):
+    benchmark(lambda: var1 + var2)
+
+
+@pytest.mark.parametrize("size", [10**5, 10**6, 10**7])
+def test_assign_from_numpy(benchmark, size):
+    array = np.arange(size, dtype=np.float64)
+    var = sc.Variable(dims=['x'], values=array)
+
+    def assign():
+        var.values = array
+
+    benchmark(assign)

pyproject.toml

@@ -42,6 +42,7 @@ dynamic = ["version"]
 [project.optional-dependencies]
 test = [
   "pytest",
+  "pytest-codspeed",
   "matplotlib",
   "beautifulsoup4",
   "ipython",
@@ -174,7 +175,7 @@ filterwarnings = [
   'ignore:sc.geometry.rotation_matrix_from_quaternion_coeffs has been deprecated:scipp.VisibleDeprecationWarning',
   # Comes from pytest_asyncio and is not our fault.
   "ignore:The 'asyncio_mode' default value will change to 'strict' in future:DeprecationWarning",
-  'ignore::scipy.optimize._optimize.OptimizeWarning',
+  # 'ignore::scipy.optimize._optimize.OptimizeWarning',
   'ignore:Support for mapping types has been deprecated and will be dropped in a future release.:DeprecationWarning',
   # Triggered by h5py
   'ignore:`product` is deprecated as of NumPy 1.25.0:DeprecationWarning'

requirements/test.in

@@ -3,4 +3,5 @@
 hypothesis
 pytest
 pytest-xdist
+pytest-codspeed
 beautifulsoup4