Add comparison

Author: anuraaga

.gitignore

@@ -4,3 +4,4 @@ __pycache__
 # VS Code configuration
 /.vscode/
 .envrc
+.benchmarks

README.md

@@ -75,6 +75,22 @@ Additionally, [protovalidate's core repository](https://github.com/bufbuild/prot
 - [Protovalidate's Protobuf API][validate-proto]
 - [Conformance testing utilities][conformance] for acceptance testing of `protovalidate` implementations
 
+## Benchmarks
+
+We include a set of benchmarks to verify different types of validation operations based on `pytest-benchmark`.
+You can run them with
+
+```shellsession
+$ uv run poe bench
+```
+
+This persists the runs results. It is often helpful to compare benchmark results between runs,
+for example before and after an optimization. To compare persisted results, use
+
+```shellsession
+$ uv run poe bench-compare
+```
+
 ## Contributing
 
 We genuinely appreciate any help! If you'd like to contribute, check out these resources:

poe_tasks.toml

@@ -17,7 +17,11 @@ go run github.com/bufbuild/buf/private/pkg/licenseheader/cmd/license-header@v1.6
 
 [tasks.bench]
 help = "Run benchmarks"
-cmd = "pytest --benchmark-enable --benchmark-only"
+cmd = "pytest --benchmark-enable --benchmark-only --benchmark-autosave"
+
+[tasks.bench-compare]
+help = "Compare saved benchmark runs"
+cmd = "pytest-benchmark compare --group-by=func,param:_id --sort=name"
 
 [tasks.check]
 help = "Run code checks"

pyproject.toml

@@ -67,6 +67,7 @@ raw-options = { fallback_version = "0.0.0" }
 [tool.pytest]
 addopts = [
   "--benchmark-disable",
+  "--benchmark-group-by=func,param:_id",
 ]
 # Turn all warnings into errors,
 # except DeprecationWarnings (which we knowingly tolerate due to using old `protobuf` APIs).

test/test_benchmark.py

@@ -113,23 +113,25 @@ def gen_complex(depth: int) -> BenchComplexSchema:
 validator = protovalidate.Validator()
 
 
+def param(*args, id: str) -> pytest.param:  # noqa: A002
+    """Copies pytest id to a fixture parameter since pytest-benchmark doesn't allow
+    grouping on the former."""
+    return pytest.param(id, *args, id=id)
+
+
 # Use lambda factories to allow random seed fixture to apply before computing
 cases = [
-    pytest.param(lambda: BenchScalar(x=42), id="scalar"),
-    pytest.param(lambda: BenchRepeatedScalar(x=[1, 2, 3, 4, 5, 6, 7, 8, 9, 10]), id="repeated_scalar"),
-    pytest.param(lambda: BenchRepeatedMessage(x=[BenchScalar(x=i + 1) for i in range(10)]), id="repeated_message"),
-    pytest.param(
-        lambda: BenchRepeatedScalarUnique(x=[1.1, 2.2, 3.3, 4.4, 5.5, 6.6, 7.7, 8.8]), id="repeated_unique_scalar"
-    ),
-    pytest.param(
-        lambda: BenchRepeatedBytesUnique(x=[gen_bytes(4, i + 1) for i in range(8)]), id="repeated_unique_bytes"
-    ),
-    pytest.param(
+    param(lambda: BenchScalar(x=42), id="scalar"),
+    param(lambda: BenchRepeatedScalar(x=[1, 2, 3, 4, 5, 6, 7, 8, 9, 10]), id="repeated_scalar"),
+    param(lambda: BenchRepeatedMessage(x=[BenchScalar(x=i + 1) for i in range(10)]), id="repeated_message"),
+    param(lambda: BenchRepeatedScalarUnique(x=[1.1, 2.2, 3.3, 4.4, 5.5, 6.6, 7.7, 8.8]), id="repeated_unique_scalar"),
+    param(lambda: BenchRepeatedBytesUnique(x=[gen_bytes(4, i + 1) for i in range(8)]), id="repeated_unique_bytes"),
+    param(
         lambda: BenchMap(entries={"k1": "v1", "k2": "v2", "k3": "v3", "k4": "v4", "k5": "v5", "k6": "v6", "k7": "v7"}),
         id="map",
     ),
-    pytest.param(lambda: gen_complex(1), id="complex_schema"),
-    pytest.param(
+    param(lambda: gen_complex(1), id="complex_schema"),
+    param(
         lambda: BenchGT(
             gt=50,
             gte=50,
@@ -150,7 +152,7 @@ def gen_complex(depth: int) -> BenchComplexSchema:
         ),
         id="int32_gt",
     ),
-    pytest.param(
+    param(
         lambda: ByteMatching(
             # 16-byte buffers; bytes.ip accepts 4 or 16 bytes (v4/v6 raw), bytes.ipv4
             # requires 4 bytes, bytes.ipv6 requires 16, bytes.uuid requires 16.
@@ -161,7 +163,7 @@ def gen_complex(depth: int) -> BenchComplexSchema:
         ),
         id="bytes_matching",
     ),
-    pytest.param(
+    param(
         lambda: StringMatching(
             hostname="example.com",
             host_and_port="example.com:8080",
@@ -170,7 +172,7 @@ def gen_complex(depth: int) -> BenchComplexSchema:
         ),
         id="string_matching",
     ),
-    pytest.param(
+    param(
         lambda: WrapperTesting(
             i32=Int32Value(value=11),
             d=DoubleValue(value=11),
@@ -184,12 +186,12 @@ def gen_complex(depth: int) -> BenchComplexSchema:
         ),
         id="wrapper_testing",
     ),
-    pytest.param(lambda: MultiRule(many=1), id="multi_rule_error"),
-    pytest.param(lambda: MultiRule(many=10), id="multi_rule_no_error"),
+    param(lambda: MultiRule(many=1), id="multi_rule_error"),
+    param(lambda: MultiRule(many=10), id="multi_rule_no_error"),
 ]
 
 
-@pytest.mark.parametrize("message_factory", cases)
-def test_benchmark(message_factory: Callable[[], Message], benchmark: BenchmarkFixture):
+@pytest.mark.parametrize(("_id", "message_factory"), cases)
+def test_benchmark(_id: str, message_factory: Callable[[], Message], benchmark: BenchmarkFixture):
     message = message_factory()
     benchmark(validator.collect_violations, message)