From b92b23f293564d4e3dff1f2820a9923bcccbbc43 Mon Sep 17 00:00:00 2001 From: Michael Herold Date: Tue, 29 Jul 2025 21:24:24 -0500 Subject: [PATCH] Allow comparison against a baseline This change introduces a new comparison ordering strategy where the initial report serves as the baseline against which the job compares the other reports. This is useful in cases where you're testing out different implementations and want to see whether they are better or worse (and the comparative increase or decrease) in an easy-to-decipher order. --- CHANGELOG.md | 6 +++++ README.md | 15 +++++++++++ .../job/io_output/comparison_formatter.rb | 25 +++++++++++-------- lib/benchmark/memory/report/comparator.rb | 21 ++++++++++++++-- lib/benchmark/memory/report/comparison.rb | 14 +++++++++-- .../io_output/comparison_formatter_spec.rb | 12 +++++++++ .../memory/report/comparison_spec.rb | 18 +++++++++++++ 7 files changed, 96 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9b2f4d2..14fb974 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,12 @@ All notable changes to this project will be documented in this file. This projec [semver]: http://semver.org/spec/v2.0.0.html +## [Unreleased](https://github.com/michaelherold/benchmark-memory/compare/v0.2.0...main) + +### Added + +- [#28](https://github.com/michaelherold/benchmark-memory/pull/28): Add the `order: :baseline` comparison method to compare results against a baseline report - [@michaelherold](https://github.com/michaelherold). + ## [0.2.0](https://github.com/michaelherold/benchmark-memory/compare/v0.1.1...v0.2.0) ### Added diff --git a/README.md b/README.md index 536b926..0bcaf52 100644 --- a/README.md +++ b/README.md @@ -113,6 +113,21 @@ end When you're looking for a memory leak, this configuration can help you because it compares your reports by the amount of memory that the garbage collector does not collect after the benchmark. +#### Ordering by a baseline + +When you're looking to see whether you can improve memory performance by refactoring some code, it can help to visually compare your reports against a baseline. To do so, place your baseline report first and enable the `order: :baseline` option like so: + +```ruby +Benchmark.memory do |bench| + bench.report("my baseline") {} + bench.report("another test") {} + + bench.compare! order: :baseline +end +``` + +This will always show the baseline at the top of the comparison and order the alternatives in ascending order against the baseline. + ### Hold results between invocations ```ruby diff --git a/lib/benchmark/memory/job/io_output/comparison_formatter.rb b/lib/benchmark/memory/job/io_output/comparison_formatter.rb index 9154676..a2862fb 100644 --- a/lib/benchmark/memory/job/io_output/comparison_formatter.rb +++ b/lib/benchmark/memory/job/io_output/comparison_formatter.rb @@ -28,13 +28,13 @@ def to_s return '' unless comparison.possible? output = StringIO.new - best, *rest = comparison.entries + baseline, *rest = comparison.entries rest = Array(rest) - add_best_summary(best, output) + add_baseline_summary(baseline, output) rest.each do |entry| - add_comparison(entry, best, output) + add_comparison(entry, baseline, output) end output.string @@ -42,21 +42,24 @@ def to_s private - def add_best_summary(best, output) - output << summary_message("%20s: %10i %s\n", best) + def add_baseline_summary(baseline, output) + output << summary_message('%20s: %10i %s', baseline) + output << "\n" end - def add_comparison(entry, best, output) + def add_comparison(entry, baseline, output) output << summary_message('%20s: %10i %s - ', entry) - output << comparison_between(entry, best) + output << comparison_between(entry, baseline) output << "\n" end - def comparison_between(entry, best) - ratio = entry.compared_metric(comparison).to_f / best.compared_metric(comparison) + def comparison_between(entry, baseline) + ratio = entry.compared_metric(comparison).to_f / baseline.compared_metric(comparison) - if ratio.abs > 1 - format('%.2fx more', ratio: ratio) + if ratio > 1 + format('%.2fx more', ratio) + elsif ratio < 1 + format('%.2fx less', 1.0 / ratio) else 'same' end diff --git a/lib/benchmark/memory/report/comparator.rb b/lib/benchmark/memory/report/comparator.rb index 619a742..7ef482b 100644 --- a/lib/benchmark/memory/report/comparator.rb +++ b/lib/benchmark/memory/report/comparator.rb @@ -16,30 +16,40 @@ class Comparator # @param spec [Hash] The specification given for the {Comparator} # @return [Comparator] def self.from_spec(spec) + order = + case spec.delete(:order) + when :baseline then :baseline + else :lowest + end + raise ArgumentError, 'Only send a single metric and value, in the form memory: :allocated' if spec.length > 1 metric, value = *spec.first metric ||= :memory value ||= :allocated - new(metric: metric, value: value) + new(metric: metric, order: order, value: value) end # Instantiate a new comparator # # @param metric [Symbol] (see #metric) # @param value [Symbol] (see #value) - def initialize(metric: :memory, value: :allocated) + def initialize(metric: :memory, order: :lowest, value: :allocated) raise ArgumentError, "Invalid metric: #{metric.inspect}" unless METRICS.include? metric raise ArgumentError, "Invalid value: #{value.inspect}" unless VALUES.include? value @metric = metric + @order = order @value = value end # @return [Symbol] The metric to compare, one of `:memory`, `:objects`, or `:strings` attr_reader :metric + # @return [Symbol] The order in which to report results, one of `:lowest` (default) or `:baseline` + attr_reader :order + # @return [Symbol] The value to compare, one of `:allocated` or `:retained` attr_reader :value @@ -52,6 +62,13 @@ def ==(other) metric == other.metric && value == other.value end + # Checks whether comparing by the baseline + # + # @return [Boolean] + def baseline? + order == :baseline + end + # Converts the {Comparator} to a Proc for passing to a block # # @return [Proc] diff --git a/lib/benchmark/memory/report/comparison.rb b/lib/benchmark/memory/report/comparison.rb index 1429459..7c94c77 100644 --- a/lib/benchmark/memory/report/comparison.rb +++ b/lib/benchmark/memory/report/comparison.rb @@ -12,7 +12,13 @@ class Comparison # @param entries [Array] The entries to compare. # @param comparator [Comparator] The comparator to use when generating. def initialize(entries, comparator) - @entries = entries.sort_by(&comparator) + @entries = + if comparator.baseline? + baseline = entries.shift + [baseline, *entries.sort_by(&comparator)] + else + entries.sort_by(&comparator) + end @comparator = comparator end @@ -22,11 +28,15 @@ def initialize(entries, comparator) # @return [Array] The entries to compare. attr_reader :entries + # @!method baseline? + # @return [Boolean] Whether the comparison will print in baseline order. # @!method metric # @return [Symbol] The metric to compare, one of `:memory`, `:objects`, or `:strings` + # @!method order + # @return [Symbol] The order to report results, one of `:lowest`, or `:baseline` # @!method value # @return [Symbol] The value to compare, one of `:allocated` or `:retained` - def_delegators :@comparator, :metric, :value + def_delegators :@comparator, :baseline?, :order, :metric, :value # Check if the comparison is possible # diff --git a/spec/benchmark/memory/job/io_output/comparison_formatter_spec.rb b/spec/benchmark/memory/job/io_output/comparison_formatter_spec.rb index aa6c27b..7e8bb91 100644 --- a/spec/benchmark/memory/job/io_output/comparison_formatter_spec.rb +++ b/spec/benchmark/memory/job/io_output/comparison_formatter_spec.rb @@ -35,6 +35,18 @@ expect(formatter.to_s).to match(/same/) end + + it 'does not output a comparison for the baseline' do + entries = [create_low_entry, create_high_entry] + comp = Benchmark::Memory::Report::Comparison.new( + entries, + Benchmark::Memory::Report::Comparator.new(order: :baseline) + ) + + formatter = described_class.new(comp) + + expect(formatter.to_s).to match(/2500 allocated\n.*/).and(match(/10000 allocated - 4.00x more/)) + end end def comparison(entries) diff --git a/spec/benchmark/memory/report/comparison_spec.rb b/spec/benchmark/memory/report/comparison_spec.rb index dee3a68..27f9873 100644 --- a/spec/benchmark/memory/report/comparison_spec.rb +++ b/spec/benchmark/memory/report/comparison_spec.rb @@ -13,6 +13,17 @@ expect(comparison.entries).to eq([low_entry, high_entry]) end + + it 'sorts the baseline first when there is one' do + high_entry = create_high_entry + mid_entry = create_mid_entry + low_entry = create_low_entry + comparator = Benchmark::Memory::Report::Comparator.from_spec({ order: :baseline }) + + comparison = described_class.new([high_entry, mid_entry, low_entry], comparator) + + expect(comparison.entries).to eq([high_entry, low_entry, mid_entry]) + end end def create_high_entry @@ -23,6 +34,13 @@ def create_high_entry end alias_method :create_entry, :create_high_entry + def create_mid_entry + Benchmark::Memory::Report::Entry.new( + 'mid', + create_measurement(5_000, 2_500) + ) + end + def create_low_entry Benchmark::Memory::Report::Entry.new( 'low',