Allow comparison against a baseline

CHANGELOG.md

@@ -4,6 +4,12 @@ All notable changes to this project will be documented in this file. This projec
 
 [semver]: http://semver.org/spec/v2.0.0.html
 
+## [Unreleased](https://github.com/michaelherold/benchmark-memory/compare/v0.2.0...main)
+
+### Added
+
+- [#28](https://github.com/michaelherold/benchmark-memory/pull/28): Add the `order: :baseline` comparison method to compare results against a baseline report - [@michaelherold](https://github.com/michaelherold).
+
 ## [0.2.0](https://github.com/michaelherold/benchmark-memory/compare/v0.1.1...v0.2.0)
 
 ### Added

README.md

@@ -113,6 +113,21 @@ end
 
 When you're looking for a memory leak, this configuration can help you because it compares your reports by the amount of memory that the garbage collector does not collect after the benchmark.
 
+#### Ordering by a baseline
+
+When you're looking to see whether you can improve memory performance by refactoring some code, it can help to visually compare your reports against a baseline. To do so, place your baseline report first and enable the `order: :baseline` option like so:
+
+```ruby
+Benchmark.memory do |bench|
+  bench.report("my baseline") {}
+  bench.report("another test") {}
+
+  bench.compare! order: :baseline
+end
+```
+
+This will always show the baseline at the top of the comparison and order the alternatives in ascending order against the baseline.
+
 ### Hold results between invocations
 
 ```ruby

lib/benchmark/memory/job/io_output/comparison_formatter.rb

@@ -28,35 +28,38 @@ def to_s
             return '' unless comparison.possible?
 
             output = StringIO.new
-            best, *rest = comparison.entries
+            baseline, *rest = comparison.entries
             rest = Array(rest)
 
-            add_best_summary(best, output)
+            add_baseline_summary(baseline, output)
 
             rest.each do |entry|
-              add_comparison(entry, best, output)
+              add_comparison(entry, baseline, output)
             end
 
             output.string
           end
 
           private
 
-          def add_best_summary(best, output)
-            output << summary_message("%20s: %10i %s\n", best)
+          def add_baseline_summary(baseline, output)
+            output << summary_message('%20s: %10i %s', baseline)
+            output << "\n"
           end
 
-          def add_comparison(entry, best, output)
+          def add_comparison(entry, baseline, output)
             output << summary_message('%20s: %10i %s - ', entry)
-            output << comparison_between(entry, best)
+            output << comparison_between(entry, baseline)
             output << "\n"
           end
 
-          def comparison_between(entry, best)
-            ratio = entry.compared_metric(comparison).to_f / best.compared_metric(comparison)
+          def comparison_between(entry, baseline)
+            ratio = entry.compared_metric(comparison).to_f / baseline.compared_metric(comparison)
 
-            if ratio.abs > 1
-              format('%<ratio>.2fx more', ratio: ratio)
+            if ratio > 1
+              format('%.2fx more', ratio)
+            elsif ratio < 1
+              format('%.2fx less', 1.0 / ratio)
             else
               'same'
             end

lib/benchmark/memory/report/comparator.rb

@@ -16,30 +16,40 @@ class Comparator
         # @param spec [Hash<Symbol, Symbol>] The specification given for the {Comparator}
         # @return [Comparator]
         def self.from_spec(spec)
+          order =
+            case spec.delete(:order)
+            when :baseline then :baseline
+            else :lowest
+            end
+
           raise ArgumentError, 'Only send a single metric and value, in the form memory: :allocated' if spec.length > 1
 
           metric, value = *spec.first
           metric ||= :memory
           value  ||= :allocated
 
-          new(metric: metric, value: value)
+          new(metric: metric, order: order, value: value)
         end
 
         # Instantiate a new comparator
         #
         # @param metric [Symbol] (see #metric)
         # @param value [Symbol] (see #value)
-        def initialize(metric: :memory, value: :allocated)
+        def initialize(metric: :memory, order: :lowest, value: :allocated)
           raise ArgumentError, "Invalid metric: #{metric.inspect}" unless METRICS.include? metric
           raise ArgumentError, "Invalid value: #{value.inspect}" unless VALUES.include? value
 
           @metric = metric
+          @order = order
           @value = value
         end
 
         # @return [Symbol] The metric to compare, one of `:memory`, `:objects`, or `:strings`
         attr_reader :metric
 
+        # @return [Symbol] The order in which to report results, one of `:lowest` (default) or `:baseline`
+        attr_reader :order
+
         # @return [Symbol] The value to compare, one of `:allocated` or `:retained`
         attr_reader :value
 
@@ -52,6 +62,13 @@ def ==(other)
           metric == other.metric && value == other.value
         end
 
+        # Checks whether comparing by the baseline
+        #
+        # @return [Boolean]
+        def baseline?
+          order == :baseline
+        end
+
         # Converts the {Comparator} to a Proc for passing to a block
         #
         # @return [Proc]

lib/benchmark/memory/report/comparison.rb

@@ -12,7 +12,13 @@ class Comparison
         # @param entries [Array<Entry>] The entries to compare.
         # @param comparator [Comparator] The comparator to use when generating.
         def initialize(entries, comparator)
-          @entries = entries.sort_by(&comparator)
+          @entries =
+            if comparator.baseline?
+              baseline = entries.shift
+              [baseline, *entries.sort_by(&comparator)]
+            else
+              entries.sort_by(&comparator)
+            end
           @comparator = comparator
         end
 
@@ -22,11 +28,15 @@ def initialize(entries, comparator)
         # @return [Array<Entry>] The entries to compare.
         attr_reader :entries
 
+        # @!method baseline?
+        #   @return [Boolean] Whether the comparison will print in baseline order.
         # @!method metric
         #   @return [Symbol] The metric to compare, one of `:memory`, `:objects`, or `:strings`
+        # @!method order
+        #   @return [Symbol] The order to report results, one of `:lowest`, or `:baseline`
         # @!method value
         #   @return [Symbol] The value to compare, one of `:allocated` or `:retained`
-        def_delegators :@comparator, :metric, :value
+        def_delegators :@comparator, :baseline?, :order, :metric, :value
 
         # Check if the comparison is possible
         #

spec/benchmark/memory/job/io_output/comparison_formatter_spec.rb

@@ -35,6 +35,18 @@
 
       expect(formatter.to_s).to match(/same/)
     end
+
+    it 'does not output a comparison for the baseline' do
+      entries = [create_low_entry, create_high_entry]
+      comp = Benchmark::Memory::Report::Comparison.new(
+        entries,
+        Benchmark::Memory::Report::Comparator.new(order: :baseline)
+      )
+
+      formatter = described_class.new(comp)
+
+      expect(formatter.to_s).to match(/2500 allocated\n.*/).and(match(/10000 allocated - 4.00x more/))
+    end
   end
 
   def comparison(entries)

spec/benchmark/memory/report/comparison_spec.rb

@@ -13,6 +13,17 @@
 
       expect(comparison.entries).to eq([low_entry, high_entry])
     end
+
+    it 'sorts the baseline first when there is one' do
+      high_entry = create_high_entry
+      mid_entry = create_mid_entry
+      low_entry = create_low_entry
+      comparator = Benchmark::Memory::Report::Comparator.from_spec({ order: :baseline })
+
+      comparison = described_class.new([high_entry, mid_entry, low_entry], comparator)
+
+      expect(comparison.entries).to eq([high_entry, low_entry, mid_entry])
+    end
   end
 
   def create_high_entry
@@ -23,6 +34,13 @@ def create_high_entry
   end
   alias_method :create_entry, :create_high_entry
 
+  def create_mid_entry
+    Benchmark::Memory::Report::Entry.new(
+      'mid',
+      create_measurement(5_000, 2_500)
+    )
+  end
+
   def create_low_entry
     Benchmark::Memory::Report::Entry.new(
       'low',