Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions Gemfile
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ gemspec
group :development, :test do
gem "appraisal", "~> 2.5"
gem "minitest", "~> 6.0"
gem "minitest-mock", "~> 5.27"
gem "rake", "~> 13.0"
gem "rubocop", "~> 1.86"
gem "rubocop-minitest", "~> 0.39"
Expand Down
2 changes: 2 additions & 0 deletions Gemfile.lock
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ GEM
minitest (6.0.5)
drb (~> 2.0)
prism (~> 1.5)
minitest-mock (5.27.0)
multipart-post (2.4.1)
net-http (0.9.1)
uri (>= 0.11.1)
Expand Down Expand Up @@ -96,6 +97,7 @@ PLATFORMS
DEPENDENCIES
appraisal (~> 2.5)
minitest (~> 6.0)
minitest-mock (~> 5.27)
rake (~> 13.0)
rubocop (~> 1.86)
rubocop-minitest (~> 0.39)
Expand Down
1 change: 1 addition & 0 deletions gemfiles/ruby_llm_1.10.gemfile
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ gem "ruby_llm", "~> 1.10.0"
group :development, :test do
gem "appraisal", "~> 2.5"
gem "minitest", "~> 6.0"
gem "minitest-mock", "~> 5.27"
gem "rake", "~> 13.0"
gem "rubocop", "~> 1.86"
gem "rubocop-minitest", "~> 0.39"
Expand Down
1 change: 1 addition & 0 deletions gemfiles/ruby_llm_1.13.gemfile
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ gem "ruby_llm", "~> 1.13.0"
group :development, :test do
gem "appraisal", "~> 2.5"
gem "minitest", "~> 6.0"
gem "minitest-mock", "~> 5.27"
gem "rake", "~> 13.0"
gem "rubocop", "~> 1.86"
gem "rubocop-minitest", "~> 0.39"
Expand Down
1 change: 1 addition & 0 deletions gemfiles/ruby_llm_1.14.gemfile
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ gem "ruby_llm", "~> 1.14.0"
group :development, :test do
gem "appraisal", "~> 2.5"
gem "minitest", "~> 6.0"
gem "minitest-mock", "~> 5.27"
gem "rake", "~> 13.0"
gem "rubocop", "~> 1.86"
gem "rubocop-minitest", "~> 0.39"
Expand Down
1 change: 1 addition & 0 deletions gemfiles/ruby_llm_1.5.gemfile
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ gem "ruby_llm", "~> 1.5.0"
group :development, :test do
gem "appraisal", "~> 2.5"
gem "minitest", "~> 6.0"
gem "minitest-mock", "~> 5.27"
gem "rake", "~> 13.0"
gem "rubocop", "~> 1.86"
gem "rubocop-minitest", "~> 0.39"
Expand Down
1 change: 1 addition & 0 deletions gemfiles/ruby_llm_1.6.gemfile
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ gem "ruby_llm", "~> 1.6.0"
group :development, :test do
gem "appraisal", "~> 2.5"
gem "minitest", "~> 6.0"
gem "minitest-mock", "~> 5.27"
gem "rake", "~> 13.0"
gem "rubocop", "~> 1.86"
gem "rubocop-minitest", "~> 0.39"
Expand Down
1 change: 1 addition & 0 deletions gemfiles/ruby_llm_current.gemfile
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ gem "ruby_llm"
group :development, :test do
gem "appraisal", "~> 2.5"
gem "minitest", "~> 6.0"
gem "minitest-mock", "~> 5.27"
gem "rake", "~> 13.0"
gem "rubocop", "~> 1.86"
gem "rubocop-minitest", "~> 0.39"
Expand Down
112 changes: 92 additions & 20 deletions lib/ruby_llm/test.rb
Original file line number Diff line number Diff line change
Expand Up @@ -7,43 +7,115 @@
loader.setup

module RubyLLM
# The Test module provides a simple way to stub responses from an LLM for testing purposes. You can use it to set up
# predetermined responses that your tests can rely on, allowing you to test your code's behavior without making
# actual calls to an LLM.
# The Test module provides a simple way to stub responses from an LLM for testing purposes.
# You can use it to set up predetermined responses that your tests can rely on, allowing
# you to test your code's behavior without making actual calls to an LLM.
module Test
class << self
# Reset the test harness state.
#
# This clears all queued stubbed responses and all recorded requests.
#
# Use this at the start of a test, or whenever you want to ensure no state
# carries over from a previous example.
def reset
@responses = nil
harness.reset
end

# Pass in a RubyLLM::Message to have full control; strings and hashes will be wrapped in a message
# Queue a single stubbed response.
#
# Parameters:
#
# - `body`: The response to queue. Pass a `RubyLLM::Message` to control the
# message directly. Strings and hashes are wrapped in a message
# automatically.
#
# This is useful when your test only needs one response from the model.
#
# Example:
#
# RubyLLM::Test.stub_response("Hello from the test harness!")
#
# chat = RubyLLM.chat
# response = chat.ask("Say hello")
# response.content
# # => "Hello from the test harness!"
def stub_response(body)
responses << body
harness.stub_response(body)
end

# Queue multiple stubbed responses.
#
# Parameters:
#
# - `*bodies`: One or more responses to queue.
#
# Responses are returned in the same order they are provided, making this
# useful for tests that perform multiple LLM calls in sequence.
#
# Example:
#
# RubyLLM::Test.stub_responses("First reply", "Second reply")
#
# chat = RubyLLM.chat
# first_response = chat.ask("First question")
# second_response = chat.ask("Second question")
#
# first_response.content
# # => "First reply"
# second_response.content
# # => "Second reply"
def stub_responses(*bodies)
responses.concat(bodies)
harness.stub_responses(*bodies)
end

def with_responses(*bodies)
previous_responses = responses.dup
@responses = []
stub_responses(*bodies)
yield
ensure
@responses = previous_responses
# Run a block with a temporary set of stubbed responses.
#
# Parameters:
#
# - `*bodies`: The responses to make available inside the block.
# - `&block`: The code to run while those responses are active.
#
# The provided responses are available only for the duration of the block.
# This is useful when you want to scope stubbed responses to a single part
# of a test without affecting later assertions.
#
# Example:
#
# RubyLLM::Test.with_responses("Scoped reply") do
# chat = RubyLLM.chat
# chat.ask("Question")
# end
def with_responses(*bodies, &)
harness.with_responses(*bodies, &)
end

def next_response
responses.shift
# Return all recorded requests.
#
# This is useful for assertions about prompts, parameters, or other request
# details captured by the harness.
def requests
harness.requests
end

def responses_empty?
responses.empty?
# Return the most recent request.
#
# Use this when you only care about the latest request made during a test.
def last_request
harness.last_request
end

def responses
@responses ||= []
# Clear all recorded requests.
#
# This leaves queued responses unchanged.
def clear_requests
harness.clear_requests
end

private

def harness
@harness ||= Harness.new
end
end
end
Expand Down
65 changes: 65 additions & 0 deletions lib/ruby_llm/test/harness.rb
Original file line number Diff line number Diff line change
@@ -0,0 +1,65 @@
# frozen_string_literal: true

module RubyLLM
module Test
# This class serves as a test harness for stubbing responses and recording requests sent to the provider.
# It allows tests to set up expected responses and then verify that the provider received the correct
# parameters.
class Harness
attr_reader :requests

def initialize
reset
end

# Pass in a RubyLLM::Message to have full control; strings and hashes will be wrapped in a message
def stub_response(body)
responses << body
end

def stub_responses(*bodies)
responses.concat(bodies)
end

def with_responses(*bodies)
previous_responses = responses.dup
@responses = []
stub_responses(*bodies)
yield
ensure
@responses = previous_responses
end

def next_response
responses.shift
end

def responses_empty?
responses.empty?
end

def record_request(request)
requests << request
end

def last_request
requests.last
end

def clear_requests
requests.clear
end

def reset
@responses = []
@requests = []
end

private

def responses
@responses ||= []
end
end
end
end
2 changes: 1 addition & 1 deletion lib/ruby_llm/test/resolve_with_test_provider.rb
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ module Test
module ResolveWithTestProvider
def resolve(...)
model, provider_instance = super
[ model, Test::TestProvider.new(provider_instance, RubyLLM::Test) ]
[ model, Test::TestProvider.new(provider_instance, RubyLLM::Test.send(:harness)) ]
end
end
end
Expand Down
19 changes: 4 additions & 15 deletions lib/ruby_llm/test/test_provider.rb
Original file line number Diff line number Diff line change
Expand Up @@ -6,22 +6,15 @@ module Test
# the `complete` method, allowing tests to assert that the correct parameters were passed and to simulate
# responses from the provider.
class TestProvider < SimpleDelegator
extend Forwardable

attr_reader :complete_calls

def_delegators :last_call, :messages, :block_received?

def initialize(provider, test_harness = RubyLLM::Test)
def initialize(provider, test_harness)
super(provider)
@test_harness = test_harness
@complete_calls = []
end

def complete(...)
call = CompleteParameters.capture_from(__getobj__, ...)
@complete_calls << call
raise Errors::NoResponseProvidedError, call.messages if @test_harness.responses_empty?
parameters = CompleteParameters.capture_from(__getobj__, ...)
@test_harness.record_request(parameters)
raise Errors::NoResponseProvidedError, parameters.messages if @test_harness.responses_empty?

response = @test_harness.next_response
return response if response.is_a?(Message)
Expand All @@ -31,10 +24,6 @@ def complete(...)
content: response.is_a?(Hash) ? response.to_json : response
)
end

def last_call
@complete_calls.last
end
end
end
end
Loading
Loading