diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index f65159d..6d63153 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -17,7 +17,16 @@ jobs: - "3.3" - "3.4" - "4.0" - + ruby_llm: + - "1.5" + - "1.6" + - "1.10" + - "1.13" + - "1.14" + - "current" + env: + BUNDLE_GEMFILE: ${{ github.workspace }}/gemfiles/ruby_llm_${{ matrix.ruby_llm }}.gemfile + BUNDLE_PATH_RELATIVE_TO_CWD: true steps: - uses: actions/checkout@v6 - name: Set up Ruby diff --git a/.gitignore b/.gitignore index 9106b2a..998e5f2 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ /_yardoc/ /coverage/ /doc/ +/gemfiles/*.lock /pkg/ /spec/reports/ /tmp/ diff --git a/.rubocop.yml b/.rubocop.yml index 39be2af..898d9dc 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -6,6 +6,7 @@ plugins: AllCops: TargetRubyVersion: 3.3 Exclude: + - "gemfiles/**/*" - "ruby_llm-test.gemspec" - "Rakefile" - "vendor/**/*" @@ -15,6 +16,13 @@ AllCops: Layout/SpaceInsideArrayLiteralBrackets: EnforcedStyle: space +Metrics/ClassLength: + Exclude: + - "test/**/*" +Metrics/MethodLength: + Exclude: + - "test/**/*" + Style/StringLiterals: EnforcedStyle: double_quotes Style/StringLiteralsInInterpolation: diff --git a/Appraisals b/Appraisals new file mode 100644 index 0000000..6fc69a7 --- /dev/null +++ b/Appraisals @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +appraise "ruby_llm-1.5" do + gem "ruby_llm", "~> 1.5.0" +end + +appraise "ruby_llm-1.6" do + gem "ruby_llm", "~> 1.6.0" +end + +appraise "ruby_llm-1.10" do + gem "ruby_llm", "~> 1.10.0" +end + +appraise "ruby_llm-1.13" do + gem "ruby_llm", "~> 1.13.0" +end + +appraise "ruby_llm-1.14" do + gem "ruby_llm", "~> 1.14.0" +end + +appraise "ruby_llm-current" do + gem "ruby_llm" +end diff --git a/Gemfile b/Gemfile index 2cb825a..e3b089f 100644 --- a/Gemfile +++ b/Gemfile @@ -8,6 +8,7 @@ git_source(:github) { |repo_name| "https://github.com/#{repo_name}" } gemspec group :development, :test do + gem "appraisal", "~> 2.5" gem "minitest", "~> 6.0" gem "rake", "~> 13.0" gem "rubocop", "~> 1.86" diff --git a/Gemfile.lock b/Gemfile.lock index 748a59d..c7530fe 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -2,11 +2,15 @@ PATH remote: . specs: ruby_llm-test (0.1.0) - ruby_llm (>= 1.14.0) + ruby_llm (>= 1.5.0) GEM remote: https://gem.coop/ specs: + appraisal (2.5.0) + bundler + rake + thor (>= 0.14.0) ast (2.4.3) base64 (0.3.0) docile (1.4.1) @@ -78,6 +82,7 @@ GEM simplecov_json_formatter (~> 0.1) simplecov-html (0.13.2) simplecov_json_formatter (0.1.4) + thor (1.5.0) unicode-display_width (3.2.0) unicode-emoji (~> 4.1) unicode-emoji (4.2.0) @@ -89,6 +94,7 @@ PLATFORMS ruby DEPENDENCIES + appraisal (~> 2.5) minitest (~> 6.0) rake (~> 13.0) rubocop (~> 1.86) @@ -97,4 +103,4 @@ DEPENDENCIES simplecov (~> 0.22) BUNDLED WITH - 4.0.10 + 4.0.10 diff --git a/gemfiles/ruby_llm_1.10.gemfile b/gemfiles/ruby_llm_1.10.gemfile new file mode 100644 index 0000000..4fec377 --- /dev/null +++ b/gemfiles/ruby_llm_1.10.gemfile @@ -0,0 +1,16 @@ +# This file was generated by Appraisal + +source "https://gem.coop" + +gem "ruby_llm", "~> 1.10.0" + +group :development, :test do + gem "appraisal", "~> 2.5" + gem "minitest", "~> 6.0" + gem "rake", "~> 13.0" + gem "rubocop", "~> 1.86" + gem "rubocop-minitest", "~> 0.39" + gem "simplecov", "~> 0.22" +end + +gemspec path: "../" diff --git a/gemfiles/ruby_llm_1.13.gemfile b/gemfiles/ruby_llm_1.13.gemfile new file mode 100644 index 0000000..0623e37 --- /dev/null +++ b/gemfiles/ruby_llm_1.13.gemfile @@ -0,0 +1,16 @@ +# This file was generated by Appraisal + +source "https://gem.coop" + +gem "ruby_llm", "~> 1.13.0" + +group :development, :test do + gem "appraisal", "~> 2.5" + gem "minitest", "~> 6.0" + gem "rake", "~> 13.0" + gem "rubocop", "~> 1.86" + gem "rubocop-minitest", "~> 0.39" + gem "simplecov", "~> 0.22" +end + +gemspec path: "../" diff --git a/gemfiles/ruby_llm_1.14.gemfile b/gemfiles/ruby_llm_1.14.gemfile new file mode 100644 index 0000000..b66c6a9 --- /dev/null +++ b/gemfiles/ruby_llm_1.14.gemfile @@ -0,0 +1,16 @@ +# This file was generated by Appraisal + +source "https://gem.coop" + +gem "ruby_llm", "~> 1.14.0" + +group :development, :test do + gem "appraisal", "~> 2.5" + gem "minitest", "~> 6.0" + gem "rake", "~> 13.0" + gem "rubocop", "~> 1.86" + gem "rubocop-minitest", "~> 0.39" + gem "simplecov", "~> 0.22" +end + +gemspec path: "../" diff --git a/gemfiles/ruby_llm_1.5.gemfile b/gemfiles/ruby_llm_1.5.gemfile new file mode 100644 index 0000000..56aaa79 --- /dev/null +++ b/gemfiles/ruby_llm_1.5.gemfile @@ -0,0 +1,16 @@ +# This file was generated by Appraisal + +source "https://gem.coop" + +gem "ruby_llm", "~> 1.5.0" + +group :development, :test do + gem "appraisal", "~> 2.5" + gem "minitest", "~> 6.0" + gem "rake", "~> 13.0" + gem "rubocop", "~> 1.86" + gem "rubocop-minitest", "~> 0.39" + gem "simplecov", "~> 0.22" +end + +gemspec path: "../" diff --git a/gemfiles/ruby_llm_1.6.gemfile b/gemfiles/ruby_llm_1.6.gemfile new file mode 100644 index 0000000..4056125 --- /dev/null +++ b/gemfiles/ruby_llm_1.6.gemfile @@ -0,0 +1,16 @@ +# This file was generated by Appraisal + +source "https://gem.coop" + +gem "ruby_llm", "~> 1.6.0" + +group :development, :test do + gem "appraisal", "~> 2.5" + gem "minitest", "~> 6.0" + gem "rake", "~> 13.0" + gem "rubocop", "~> 1.86" + gem "rubocop-minitest", "~> 0.39" + gem "simplecov", "~> 0.22" +end + +gemspec path: "../" diff --git a/gemfiles/ruby_llm_current.gemfile b/gemfiles/ruby_llm_current.gemfile new file mode 100644 index 0000000..8c11c66 --- /dev/null +++ b/gemfiles/ruby_llm_current.gemfile @@ -0,0 +1,16 @@ +# This file was generated by Appraisal + +source "https://gem.coop" + +gem "ruby_llm" + +group :development, :test do + gem "appraisal", "~> 2.5" + gem "minitest", "~> 6.0" + gem "rake", "~> 13.0" + gem "rubocop", "~> 1.86" + gem "rubocop-minitest", "~> 0.39" + gem "simplecov", "~> 0.22" +end + +gemspec path: "../" diff --git a/lib/ruby_llm/test/complete_parameters.rb b/lib/ruby_llm/test/complete_parameters.rb index 9753cb4..9565920 100644 --- a/lib/ruby_llm/test/complete_parameters.rb +++ b/lib/ruby_llm/test/complete_parameters.rb @@ -2,27 +2,65 @@ module RubyLLM module Test - # This class encapsulates all the parameters that are passed to the `complete` method of the `Test` class. It - # serves as a structured way to manage and access these parameters throughout the testing process. + # This class encapsulates all the parameters that are passed to the `complete` method of the wrapped provider. It + # stores the raw arguments and uses the wrapped provider's actual method signature to expose named accessors for + # inspection in tests. class CompleteParameters - attr_reader :messages, :tools, :temperature, :model, :params, :headers, :schema, :thinking, :tool_prefs, :block - - def initialize(messages:, tools:, temperature:, model:, params:, # rubocop:disable Metrics/ParameterLists - headers:, schema:, thinking:, tool_prefs:, block:) - @messages = messages - @tools = tools - @temperature = temperature - @model = model - @params = params - @headers = headers - @schema = schema - @thinking = thinking - @tool_prefs = tool_prefs + attr_reader :args, :kwargs, :block, :parameter_definitions + + def self.capture_from(provider, *args, **kwargs, &block) + parameters = provider.method(:complete).parameters + new(args:, kwargs:, block:, parameter_definitions: parameters) + end + + def initialize(args:, kwargs:, block:, parameter_definitions:) + @args = args + @kwargs = kwargs @block = block + @parameter_definitions = parameter_definitions end def block_received? - !@block.nil? + !block.nil? + end + + def [](name) + name = name.to_sym + return block if name == :block + + positional_name_to_value[name] || kwargs[name] + end + + def key?(name) + name = name.to_sym + name == :block || positional_name_to_value.key?(name) || kwargs.key?(name) + end + + def to_h + positional_name_to_value.merge(kwargs).merge(block: block) + end + + def method_missing(name, *call_args) + return super unless call_args.empty? + return self[name] if key?(name) + + super + end + + def respond_to_missing?(name, include_private = false) + key?(name) || super + end + + private + + def positional_name_to_value + @positional_name_to_value ||= begin + positional_names = parameter_definitions + .select { |parameter| %i[req opt].include?(parameter.first) } + .map(&:last) + + positional_names.each_with_index.to_h { |param_name, index| [ param_name, args[index] ] } + end end end end diff --git a/lib/ruby_llm/test/test_provider.rb b/lib/ruby_llm/test/test_provider.rb index f5a090e..5e268a0 100644 --- a/lib/ruby_llm/test/test_provider.rb +++ b/lib/ruby_llm/test/test_provider.rb @@ -10,8 +10,7 @@ class TestProvider < SimpleDelegator attr_reader :complete_calls - def_delegators :last_call, :messages, :tools, :temperature, :model, :params, :headers, :schema, :thinking, - :tool_prefs, :block_received? + def_delegators :last_call, :messages, :block_received? def initialize(provider, test_harness = RubyLLM::Test) super(provider) @@ -19,11 +18,10 @@ def initialize(provider, test_harness = RubyLLM::Test) @complete_calls = [] end - def complete(messages, tools:, temperature:, model:, params: {}, # rubocop:disable Metrics/ParameterLists - headers: {}, schema: nil, thinking: nil, tool_prefs: nil, &block) - @complete_calls << CompleteParameters.new(messages:, tools:, temperature:, model:, params:, headers:, schema:, - thinking:, tool_prefs:, block:) - raise Errors::NoResponseProvidedError, messages if @test_harness.responses_empty? + def complete(...) + call = CompleteParameters.capture_from(__getobj__, ...) + @complete_calls << call + raise Errors::NoResponseProvidedError, call.messages if @test_harness.responses_empty? response = @test_harness.next_response return response if response.is_a?(Message) diff --git a/ruby_llm-test.gemspec b/ruby_llm-test.gemspec index 342df24..b025c20 100644 --- a/ruby_llm-test.gemspec +++ b/ruby_llm-test.gemspec @@ -19,5 +19,5 @@ Gem::Specification.new do |spec| spec.files = Dir["lib/**/*", "LICENSE.txt", "README.md"] - spec.add_dependency "ruby_llm", ">= 1.14.0" + spec.add_dependency "ruby_llm", ">= 1.5.0" end diff --git a/test/ruby_llm/test/complete_parameters_test.rb b/test/ruby_llm/test/complete_parameters_test.rb index 3695368..fe85908 100644 --- a/test/ruby_llm/test/complete_parameters_test.rb +++ b/test/ruby_llm/test/complete_parameters_test.rb @@ -5,25 +5,131 @@ module RubyLLM module Test class CompleteParametersTest < Minitest::Test + class ProviderDouble + def complete(messages, tools:, temperature:, model:, params: {}, # rubocop:disable Metrics/ParameterLists + headers: {}, schema: nil, thinking: nil, tool_prefs: nil, &) + end + end + def test_with_block - params_with_block = CompleteParameters.new( - messages: [], tools: [], temperature: 0.5, model: "test-model", params: {}, headers: {}, schema: nil, - thinking: nil, tool_prefs: nil, - block: -> { "test block" } + params_with_block = CompleteParameters.capture_from( + ProviderDouble.new, + [], + tools: [], + temperature: 0.5, + model: "test-model", + params: {}, + headers: {}, + schema: nil, + thinking: nil, + tool_prefs: nil, + &-> { "test block" } ) assert_predicate params_with_block, :block_received? end def test_without_block - params_without_block = CompleteParameters.new( - messages: [], tools: [], temperature: 0.5, model: "test-model", params: {}, headers: {}, schema: nil, - thinking: nil, tool_prefs: nil, - block: nil + params_without_block = CompleteParameters.capture_from( + ProviderDouble.new, + [], + tools: [], + temperature: 0.5, + model: "test-model", + params: {}, + headers: {}, + schema: nil, + thinking: nil, + tool_prefs: nil ) refute_predicate params_without_block, :block_received? end + + def test_exposes_captured_arguments_by_name # rubocop:disable Metrics/AbcSize, Minitest/MultipleAssertions + params = CompleteParameters.capture_from( + ProviderDouble.new, + [ Message.new(role: :user, content: "Hello") ], + tools: %w[tool1 tool2], + temperature: 0.7, + model: "test-model", + params: { param1: "value1" }, + headers: { "Authorization" => "Bearer token" }, + schema: { type: "object" }, + thinking: "thinking process", + tool_prefs: { prefer_tool1: true } + ) + + assert_equal "Hello", params.messages.first.content.to_s + assert_equal :user, params.messages.first.role + assert_equal %w[tool1 tool2], params.tools + assert_in_delta(0.7, params.temperature) + assert_equal "test-model", params.model + assert_equal({ param1: "value1" }, params.params) + assert_equal({ "Authorization" => "Bearer token" }, params.headers) + assert_equal({ type: "object" }, params.schema) + assert_equal "thinking process", params.thinking + assert_equal({ prefer_tool1: true }, params.tool_prefs) + end + + def test_supports_hash_style_access # rubocop:disable Minitest/MultipleAssertions + params = CompleteParameters.capture_from( + ProviderDouble.new, + [], + tools: [], + temperature: 0.5, + model: "test-model" + ) + + assert_equal [], params[:messages] + assert_equal [], params[:tools] + assert_in_delta(0.5, params[:temperature]) + assert_equal "test-model", params[:model] + end + + def test_to_h + hash = { + messages: [ Message.new(role: :user, content: "Hello") ], + tools: %w[tool1 tool2], + temperature: 0.7, + model: "test-model", + params: { param1: "value1" }, + headers: { "Authorization" => "Bearer token" }, + schema: { type: "object" }, + thinking: "thinking process", + tool_prefs: { prefer_tool1: true }, + block: nil + } + params = CompleteParameters.capture_from(ProviderDouble.new, **hash) + + assert_equal hash, params.to_h + end + + def test_invalid_parameter_access + params = CompleteParameters.capture_from(ProviderDouble.new, [], tools: [], temperature: 0.5, + model: "test-model") + + assert_raises(NoMethodError) { params.invalid_param } + end + + def test_block_access_via_hash_style + block = -> { "test block" } + params = CompleteParameters.capture_from(ProviderDouble.new, [], tools: [], temperature: 0.5, + model: "test-model", &block) + + assert_equal block, params[:block] + end + + def test_respond_to_missing # rubocop:disable Minitest/MultipleAssertions + params = CompleteParameters.capture_from(ProviderDouble.new, [], tools: [], temperature: 0.5, + model: "test-model") + + assert_respond_to params, :messages + assert_respond_to params, :tools + assert_respond_to params, :temperature + assert_respond_to params, :model + refute_respond_to params, :invalid_param + end end end end diff --git a/test/ruby_llm/test/test_provider_test.rb b/test/ruby_llm/test/test_provider_test.rb index 208ef7b..c969f69 100644 --- a/test/ruby_llm/test/test_provider_test.rb +++ b/test/ruby_llm/test/test_provider_test.rb @@ -5,6 +5,12 @@ module RubyLLM module Test class TestProviderTest < Minitest::Test + class ProviderDouble + def complete(messages, tools:, temperature:, model:, params: {}, # rubocop:disable Metrics/ParameterLists + headers: {}, schema: nil, thinking: nil, tool_prefs: nil, &) + end + end + def setup RubyLLM::Test.reset end @@ -12,7 +18,7 @@ def setup def test_provider_returns_stubbed_response RubyLLM::Test.stub_response("stubbed response") - provider = TestProvider.new(nil, RubyLLM::Test) + provider = TestProvider.new(ProviderDouble.new, RubyLLM::Test) response = provider.complete([], tools: [], temperature: 0.5, model: "test-model", params: {}, headers: {}) assert_kind_of Message, response @@ -20,7 +26,7 @@ def test_provider_returns_stubbed_response end def test_when_response_not_stubbed - provider = TestProvider.new(nil, RubyLLM::Test) + provider = TestProvider.new(ProviderDouble.new, RubyLLM::Test) message = Message.new(role: :user, content: "What is the capital of France?") exception = assert_raises(Errors::NoResponseProvidedError) do @@ -34,7 +40,7 @@ def test_when_stub_is_a_message stubbed_message = Message.new(role: :assistant, content: "This is a stubbed message.") RubyLLM::Test.stub_response(stubbed_message) - provider = TestProvider.new(nil, RubyLLM::Test) + provider = TestProvider.new(ProviderDouble.new, RubyLLM::Test) response = provider.complete([], tools: [], temperature: 0.5, model: "test-model", params: {}, headers: {}) assert_equal stubbed_message, response @@ -44,7 +50,7 @@ def test_hash_stub_is_converted_to_json_message stubbed_hash = { answer: "42" } RubyLLM::Test.stub_response(stubbed_hash) - provider = TestProvider.new(nil, RubyLLM::Test) + provider = TestProvider.new(ProviderDouble.new, RubyLLM::Test) response = provider.complete([], tools: [], temperature: 0.5, model: "test-model", params: {}, headers: {}) assert_kind_of Message, response @@ -53,12 +59,18 @@ def test_hash_stub_is_converted_to_json_message end class TestProviderLastCallTest < Minitest::Test + class ProviderDouble + def complete(messages, tools:, temperature:, model:, params: {}, # rubocop:disable Metrics/ParameterLists + headers: {}, schema: nil, thinking: nil, tool_prefs: nil, &) + end + end + def setup RubyLLM::Test.reset RubyLLM::Test.stub_response("stubbed response") initialize_arguments - provider = TestProvider.new(nil, RubyLLM::Test) + provider = TestProvider.new(ProviderDouble.new, RubyLLM::Test) provider.complete(@messages, tools: @tools, temperature: @temperature, model: @model, params: @params, headers: @headers, schema: @schema, thinking: @thinking, tool_prefs: @tool_prefs) diff --git a/test/system/chat_test.rb b/test/system/chat_test.rb new file mode 100644 index 0000000..2f51a65 --- /dev/null +++ b/test/system/chat_test.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +require "test_helper" + +class ChatTest < Minitest::Test + def test_chat_response + RubyLLM::Test.reset + RubyLLM::Test.stub_response("42") + + chat = RubyLLM::Chat.new(model: "gpt-4") + response = chat.ask("What is the meaning of life?") + + assert_equal "42", response.content + end + + def test_block_stub + RubyLLM::Test.with_responses("stubbed response") do + chat = RubyLLM::Chat.new(model: "gpt-4") + response = chat.ask("What is the meaning of life?") + + assert_equal "stubbed response", response.content + end + end +end diff --git a/test/test_helper.rb b/test/test_helper.rb index 0e40b00..1be6da0 100644 --- a/test/test_helper.rb +++ b/test/test_helper.rb @@ -9,3 +9,12 @@ require "ruby_llm/test" require "minitest/autorun" + +RubyLLM::Models.singleton_class.prepend(RubyLLM::Test::ResolveWithTestProvider) + +# need to provide a dummy value for the API key +# the two models used need to exist in the JSON file that comes with the gem +RubyLLM.configure do |config| + config.openai_api_key = "dummy-test-api-key" + config.default_model = "gpt-5.4" +end