From d177c64ce27b838a7fb3c545fbaf8fdad0900ffe Mon Sep 17 00:00:00 2001 From: Alex Kholodniak Date: Sat, 1 Mar 2025 13:01:18 -0600 Subject: [PATCH] feat: add configurable missing data strategies and expanded tests - Introduce a missing_strategy parameter for Rasch, 2PL, 3PL models (:ignore, :treat_as_incorrect, :treat_as_correct). - Add logic to handle nil responses according to the chosen strategy or skip them when ignoring. - Expand RSpec tests to cover repeated fitting, deterministic seeds, large random datasets, and missing data strategies. - Minor code cleanup and improved documentation around usage. --- .rubocop.yml | 3 + lib/irt_ruby.rb | 1 + lib/irt_ruby/rasch_model.rb | 61 ++++++++---- lib/irt_ruby/three_parameter_model.rb | 84 +++++++++------- lib/irt_ruby/two_parameter_model.rb | 60 +++++++----- spec/irt_ruby/rasch_model_spec.rb | 51 ++++++++++ spec/irt_ruby/three_parameter_model_spec.rb | 101 +++++++++++++++++++- spec/irt_ruby/two_parameter_model_spec.rb | 64 ++++++++++++- 8 files changed, 348 insertions(+), 77 deletions(-) diff --git a/.rubocop.yml b/.rubocop.yml index ac57ae2..a86c422 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -35,3 +35,6 @@ Metrics/CyclomaticComplexity: Metrics/PerceivedComplexity: Enabled: false + +Style/HashLikeCase: + Enabled: false \ No newline at end of file diff --git a/lib/irt_ruby.rb b/lib/irt_ruby.rb index a7e273c..c0bf58d 100644 --- a/lib/irt_ruby.rb +++ b/lib/irt_ruby.rb @@ -1,6 +1,7 @@ # frozen_string_literal: true require "irt_ruby/version" +require "matrix" require "irt_ruby/rasch_model" require "irt_ruby/two_parameter_model" require "irt_ruby/three_parameter_model" diff --git a/lib/irt_ruby/rasch_model.rb b/lib/irt_ruby/rasch_model.rb index c6ce120..0df0e82 100644 --- a/lib/irt_ruby/rasch_model.rb +++ b/lib/irt_ruby/rasch_model.rb @@ -1,7 +1,5 @@ # frozen_string_literal: true -require "matrix" - module IrtRuby # A class representing the Rasch model for Item Response Theory (ability - difficulty). # Incorporates: @@ -9,39 +7,64 @@ module IrtRuby # - Missing data handling (skip nil) # - Multiple convergence checks (log-likelihood + parameter updates) class RaschModel - def initialize(data, max_iter: 1000, tolerance: 1e-6, param_tolerance: 1e-6, - learning_rate: 0.01, decay_factor: 0.5) + MISSING_STRATEGIES = %i[ignore treat_as_incorrect treat_as_correct].freeze + + def initialize(data, + max_iter: 1000, + tolerance: 1e-6, + param_tolerance: 1e-6, + learning_rate: 0.01, + decay_factor: 0.5, + missing_strategy: :ignore) # data: A Matrix or array-of-arrays of responses (0/1 or nil for missing). - # Rows = respondents, Columns = items. + # missing_strategy: :ignore (skip), :treat_as_incorrect, :treat_as_correct @data = data @data_array = data.to_a num_rows = @data_array.size num_cols = @data_array.first.size + raise ArgumentError, "missing_strategy must be one of #{MISSING_STRATEGIES}" unless MISSING_STRATEGIES.include?(missing_strategy) + + @missing_strategy = missing_strategy + # Initialize parameters near zero @abilities = Array.new(num_rows) { rand(-0.25..0.25) } @difficulties = Array.new(num_cols) { rand(-0.25..0.25) } - @max_iter = max_iter - @tolerance = tolerance + @max_iter = max_iter + @tolerance = tolerance @param_tolerance = param_tolerance - @learning_rate = learning_rate - @decay_factor = decay_factor + @learning_rate = learning_rate + @decay_factor = decay_factor end def sigmoid(x) 1.0 / (1.0 + Math.exp(-x)) end + def resolve_missing(resp) + return [resp, false] unless resp.nil? + + case @missing_strategy + when :ignore + [nil, true] + when :treat_as_incorrect + [0, false] + when :treat_as_correct + [1, false] + end + end + def log_likelihood total_ll = 0.0 @data_array.each_with_index do |row, i| row.each_with_index do |resp, j| - next if resp.nil? + value, skip = resolve_missing(resp) + next if skip prob = sigmoid(@abilities[i] - @difficulties[j]) - total_ll += if resp == 1 + total_ll += if value == 1 Math.log(prob + 1e-15) else Math.log((1 - prob) + 1e-15) @@ -57,10 +80,11 @@ def compute_gradient @data_array.each_with_index do |row, i| row.each_with_index do |resp, j| - next if resp.nil? + value, skip = resolve_missing(resp) + next if skip prob = sigmoid(@abilities[i] - @difficulties[j]) - error = resp - prob + error = value - prob grad_abilities[i] += error grad_difficulties[j] -= error @@ -102,18 +126,17 @@ def fit @max_iter.times do grad_abilities, grad_difficulties = compute_gradient - old_abilities, old_difficulties = apply_gradient_update(grad_abilities, grad_difficulties) + old_a, old_d = apply_gradient_update(grad_abilities, grad_difficulties) - current_ll = log_likelihood - param_delta = average_param_update(old_abilities, old_difficulties) + current_ll = log_likelihood + param_delta = average_param_update(old_a, old_d) if current_ll < prev_ll - @abilities = old_abilities - @difficulties = old_difficulties + @abilities = old_a + @difficulties = old_d @learning_rate *= @decay_factor else ll_diff = (current_ll - prev_ll).abs - break if ll_diff < @tolerance && param_delta < @param_tolerance prev_ll = current_ll diff --git a/lib/irt_ruby/three_parameter_model.rb b/lib/irt_ruby/three_parameter_model.rb index d8ddd4d..0127ed7 100644 --- a/lib/irt_ruby/three_parameter_model.rb +++ b/lib/irt_ruby/three_parameter_model.rb @@ -1,7 +1,5 @@ # frozen_string_literal: true -require "matrix" - module IrtRuby # A class representing the Three-Parameter model (3PL) for Item Response Theory. # Incorporates: @@ -11,14 +9,25 @@ module IrtRuby # - Multiple convergence checks # - Separate gradient calculation & updates class ThreeParameterModel - def initialize(data, max_iter: 1000, tolerance: 1e-6, param_tolerance: 1e-6, - learning_rate: 0.01, decay_factor: 0.5) + MISSING_STRATEGIES = %i[ignore treat_as_incorrect treat_as_correct].freeze + + def initialize(data, + max_iter: 1000, + tolerance: 1e-6, + param_tolerance: 1e-6, + learning_rate: 0.01, + decay_factor: 0.5, + missing_strategy: :ignore) @data = data @data_array = data.to_a num_rows = @data_array.size num_cols = @data_array.first.size - # Typical initialization for 3PL + raise ArgumentError, "missing_strategy must be one of #{MISSING_STRATEGIES}" unless MISSING_STRATEGIES.include?(missing_strategy) + + @missing_strategy = missing_strategy + + # Initialize parameters @abilities = Array.new(num_rows) { rand(-0.25..0.25) } @difficulties = Array.new(num_cols) { rand(-0.25..0.25) } @discriminations = Array.new(num_cols) { rand(0.5..1.5) } @@ -40,15 +49,32 @@ def probability(theta, a, b, c) c + (1.0 - c) * sigmoid(a * (theta - b)) end + def resolve_missing(resp) + return [resp, false] unless resp.nil? + + case @missing_strategy + when :ignore + [nil, true] + when :treat_as_incorrect + [0, false] + when :treat_as_correct + [1, false] + end + end + def log_likelihood ll = 0.0 @data_array.each_with_index do |row, i| row.each_with_index do |resp, j| - next if resp.nil? + value, skip = resolve_missing(resp) + next if skip - prob = probability(@abilities[i], @discriminations[j], - @difficulties[j], @guessings[j]) - ll += if resp == 1 + prob = probability(@abilities[i], + @discriminations[j], + @difficulties[j], + @guessings[j]) + + ll += if value == 1 Math.log(prob + 1e-15) else Math.log((1 - prob) + 1e-15) @@ -66,7 +92,8 @@ def compute_gradient @data_array.each_with_index do |row, i| row.each_with_index do |resp, j| - next if resp.nil? + value, skip = resolve_missing(resp) + next if skip theta = @abilities[i] a = @discriminations[j] @@ -74,13 +101,13 @@ def compute_gradient c = @guessings[j] prob = probability(theta, a, b, c) - error = resp - prob + error = value - prob - grad_abilities[i] += error * a * (1 - c) - grad_difficulties[j] -= error * a * (1 - c) + grad_abilities[i] += error * a * (1 - c) + grad_difficulties[j] -= error * a * (1 - c) grad_discriminations[j] += error * (theta - b) * (1 - c) - grad_guessings[j] += error * 1.0 + grad_guessings[j] += error * 1.0 end end @@ -88,10 +115,10 @@ def compute_gradient end def apply_gradient_update(ga, gd, gdisc, gc) - old_abilities = @abilities.dup - old_difficulties = @difficulties.dup - old_discriminations = @discriminations.dup - old_guessings = @guessings.dup + old_a = @abilities.dup + old_d = @difficulties.dup + old_disc = @discriminations.dup + old_c = @guessings.dup @abilities.each_index do |i| @abilities[i] += @learning_rate * ga[i] @@ -113,23 +140,15 @@ def apply_gradient_update(ga, gd, gdisc, gc) @guessings[j] = 0.35 if @guessings[j] > 0.35 end - [old_abilities, old_difficulties, old_discriminations, old_guessings] + [old_a, old_d, old_disc, old_c] end def average_param_update(old_a, old_d, old_disc, old_c) deltas = [] - @abilities.each_with_index do |x, i| - deltas << (x - old_a[i]).abs - end - @difficulties.each_with_index do |x, j| - deltas << (x - old_d[j]).abs - end - @discriminations.each_with_index do |x, j| - deltas << (x - old_disc[j]).abs - end - @guessings.each_with_index do |x, j| - deltas << (x - old_c[j]).abs - end + @abilities.each_with_index { |x, i| deltas << (x - old_a[i]).abs } + @difficulties.each_with_index { |x, j| deltas << (x - old_d[j]).abs } + @discriminations.each_with_index { |x, j| deltas << (x - old_disc[j]).abs } + @guessings.each_with_index { |x, j| deltas << (x - old_c[j]).abs } deltas.sum / deltas.size end @@ -140,7 +159,7 @@ def fit ga, gd, gdisc, gc = compute_gradient old_a, old_d, old_disc, old_c = apply_gradient_update(ga, gd, gdisc, gc) - curr_ll = log_likelihood + curr_ll = log_likelihood param_delta = average_param_update(old_a, old_d, old_disc, old_c) if curr_ll < prev_ll @@ -148,7 +167,6 @@ def fit @difficulties = old_d @discriminations = old_disc @guessings = old_c - @learning_rate *= @decay_factor else ll_diff = (curr_ll - prev_ll).abs diff --git a/lib/irt_ruby/two_parameter_model.rb b/lib/irt_ruby/two_parameter_model.rb index 5004f69..fd48a6a 100644 --- a/lib/irt_ruby/two_parameter_model.rb +++ b/lib/irt_ruby/two_parameter_model.rb @@ -1,7 +1,5 @@ # frozen_string_literal: true -require "matrix" - module IrtRuby # A class representing the Two-Parameter model (2PL) for IRT. # Incorporates: @@ -11,18 +9,25 @@ module IrtRuby # - Multiple convergence checks # - Separate gradient calculation & parameter update class TwoParameterModel + MISSING_STRATEGIES = %i[ignore treat_as_incorrect treat_as_correct].freeze + def initialize(data, max_iter: 1000, tolerance: 1e-6, param_tolerance: 1e-6, - learning_rate: 0.01, decay_factor: 0.5) + learning_rate: 0.01, decay_factor: 0.5, + missing_strategy: :ignore) @data = data @data_array = data.to_a num_rows = @data_array.size num_cols = @data_array.first.size + raise ArgumentError, "missing_strategy must be one of #{MISSING_STRATEGIES}" unless MISSING_STRATEGIES.include?(missing_strategy) + + @missing_strategy = missing_strategy + # Initialize parameters # Typically: ability ~ 0, difficulty ~ 0, discrimination ~ 1 @abilities = Array.new(num_rows) { rand(-0.25..0.25) } @difficulties = Array.new(num_cols) { rand(-0.25..0.25) } - @discriminations = Array.new(num_cols) { rand(0.5..1.5) } # Start around 1.0 + @discriminations = Array.new(num_cols) { rand(0.5..1.5) } @max_iter = max_iter @tolerance = tolerance @@ -35,14 +40,28 @@ def sigmoid(x) 1.0 / (1.0 + Math.exp(-x)) end + def resolve_missing(resp) + return [resp, false] unless resp.nil? + + case @missing_strategy + when :ignore + [nil, true] + when :treat_as_incorrect + [0, false] + when :treat_as_correct + [1, false] + end + end + def log_likelihood ll = 0.0 @data_array.each_with_index do |row, i| row.each_with_index do |resp, j| - next if resp.nil? + value, skip = resolve_missing(resp) + next if skip prob = sigmoid(@discriminations[j] * (@abilities[i] - @difficulties[j])) - ll += if resp == 1 + ll += if value == 1 Math.log(prob + 1e-15) else Math.log((1 - prob) + 1e-15) @@ -59,10 +78,11 @@ def compute_gradient @data_array.each_with_index do |row, i| row.each_with_index do |resp, j| - next if resp.nil? + value, skip = resolve_missing(resp) + next if skip - prob = sigmoid(@discriminations[j] * (@abilities[i] - @difficulties[j])) - error = resp - prob + prob = sigmoid(@discriminations[j] * (@abilities[i] - @difficulties[j])) + error = value - prob grad_abilities[i] += error * @discriminations[j] grad_difficulties[j] -= error * @discriminations[j] @@ -74,9 +94,9 @@ def compute_gradient end def apply_gradient_update(ga, gd, gdisc) - old_abilities = @abilities.dup - old_difficulties = @difficulties.dup - old_discriminations = @discriminations.dup + old_a = @abilities.dup + old_d = @difficulties.dup + old_disc = @discriminations.dup @abilities.each_index do |i| @abilities[i] += @learning_rate * ga[i] @@ -92,20 +112,14 @@ def apply_gradient_update(ga, gd, gdisc) @discriminations[j] = 5.0 if @discriminations[j] > 5.0 end - [old_abilities, old_difficulties, old_discriminations] + [old_a, old_d, old_disc] end def average_param_update(old_a, old_d, old_disc) deltas = [] - @abilities.each_with_index do |x, i| - deltas << (x - old_a[i]).abs - end - @difficulties.each_with_index do |x, j| - deltas << (x - old_d[j]).abs - end - @discriminations.each_with_index do |x, j| - deltas << (x - old_disc[j]).abs - end + @abilities.each_with_index { |x, i| deltas << (x - old_a[i]).abs } + @difficulties.each_with_index { |x, j| deltas << (x - old_d[j]).abs } + @discriminations.each_with_index { |x, j| deltas << (x - old_disc[j]).abs } deltas.sum / deltas.size end @@ -116,7 +130,7 @@ def fit ga, gd, gdisc = compute_gradient old_a, old_d, old_disc = apply_gradient_update(ga, gd, gdisc) - curr_ll = log_likelihood + curr_ll = log_likelihood param_delta = average_param_update(old_a, old_d, old_disc) if curr_ll < prev_ll diff --git a/spec/irt_ruby/rasch_model_spec.rb b/spec/irt_ruby/rasch_model_spec.rb index d618ae9..2e7a417 100644 --- a/spec/irt_ruby/rasch_model_spec.rb +++ b/spec/irt_ruby/rasch_model_spec.rb @@ -51,6 +51,12 @@ expect(result[:abilities]).not_to be_empty expect(result[:difficulties]).not_to be_empty end + + it "treats missing as incorrect" do + missing_data = [[1, nil], [nil, 0]] + model = described_class.new(missing_data, missing_strategy: :treat_as_incorrect) + expect { model.fit }.not_to raise_error + end end describe "Edge cases" do @@ -136,4 +142,49 @@ expect(final_ll).to be > initial_ll end end + + describe "Additional tests" do + context "Repeated fitting" do + it "handles multiple calls to fit without error" do + model = described_class.new(data_array, max_iter: 100) + first_result = model.fit + second_result = model.fit + + # Usually the second call won't change much, but let's ensure it's valid + expect(second_result[:abilities].size).to eq(first_result[:abilities].size) + expect(second_result[:difficulties].size).to eq(first_result[:difficulties].size) + end + end + + context "Deterministic seed" do + it "produces consistent results with the same seed" do + srand(123) + model1 = described_class.new(data_array, max_iter: 200, learning_rate: 0.05) + result1 = model1.fit + + srand(123) + model2 = described_class.new(data_array, max_iter: 200, learning_rate: 0.05) + result2 = model2.fit + + expect(result1[:abilities]).to eq(result2[:abilities]) + expect(result1[:difficulties]).to eq(result2[:difficulties]) + end + end + + context "Larger random dataset" do + it "handles a moderately large dataset without error" do + n_examinees = 20 + n_items = 10 + big_data = Array.new(n_examinees) do + Array.new(n_items) { rand < 0.5 ? 1 : 0 } + end + model = described_class.new(big_data, max_iter: 300, learning_rate: 0.05) + expect { model.fit }.not_to raise_error + + results = model.fit + expect(results[:abilities].size).to eq(n_examinees) + expect(results[:difficulties].size).to eq(n_items) + end + end + end end diff --git a/spec/irt_ruby/three_parameter_model_spec.rb b/spec/irt_ruby/three_parameter_model_spec.rb index 0f92b02..8693393 100644 --- a/spec/irt_ruby/three_parameter_model_spec.rb +++ b/spec/irt_ruby/three_parameter_model_spec.rb @@ -1,7 +1,6 @@ # frozen_string_literal: true require "spec_helper" -require "matrix" RSpec.describe IrtRuby::ThreeParameterModel do let(:data_array) do @@ -62,6 +61,36 @@ end end + describe "Missing data strategies" do + let(:data_with_missing) do + [ + [1, nil, 0], + [nil, 0, 1] + ] + end + + it "ignores missing data by default" do + model = IrtRuby::ThreeParameterModel.new(data_with_missing) + expect { model.fit }.not_to raise_error + end + + it "treats missing as incorrect" do + model = IrtRuby::ThreeParameterModel.new(data_with_missing, missing_strategy: :treat_as_incorrect) + expect { model.fit }.not_to raise_error + end + + it "treats missing as correct" do + model = IrtRuby::ThreeParameterModel.new(data_with_missing, missing_strategy: :treat_as_correct) + expect { model.fit }.not_to raise_error + end + + it "raises an error on invalid strategy" do + expect do + IrtRuby::ThreeParameterModel.new(data_with_missing, missing_strategy: :not_a_valid_strategy) + end.to raise_error(ArgumentError) + end + end + describe "Edge cases" do it "works with a single examinee and single item" do data = [[0]] @@ -162,4 +191,74 @@ expect(final_ll).to be > initial_ll end end + + describe "Additional tests" do + context "Repeated fitting" do + it "handles multiple calls to fit" do + model = described_class.new(data_array, max_iter: 100) + first_result = model.fit + second_result = model.fit + + expect(second_result[:abilities].size).to eq(first_result[:abilities].size) + expect(second_result[:difficulties].size).to eq(first_result[:difficulties].size) + expect(second_result[:discriminations].size).to eq(first_result[:discriminations].size) + expect(second_result[:guessings].size).to eq(first_result[:guessings].size) + end + end + + context "Deterministic seed" do + it "produces consistent results with the same seed" do + srand(123) + model1 = described_class.new(data_array, max_iter: 200, learning_rate: 0.05) + result1 = model1.fit + + srand(123) + model2 = described_class.new(data_array, max_iter: 200, learning_rate: 0.05) + result2 = model2.fit + + expect(result1[:abilities]).to eq(result2[:abilities]) + expect(result1[:difficulties]).to eq(result2[:difficulties]) + expect(result1[:discriminations]).to eq(result2[:discriminations]) + expect(result1[:guessings]).to eq(result2[:guessings]) + end + end + + context "Larger random dataset" do + it "handles a moderately large dataset without error" do + n_examinees = 20 + n_items = 8 + big_data = Array.new(n_examinees) do + Array.new(n_items) { rand < 0.5 ? 1 : 0 } + end + + model = described_class.new(big_data, max_iter: 300, learning_rate: 0.05) + expect { model.fit }.not_to raise_error + + results = model.fit + expect(results[:abilities].size).to eq(n_examinees) + expect(results[:difficulties].size).to eq(n_items) + expect(results[:discriminations].size).to eq(n_items) + expect(results[:guessings].size).to eq(n_items) + end + end + + context "Known parameter test (optional)" do + it "attempts to recover small synthetic data parameters" do + data = [ + [1, 1], + [1, 1] + ] + + model = described_class.new(data, max_iter: 200, learning_rate: 0.05) + results = model.fit + + results[:discriminations].each do |disc| + expect(disc).to be_between(0.01, 5.0) + end + results[:guessings].each do |g| + expect(g).to be_between(0.0, 0.35) + end + end + end + end end diff --git a/spec/irt_ruby/two_parameter_model_spec.rb b/spec/irt_ruby/two_parameter_model_spec.rb index bb1bb89..6687b87 100644 --- a/spec/irt_ruby/two_parameter_model_spec.rb +++ b/spec/irt_ruby/two_parameter_model_spec.rb @@ -1,7 +1,6 @@ # frozen_string_literal: true require "spec_helper" -require "matrix" RSpec.describe IrtRuby::TwoParameterModel do let(:data_array) do @@ -152,4 +151,67 @@ expect(final_ll).to be > initial_ll end end + + describe "Additional tests" do + context "Repeated fitting" do + it "handles multiple calls to fit without error" do + model = described_class.new(data_array, max_iter: 100) + first_result = model.fit + second_result = model.fit + + expect(second_result[:abilities].size).to eq(first_result[:abilities].size) + expect(second_result[:difficulties].size).to eq(first_result[:difficulties].size) + expect(second_result[:discriminations].size).to eq(first_result[:discriminations].size) + end + end + + context "Deterministic seed" do + it "yields consistent results with the same seed" do + srand(123) + model1 = described_class.new(data_array, max_iter: 200, learning_rate: 0.05) + result1 = model1.fit + + srand(123) + model2 = described_class.new(data_array, max_iter: 200, learning_rate: 0.05) + result2 = model2.fit + + expect(result1[:abilities]).to eq(result2[:abilities]) + expect(result1[:difficulties]).to eq(result2[:difficulties]) + expect(result1[:discriminations]).to eq(result2[:discriminations]) + end + end + + context "Larger random dataset" do + it "handles a moderately sized dataset without error" do + n_examinees = 20 + n_items = 8 + big_data = Array.new(n_examinees) do + Array.new(n_items) { rand < 0.5 ? 1 : 0 } + end + + model = described_class.new(big_data, max_iter: 300, learning_rate: 0.05) + expect { model.fit }.not_to raise_error + + results = model.fit + expect(results[:abilities].size).to eq(n_examinees) + expect(results[:difficulties].size).to eq(n_items) + expect(results[:discriminations].size).to eq(n_items) + end + end + + context "Known parameter test (optional)" do + it "checks parameter ranges on a small synthetic dataset" do + data = [ + [1, 1], + [1, 1] + ] + model = described_class.new(data, max_iter: 200, learning_rate: 0.05) + results = model.fit + + results[:discriminations].each do |disc| + expect(disc).to be_between(0.01, 5.0) + end + end + end + end end