Merge pull request #11 from andrew/sequel-migration

Replace ActiveRecord with Sequel

Author: andrew

Gemfile

@@ -4,7 +4,8 @@ source "https://rubygems.org"
 
 gemspec
 
-#gem "ecosystems-bibliothecary", git: "https://github.com/ecosyste-ms/bibliothecary.git", require: "bibliothecary"
+gem "ecosystems-bibliothecary", git: "https://github.com/ecosyste-ms/bibliothecary.git", require: "bibliothecary"
+# gem "ecosystems-bibliothecary", path: "/Users/andrew/code/ecosystems/bibliothecary", require: "bibliothecary"
 gem "ostruct"
 
 gem "irb"

Gemfile.lock

@@ -1,59 +1,38 @@
+GIT
+  remote: https://github.com/ecosyste-ms/bibliothecary.git
+  revision: 827bda17dc8866d3f712d40942760f4d52e23b63
+  specs:
+    ecosystems-bibliothecary (15.0.1)
+      bundler
+      csv
+      json (~> 2.8)
+      ox (>= 2.8.1)
+      racc
+      tomlrb (~> 2.0)
+
 PATH
   remote: .
   specs:
     git-pkgs (0.5.0)
-      activerecord (>= 7.0)
       ecosystems-bibliothecary (~> 15.0)
       rugged (~> 1.0)
+      sequel (>= 5.0)
       sqlite3 (>= 2.0)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    activemodel (8.1.1)
-      activesupport (= 8.1.1)
-    activerecord (8.1.1)
-      activemodel (= 8.1.1)
-      activesupport (= 8.1.1)
-      timeout (>= 0.4.0)
-    activesupport (8.1.1)
-      base64
-      bigdecimal
-      concurrent-ruby (~> 1.0, >= 1.3.1)
-      connection_pool (>= 2.2.5)
-      drb
-      i18n (>= 1.6, < 2)
-      json
-      logger (>= 1.4.2)
-      minitest (>= 5.1)
-      securerandom (>= 0.3)
-      tzinfo (~> 2.0, >= 2.0.5)
-      uri (>= 0.13.1)
-    base64 (0.3.0)
     benchmark (0.5.0)
     bigdecimal (4.0.1)
-    concurrent-ruby (1.3.6)
-    connection_pool (3.0.2)
     csv (3.3.5)
     date (3.5.1)
-    drb (2.2.3)
-    ecosystems-bibliothecary (15.0.1)
-      bundler
-      csv
-      json (~> 2.8)
-      ox (>= 2.8.1)
-      racc
-      tomlrb (~> 2.0)
     erb (6.0.1)
-    i18n (1.14.8)
-      concurrent-ruby (~> 1.0)
     io-console (0.8.2)
     irb (1.16.0)
       pp (>= 0.6.0)
       rdoc (>= 4.0.0)
       reline (>= 0.4.2)
     json (2.18.0)
-    logger (1.7.0)
     minitest (6.0.1)
       prism (~> 1.5)
     ostruct (0.6.3)
@@ -75,7 +54,8 @@ GEM
     reline (0.6.3)
       io-console (~> 0.5)
     rugged (1.9.0)
-    securerandom (0.4.1)
+    sequel (5.100.0)
+      bigdecimal
     sqlite3 (2.9.0-aarch64-linux-gnu)
     sqlite3 (2.9.0-aarch64-linux-musl)
     sqlite3 (2.9.0-arm-linux-gnu)
@@ -87,12 +67,8 @@ GEM
     sqlite3 (2.9.0-x86_64-linux-gnu)
     sqlite3 (2.9.0-x86_64-linux-musl)
     stringio (3.2.0)
-    timeout (0.6.0)
     tomlrb (2.0.4)
     tsort (0.2.0)
-    tzinfo (2.0.6)
-      concurrent-ruby (~> 1.0)
-    uri (1.1.1)
 
 PLATFORMS
   aarch64-linux-gnu
@@ -108,32 +84,24 @@ PLATFORMS
 
 DEPENDENCIES
   benchmark
+  ecosystems-bibliothecary!
   git-pkgs!
   irb
   minitest
   ostruct
   rake
 
 CHECKSUMS
-  activemodel (8.1.1) sha256=8b7e2496b9e333ced06248c16a43217b950192c98e0fe3aa117eee21501c6fbd
-  activerecord (8.1.1) sha256=e32c3a03e364fd803498eb4150c21bedc995aa83bc27122a94d480ab1dcb3d17
-  activesupport (8.1.1) sha256=5e92534e8d0c8b8b5e6b16789c69dbea65c1d7b752269f71a39422e9546cea67
-  base64 (0.3.0) sha256=27337aeabad6ffae05c265c450490628ef3ebd4b67be58257393227588f5a97b
   benchmark (0.5.0) sha256=465df122341aedcb81a2a24b4d3bd19b6c67c1530713fd533f3ff034e419236c
   bigdecimal (4.0.1) sha256=8b07d3d065a9f921c80ceaea7c9d4ae596697295b584c296fe599dd0ad01c4a7
-  concurrent-ruby (1.3.6) sha256=6b56837e1e7e5292f9864f34b69c5a2cbc75c0cf5338f1ce9903d10fa762d5ab
-  connection_pool (3.0.2) sha256=33fff5ba71a12d2aa26cb72b1db8bba2a1a01823559fb01d29eb74c286e62e0a
   csv (3.3.5) sha256=6e5134ac3383ef728b7f02725d9872934f523cb40b961479f69cf3afa6c8e73f
   date (3.5.1) sha256=750d06384d7b9c15d562c76291407d89e368dda4d4fff957eb94962d325a0dc0
-  drb (2.2.3) sha256=0b00d6fdb50995fe4a45dea13663493c841112e4068656854646f418fda13373
-  ecosystems-bibliothecary (15.0.1) sha256=e557381cbe1a56931cb68c8a685a33dd9156a7b331a583874b4ac05f94df6f1c
+  ecosystems-bibliothecary (15.0.1)
   erb (6.0.1) sha256=28ecdd99c5472aebd5674d6061e3c6b0a45c049578b071e5a52c2a7f13c197e5
   git-pkgs (0.5.0)
-  i18n (1.14.8) sha256=285778639134865c5e0f6269e0b818256017e8cde89993fdfcbfb64d088824a5
   io-console (0.8.2) sha256=d6e3ae7a7cc7574f4b8893b4fca2162e57a825b223a177b7afa236c5ef9814cc
   irb (1.16.0) sha256=2abe56c9ac947cdcb2f150572904ba798c1e93c890c256f8429981a7675b0806
   json (2.18.0) sha256=b10506aee4183f5cf49e0efc48073d7b75843ce3782c68dbeb763351c08fd505
-  logger (1.7.0) sha256=196edec7cc44b66cfb40f9755ce11b392f21f7967696af15d274dde7edff0203
   minitest (6.0.1) sha256=7854c74f48e2e975969062833adc4013f249a4b212f5e7b9d5c040bf838d54bb
   ostruct (0.6.3) sha256=95a2ed4a4bd1d190784e666b47b2d3f078e4a9efda2fccf18f84ddc6538ed912
   ox (2.14.23) sha256=4a9aedb4d6c78c5ebac1d7287dc7cc6808e14a8831d7adb727438f6a1b461b66
@@ -146,7 +114,7 @@ CHECKSUMS
   rdoc (7.0.3) sha256=dfe3d0981d19b7bba71d9dbaeb57c9f4e3a7a4103162148a559c4fc687ea81f9
   reline (0.6.3) sha256=1198b04973565b36ec0f11542ab3f5cfeeec34823f4e54cebde90968092b1835
   rugged (1.9.0) sha256=7faaa912c5888d6e348d20fa31209b6409f1574346b1b80e309dbc7e8d63efac
-  securerandom (0.4.1) sha256=cc5193d414a4341b6e225f0cb4446aceca8e50d5e1888743fac16987638ea0b1
+  sequel (5.100.0) sha256=cb0329b62287a01db68eead46759c14497a3fae01b174e2c41da108a9e9b4a12
   sqlite3 (2.9.0-aarch64-linux-gnu) sha256=cfe1e0216f46d7483839719bf827129151e6c680317b99d7b8fc1597a3e13473
   sqlite3 (2.9.0-aarch64-linux-musl) sha256=56a35cb2d70779afc2ac191baf2c2148242285ecfed72f9b021218c5c4917913
   sqlite3 (2.9.0-arm-linux-gnu) sha256=a19a21504b0d7c8c825fbbf37b358ae316b6bd0d0134c619874060b2eef05435
@@ -158,11 +126,8 @@ CHECKSUMS
   sqlite3 (2.9.0-x86_64-linux-gnu) sha256=72fff9bd750070ba3af695511ba5f0e0a2d8a9206f84869640b3e99dfaf3d5a5
   sqlite3 (2.9.0-x86_64-linux-musl) sha256=ef716ba7a66d7deb1ccc402ac3a6d7343da17fac862793b7f0be3d2917253c90
   stringio (3.2.0) sha256=c37cb2e58b4ffbd33fe5cd948c05934af997b36e0b6ca6fdf43afa234cf222e1
-  timeout (0.6.0) sha256=6d722ad619f96ee383a0c557ec6eb8c4ecb08af3af62098a0be5057bf00de1af
   tomlrb (2.0.4) sha256=262f77947ac3ac9b3366a0a5940ecd238300c553e2e14f22009e2afcd2181b99
   tsort (0.2.0) sha256=9650a793f6859a43b6641671278f79cfead60ac714148aabe4e3f0060480089f
-  tzinfo (2.0.6) sha256=8daf828cc77bcf7d63b0e3bdb6caa47e2272dcfaf4fbfe46f8c3a9df087a829b
-  uri (1.1.1) sha256=379fa58d27ffb1387eaada68c749d1426738bd0f654d812fcc07e7568f5c57c6
 
 BUNDLED WITH
   4.0.1

benchmark/bulk.rb

@@ -8,10 +8,10 @@
 repo_path = ARGV[0] || "/Users/andrew/code/octobox"
 sample_size = (ARGV[1] || 500).to_i
 
-# In-memory with WAL mode equivalent (journal_mode=memory for in-memory DB)
+# In-memory with optimized settings
 Git::Pkgs::Database.connect_memory
-ActiveRecord::Base.connection.execute("PRAGMA synchronous = OFF")
-ActiveRecord::Base.connection.execute("PRAGMA journal_mode = MEMORY")
+Git::Pkgs::Database.db.run("PRAGMA synchronous = OFF")
+Git::Pkgs::Database.db.run("PRAGMA journal_mode = MEMORY")
 
 repo = Git::Pkgs::Repository.new(repo_path)
 analyzer = Git::Pkgs::Analyzer.new(repo)
@@ -111,11 +111,11 @@
 # Bulk insert
 insert_time = Benchmark.realtime do
   # Insert commits
-  Git::Pkgs::Models::Commit.insert_all(all_commits) if all_commits.any?
+  Git::Pkgs::Models::Commit.multi_insert(all_commits) if all_commits.any?
 
   # Build SHA -> ID map
-  commit_ids = Git::Pkgs::Models::Commit.where(sha: all_commits.map { |c| c[:sha] }).pluck(:sha, :id).to_h
-  manifest_ids = Git::Pkgs::Models::Manifest.pluck(:path, :id).to_h
+  commit_ids = Git::Pkgs::Models::Commit.where(sha: all_commits.map { |c| c[:sha] }).select_map([:sha, :id]).to_h
+  manifest_ids = Git::Pkgs::Models::Manifest.select_map([:path, :id]).to_h
 
   # Insert branch_commits with resolved IDs
   branch_commit_records = all_branch_commits.map do |bc|
@@ -125,7 +125,7 @@
       position: bc[:commit_position]
     }
   end
-  Git::Pkgs::Models::BranchCommit.insert_all(branch_commit_records) if branch_commit_records.any?
+  Git::Pkgs::Models::BranchCommit.multi_insert(branch_commit_records) if branch_commit_records.any?
 
   # Insert changes with resolved IDs
   change_records = all_changes.map do |c|
@@ -142,7 +142,7 @@
       updated_at: c[:updated_at]
     }
   end
-  Git::Pkgs::Models::DependencyChange.insert_all(change_records) if change_records.any?
+  Git::Pkgs::Models::DependencyChange.multi_insert(change_records) if change_records.any?
 
   # Insert snapshots with resolved IDs
   snapshot_records = all_snapshots.map do |s|
@@ -157,7 +157,7 @@
       updated_at: s[:updated_at]
     }
   end
-  Git::Pkgs::Models::DependencySnapshot.insert_all(snapshot_records) if snapshot_records.any?
+  Git::Pkgs::Models::DependencySnapshot.multi_insert(snapshot_records) if snapshot_records.any?
 end
 
 puts "Insert time: #{insert_time.round(3)}s"

benchmark/db.rb

@@ -52,11 +52,10 @@
   counts[:commits] += 1
 
   timings[:branch_commit_create] += Benchmark.realtime do
-    Git::Pkgs::Models::BranchCommit.find_or_create_by(
-      branch: branch,
-      commit: commit,
-      position: position
-    )
+    Git::Pkgs::Models::BranchCommit.find_or_create(
+      branch_id: branch.id,
+      commit_id: commit.id
+    ) { |bc| bc.position = position }
   end
   counts[:branch_commits] += 1
 
@@ -77,7 +76,7 @@
     end
 
     timings[:change_create] += Benchmark.realtime do
-      Git::Pkgs::Models::DependencyChange.create!(
+      Git::Pkgs::Models::DependencyChange.create(
         commit: commit,
         manifest: manifest,
         name: change[:name],
@@ -95,10 +94,10 @@
 
   snapshot.each do |(manifest_path, name), dep_info|
     timings[:snapshot_create] += Benchmark.realtime do
-      manifest = Git::Pkgs::Models::Manifest.find_by(path: manifest_path)
-      Git::Pkgs::Models::DependencySnapshot.find_or_create_by(
-        commit: commit,
-        manifest: manifest,
+      manifest = Git::Pkgs::Models::Manifest.first(path: manifest_path)
+      Git::Pkgs::Models::DependencySnapshot.find_or_create(
+        commit_id: commit.id,
+        manifest_id: manifest.id,
         name: name
       ) do |s|
         s.ecosystem = dep_info[:ecosystem]

git-pkgs.gemspec

@@ -32,7 +32,7 @@ Gem::Specification.new do |spec|
   spec.require_paths = ["lib"]
 
   spec.add_dependency "rugged", "~> 1.0"
-  spec.add_dependency "activerecord", ">= 7.0"
+  spec.add_dependency "sequel", ">= 5.0"
   spec.add_dependency "sqlite3", ">= 2.0"
   spec.add_dependency "ecosystems-bibliothecary", "~> 15.0"
 end

lib/git/pkgs/commands/blame.rb

@@ -19,12 +19,12 @@ def run
 
           # Get current dependencies at the last analyzed commit
           branch_name = @options[:branch] || repo.default_branch
-          branch = Models::Branch.find_by(name: branch_name)
+          branch = Models::Branch.first(name: branch_name)
 
           error "No analysis found for branch '#{branch_name}'. Run 'git pkgs init' first." unless branch&.last_analyzed_sha
 
-          current_commit = Models::Commit.find_by(sha: branch.last_analyzed_sha)
-          snapshots = current_commit&.dependency_snapshots&.includes(:manifest) || []
+          current_commit = Models::Commit.first(sha: branch.last_analyzed_sha)
+          snapshots = current_commit&.dependency_snapshots&.eager(:manifest) || []
 
           if @options[:ecosystem]
             snapshots = snapshots.where(ecosystem: @options[:ecosystem])
@@ -41,7 +41,7 @@ def run
           names = snapshots.map(&:name).uniq
 
           all_added_changes = Models::DependencyChange
-            .includes(:commit)
+            .eager(:commit)
             .added
             .where(manifest_id: manifest_ids, name: names)
             .to_a

lib/git/pkgs/commands/branch.rb

@@ -50,7 +50,7 @@ def add_branch
 
           error "Branch '#{branch_name}' not found. Check 'git branch -a' for available branches." unless repo.branch_exists?(branch_name)
 
-          existing = Models::Branch.find_by(name: branch_name)
+          existing = Models::Branch.first(name: branch_name)
           if existing
             info "Branch '#{branch_name}' already tracked (#{existing.commits.count} commits)"
             info "Use 'git pkgs update' to refresh"
@@ -59,7 +59,7 @@ def add_branch
 
           Database.optimize_for_bulk_writes
 
-          branch = Models::Branch.create!(name: branch_name)
+          branch = Models::Branch.create(name: branch_name)
           analyzer = Analyzer.new(repo)
 
           info "Analyzing branch: #{branch_name}"
@@ -100,7 +100,7 @@ def list_branches
           puts "Tracked branches:"
           branches.each do |branch|
             commit_count = branch.commits.count
-            dep_commits = branch.commits.where(has_dependency_changes: true).count
+            dep_commits = branch.commits_dataset.where(has_dependency_changes: true).count
             last_sha = branch.last_analyzed_sha&.slice(0, 7) || "none"
             puts "  #{branch.name}: #{commit_count} commits (#{dep_commits} with deps), last: #{last_sha}"
           end
@@ -115,13 +115,13 @@ def remove_branch
 
           Database.connect(repo.git_dir)
 
-          branch = Models::Branch.find_by(name: branch_name)
+          branch = Models::Branch.first(name: branch_name)
           error "Branch '#{branch_name}' not tracked. Run 'git pkgs branch list' to see tracked branches." unless branch
 
           # Only delete branch_commits, keep shared commits
           count = branch.branch_commits.count
-          branch.branch_commits.delete_all
-          branch.destroy
+          branch.branch_commits_dataset.delete
+          branch.delete
 
           info "Removed branch '#{branch_name}' (#{count} branch-commit links)"
         end
@@ -143,28 +143,27 @@ def bulk_process_commits(commits, branch, analyzer, total, repo)
           flush = lambda do
             return if pending_commits.empty?
 
-            ActiveRecord::Base.transaction do
-              # Use upsert for commits since they may already exist from other branches
+            Database.db.transaction do
+              # Use insert with on_conflict for commits since they may already exist from other branches
               if pending_commits.any?
-                Models::Commit.upsert_all(
-                  pending_commits,
-                  unique_by: :sha
-                )
+                Models::Commit.dataset
+                  .insert_conflict(target: :sha, update: { has_dependency_changes: Sequel[:excluded][:has_dependency_changes] })
+                  .multi_insert(pending_commits)
               end
 
               commit_ids = Models::Commit
                 .where(sha: pending_commits.map { |c| c[:sha] })
-                .pluck(:sha, :id).to_h
+                .select_hash(:sha, :id)
 
               if pending_branch_commits.any?
                 branch_commit_records = pending_branch_commits.map do |bc|
                   { branch_id: bc[:branch_id], commit_id: commit_ids[bc[:sha]], position: bc[:position] }
                 end
-                Models::BranchCommit.insert_all(branch_commit_records)
+                Models::BranchCommit.dataset.multi_insert(branch_commit_records)
               end
 
               if pending_changes.any?
-                manifest_ids = Models::Manifest.pluck(:path, :id).to_h
+                manifest_ids = Models::Manifest.select_hash(:path, :id)
                 change_records = pending_changes.map do |c|
                   {
                     commit_id: commit_ids[c[:sha]],
@@ -179,11 +178,11 @@ def bulk_process_commits(commits, branch, analyzer, total, repo)
                     updated_at: now
                   }
                 end
-                Models::DependencyChange.insert_all(change_records)
+                Models::DependencyChange.dataset.multi_insert(change_records)
               end
 
               if pending_snapshots.any?
-                manifest_ids ||= Models::Manifest.pluck(:path, :id).to_h
+                manifest_ids ||= Models::Manifest.select_hash(:path, :id)
                 snapshot_records = pending_snapshots.map do |s|
                   {
                     commit_id: commit_ids[s[:sha]],
@@ -196,7 +195,7 @@ def bulk_process_commits(commits, branch, analyzer, total, repo)
                     updated_at: now
                   }
                 end
-                Models::DependencySnapshot.insert_all(snapshot_records)
+                Models::DependencySnapshot.dataset.multi_insert(snapshot_records)
               end
             end