From e986dde73584036846c272ba683e4b6dd50ac66f Mon Sep 17 00:00:00 2001 From: Cindy Xie Date: Wed, 16 Jul 2025 14:14:00 +0100 Subject: [PATCH 1/4] adding logging to see how big the drift is --- models/table.rb | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/models/table.rb b/models/table.rb index 4ecd3a7..6d668dd 100644 --- a/models/table.rb +++ b/models/table.rb @@ -115,6 +115,15 @@ def copy end end + def log_row_drift + source_count = source_connection.exec_query("SELECT COUNT(*) AS count FROM #{source_name}").first["count"].to_i + dest_count = destination_connection.exec_query("SELECT COUNT(*) AS count FROM #{destination_name}").first["count"].to_i + drift = source_count - dest_count + logger.info "Drift for #{source_name}: source=#{source_count}, destination=#{dest_count}, drift=#{drift}" + rescue => e + logger.error "Error calculating drift for #{source_name}: #{e.message}" + end + def copy_now started_at = Time.now return 0 unless (self.enabled? && self.check) @@ -144,6 +153,7 @@ def copy_now update_max_values(temp_table_name) destination_connection.execute("DROP TABLE #{temp_table_name};") + log_row_drift end #Log for benchmarking From 0157ebe30c20f3bebce8d58403a3a78d1aa3542d Mon Sep 17 00:00:00 2001 From: Cindy Xie Date: Wed, 16 Jul 2025 14:15:09 +0100 Subject: [PATCH 2/4] formatting --- models/table.rb | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/models/table.rb b/models/table.rb index 6d668dd..645e42e 100644 --- a/models/table.rb +++ b/models/table.rb @@ -116,13 +116,13 @@ def copy end def log_row_drift - source_count = source_connection.exec_query("SELECT COUNT(*) AS count FROM #{source_name}").first["count"].to_i - dest_count = destination_connection.exec_query("SELECT COUNT(*) AS count FROM #{destination_name}").first["count"].to_i - drift = source_count - dest_count - logger.info "Drift for #{source_name}: source=#{source_count}, destination=#{dest_count}, drift=#{drift}" - rescue => e - logger.error "Error calculating drift for #{source_name}: #{e.message}" - end + source_count = source_connection.exec_query("SELECT COUNT(*) AS count FROM #{source_name}").first["count"].to_i + dest_count = destination_connection.exec_query("SELECT COUNT(*) AS count FROM #{destination_name}").first["count"].to_i + drift = source_count - dest_count + logger.info "Drift for #{source_name}: source=#{source_count}, destination=#{dest_count}, drift=#{drift}" + rescue => e + logger.error "Error calculating drift for #{source_name}: #{e.message}" + end def copy_now started_at = Time.now From f66d27f6dfb3fe64cbc4388dafff555877171b6e Mon Sep 17 00:00:00 2001 From: Cindy Xie Date: Wed, 16 Jul 2025 14:19:10 +0100 Subject: [PATCH 3/4] using .execute instead --- models/table.rb | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/models/table.rb b/models/table.rb index 645e42e..9caf57e 100644 --- a/models/table.rb +++ b/models/table.rb @@ -116,9 +116,16 @@ def copy end def log_row_drift - source_count = source_connection.exec_query("SELECT COUNT(*) AS count FROM #{source_name}").first["count"].to_i - dest_count = destination_connection.exec_query("SELECT COUNT(*) AS count FROM #{destination_name}").first["count"].to_i + source_sql = "SELECT COUNT(*) AS count FROM #{source_name}" + dest_sql = "SELECT COUNT(*) AS count FROM #{destination_name}" + + source_result = source_connection.execute(source_sql) + dest_result = destination_connection.execute(dest_sql) + + source_count = source_result.first['count'].to_i + dest_count = dest_result.first['count'].to_i drift = source_count - dest_count + logger.info "Drift for #{source_name}: source=#{source_count}, destination=#{dest_count}, drift=#{drift}" rescue => e logger.error "Error calculating drift for #{source_name}: #{e.message}" From a8506e887a64f954cd8e448fd1ad85fb2dbe0534 Mon Sep 17 00:00:00 2001 From: Cindy Xie Date: Wed, 16 Jul 2025 14:20:52 +0100 Subject: [PATCH 4/4] changing when to call log_row_drift --- models/table.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/table.rb b/models/table.rb index 9caf57e..589d346 100644 --- a/models/table.rb +++ b/models/table.rb @@ -159,8 +159,8 @@ def copy_now merge_results(temp_table_name, merge_to_table_name) update_max_values(temp_table_name) - destination_connection.execute("DROP TABLE #{temp_table_name};") log_row_drift + destination_connection.execute("DROP TABLE #{temp_table_name};") end #Log for benchmarking