From 2d0c0a4a0e0d1f661a57e8cb4f6c010bed910bbd Mon Sep 17 00:00:00 2001 From: Cameron Garnham Date: Fri, 6 Feb 2026 12:16:57 +0100 Subject: [PATCH 1/3] feat: enforce non-zero completed count in torrents table Add CHECK constraint to ensure the `completed` column in the `torrents` table is always >= 1, preventing invalid zero values. This change reflects the semantic meaning that a torrent entry should only exist once it has been completed at least once. Changes: - Add migration scripts for both SQLite and MySQL to: - Delete any existing rows with completed = 0 - Add CHECK constraint (completed >= 1) - Change default value from 0 to 1 - Update hardcoded table creation SQL in database drivers to include the new constraint and default value - Document the new constraint in migrations README with complete table schema reference for all 4 tracker tables - Update module documentation to clarify permanent keys support The migration handles SQLite's limitation by recreating the table, while MySQL can add the constraint directly (requires MySQL 8.0.16+). --- packages/tracker-core/migrations/README.md | 63 +++++++++++++++++++ ...st_tracker_torrents_completed_non_zero.sql | 9 +++ ...st_tracker_torrents_completed_non_zero.sql | 16 +++++ .../src/databases/driver/mysql.rs | 2 +- .../src/databases/driver/sqlite.rs | 2 +- packages/tracker-core/src/databases/mod.rs | 17 ++++- 6 files changed, 105 insertions(+), 4 deletions(-) create mode 100644 packages/tracker-core/migrations/mysql/20260206120000_torrust_tracker_torrents_completed_non_zero.sql create mode 100644 packages/tracker-core/migrations/sqlite/20260206120000_torrust_tracker_torrents_completed_non_zero.sql diff --git a/packages/tracker-core/migrations/README.md b/packages/tracker-core/migrations/README.md index 090c46ccb..0682ea9c3 100644 --- a/packages/tracker-core/migrations/README.md +++ b/packages/tracker-core/migrations/README.md @@ -3,3 +3,66 @@ We don't support automatic migrations yet. The tracker creates all the needed tables when it starts. The SQL sentences are hardcoded in each database driver. The migrations in this folder were introduced to add some new changes (permanent keys) and to allow users to migrate to the new version. In the future, we will remove the hardcoded SQL and start using a Rust crate for database migrations. For the time being, if you are using the initial schema described in the migration `20240730183000_torrust_tracker_create_all_tables.sql` you will need to run all the subsequent migrations manually. + +## Database Tables + +The tracker uses 4 tables: + +### 1. `whitelist` + +Stores whitelisted torrent infohashes for private/whitelisted mode. + +| Column | SQLite Type | MySQL Type | Description | +|--------|-------------|------------|-------------| +| `id` | INTEGER PRIMARY KEY AUTOINCREMENT | integer PRIMARY KEY AUTO_INCREMENT | Auto-increment ID | +| `info_hash` | TEXT NOT NULL UNIQUE | VARCHAR(40) NOT NULL UNIQUE | BitTorrent V1 infohash (40-char hex string) | + +### 2. `torrents` + +Stores per-torrent metrics (completed download count). + +| Column | SQLite Type | MySQL Type | Description | +|--------|-------------|------------|-------------| +| `id` | INTEGER PRIMARY KEY AUTOINCREMENT | integer PRIMARY KEY AUTO_INCREMENT | Auto-increment ID | +| `info_hash` | TEXT NOT NULL UNIQUE | VARCHAR(40) NOT NULL UNIQUE | BitTorrent V1 infohash (40-char hex string) | +| `completed` | INTEGER DEFAULT 1 NOT NULL CHECK (completed >= 1) | INTEGER DEFAULT 1 NOT NULL CHECK (completed >= 1) | Number of times the torrent has been fully downloaded (minimum 1) | + +### 3. `keys` + +Stores authentication keys for private trackers. + +| Column | SQLite Type | MySQL Type | Description | +|--------|-------------|------------|-------------| +| `id` | INTEGER PRIMARY KEY AUTOINCREMENT | INT NOT NULL AUTO_INCREMENT | Auto-increment ID | +| `key` | TEXT NOT NULL UNIQUE | VARCHAR(32) NOT NULL UNIQUE | Authentication token (32-char alphanumeric string) | +| `valid_until` | INTEGER (nullable) | INT(10) (nullable) | Unix timestamp for key expiration; NULL means permanent key | + +### 4. `torrent_aggregate_metrics` + +Stores global/aggregate metrics not tied to specific torrents (e.g., total downloads across all torrents). + +| Column | SQLite Type | MySQL Type | Description | +|--------|-------------|------------|-------------| +| `id` | INTEGER PRIMARY KEY AUTOINCREMENT | integer PRIMARY KEY AUTO_INCREMENT | Auto-increment ID | +| `metric_name` | TEXT NOT NULL UNIQUE | VARCHAR(50) NOT NULL UNIQUE | Unique metric identifier (e.g., `torrents_downloads_total`) | +| `value` | INTEGER DEFAULT 0 NOT NULL | INTEGER DEFAULT 0 NOT NULL | The metric value | + +## Migration Files + +### SQLite + +| Migration | Description | +|-----------|-------------| +| `20240730183000_torrust_tracker_create_all_tables.sql` | Creates initial tables: `whitelist`, `torrents`, `keys` | +| `20240730183500_torrust_tracker_keys_valid_until_nullable.sql` | Makes `valid_until` column nullable in `keys` table (for permanent keys) | +| `20250527093000_torrust_tracker_new_torrent_aggregate_metrics_table.sql` | Creates `torrent_aggregate_metrics` table for global metrics | +| `20260206120000_torrust_tracker_torrents_completed_non_zero.sql` | Removes rows with completed=0 and adds CHECK constraint (completed >= 1) | + +### MySQL + +| Migration | Description | +|-----------|-------------| +| `20240730183000_torrust_tracker_create_all_tables.sql` | Creates initial tables: `whitelist`, `torrents`, `keys` | +| `20240730183500_torrust_tracker_keys_valid_until_nullable.sql` | Makes `valid_until` column nullable in `keys` table (for permanent keys) | +| `20250527093000_torrust_tracker_new_torrent_aggregate_metrics_table.sql` | Creates `torrent_aggregate_metrics` table for global metrics | +| `20260206120000_torrust_tracker_torrents_completed_non_zero.sql` | Removes rows with completed=0 and adds CHECK constraint (completed >= 1) | diff --git a/packages/tracker-core/migrations/mysql/20260206120000_torrust_tracker_torrents_completed_non_zero.sql b/packages/tracker-core/migrations/mysql/20260206120000_torrust_tracker_torrents_completed_non_zero.sql new file mode 100644 index 000000000..498a54e45 --- /dev/null +++ b/packages/tracker-core/migrations/mysql/20260206120000_torrust_tracker_torrents_completed_non_zero.sql @@ -0,0 +1,9 @@ +-- Remove any rows with completed = 0 (should not exist in normal operation) +DELETE FROM torrents WHERE completed = 0; + +-- Add CHECK constraint to enforce completed >= 1 +-- Note: MySQL 8.0.16+ enforces CHECK constraints +ALTER TABLE torrents ADD CONSTRAINT chk_completed_non_zero CHECK (completed >= 1); + +-- Change the default value from 0 to 1 +ALTER TABLE torrents ALTER completed SET DEFAULT 1; diff --git a/packages/tracker-core/migrations/sqlite/20260206120000_torrust_tracker_torrents_completed_non_zero.sql b/packages/tracker-core/migrations/sqlite/20260206120000_torrust_tracker_torrents_completed_non_zero.sql new file mode 100644 index 000000000..13ef11bb3 --- /dev/null +++ b/packages/tracker-core/migrations/sqlite/20260206120000_torrust_tracker_torrents_completed_non_zero.sql @@ -0,0 +1,16 @@ +-- Remove any rows with completed = 0 (should not exist in normal operation) +DELETE FROM torrents WHERE completed = 0; + +-- SQLite doesn't support adding CHECK constraints to existing tables directly. +-- We need to recreate the table with the new constraint. +CREATE TABLE IF NOT EXISTS torrents_new ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + info_hash TEXT NOT NULL UNIQUE, + completed INTEGER DEFAULT 1 NOT NULL CHECK (completed >= 1) +); + +INSERT INTO torrents_new (id, info_hash, completed) SELECT id, info_hash, completed FROM torrents; + +DROP TABLE torrents; + +ALTER TABLE torrents_new RENAME TO torrents; diff --git a/packages/tracker-core/src/databases/driver/mysql.rs b/packages/tracker-core/src/databases/driver/mysql.rs index da2f86ce8..92eac520c 100644 --- a/packages/tracker-core/src/databases/driver/mysql.rs +++ b/packages/tracker-core/src/databases/driver/mysql.rs @@ -83,7 +83,7 @@ impl Database for Mysql { CREATE TABLE IF NOT EXISTS torrents ( id integer PRIMARY KEY AUTO_INCREMENT, info_hash VARCHAR(40) NOT NULL UNIQUE, - completed INTEGER DEFAULT 0 NOT NULL + completed INTEGER DEFAULT 1 NOT NULL CHECK (completed >= 1) );" .to_string(); diff --git a/packages/tracker-core/src/databases/driver/sqlite.rs b/packages/tracker-core/src/databases/driver/sqlite.rs index d08351aa8..4142ae4b1 100644 --- a/packages/tracker-core/src/databases/driver/sqlite.rs +++ b/packages/tracker-core/src/databases/driver/sqlite.rs @@ -98,7 +98,7 @@ impl Database for Sqlite { CREATE TABLE IF NOT EXISTS torrents ( id INTEGER PRIMARY KEY AUTOINCREMENT, info_hash TEXT NOT NULL UNIQUE, - completed INTEGER DEFAULT 0 NOT NULL + completed INTEGER DEFAULT 1 NOT NULL CHECK (completed >= 1) );" .to_string(); diff --git a/packages/tracker-core/src/databases/mod.rs b/packages/tracker-core/src/databases/mod.rs index c9d89769a..90ef3ae3c 100644 --- a/packages/tracker-core/src/databases/mod.rs +++ b/packages/tracker-core/src/databases/mod.rs @@ -43,9 +43,22 @@ //! |---------------|------------------------------------|--------------------------------------| //! | `id` | 1 | Auto-increment id | //! | `key` | `IrweYtVuQPGbG9Jzx1DihcPmJGGpVy82` | Authentication token (32 chars) | -//! | `valid_until` | 1672419840 | Timestamp indicating expiration time | +//! | `valid_until` | 1672419840 | Unix timestamp for expiration; NULL for permanent keys | //! -//! > **NOTICE**: All authentication keys must have an expiration date. +//! > **NOTICE**: Authentication keys can be permanent (no expiration) by setting +//! > `valid_until` to NULL. +//! +//! # Torrent Aggregate Metrics +//! +//! A table for storing global/aggregate metrics that are not tied to a specific +//! torrent. Currently used for tracking the total number of downloads across +//! all torrents. +//! +//! | Field | Sample data | Description | +//! |---------------|------------------------------------------|------------------------------------------| +//! | `id` | 1 | Auto-increment id | +//! | `metric_name` | `torrents_downloads_total` | Unique identifier for the metric | +//! | `value` | 12345 | The metric value (integer) | pub mod driver; pub mod error; pub mod setup; From f935f54554e07978d533e704a95b5d2db9c81ed6 Mon Sep 17 00:00:00 2001 From: Cameron Garnham Date: Fri, 6 Feb 2026 13:05:06 +0100 Subject: [PATCH 2/3] refactor: optimize whitelist table schema for performance Migrate the `whitelist` table to `whitelist_v1` with optimized storage and indexing. This change removes the unnecessary auto-increment `id` column and uses `info_hash` directly as the PRIMARY KEY, improving both storage efficiency and query performance. Database-specific optimizations: - **SQLite**: Use `WITHOUT ROWID` optimization since info_hash is already a suitable primary key (40-char hex string). This eliminates the hidden rowid column and reduces storage overhead. - **MySQL**: Store info_hash as `BINARY(20)` instead of `VARCHAR(40)`, reducing storage from 40 bytes to 20 bytes per entry and improving index performance. Changes: - Add migration scripts for both SQLite and MySQL to: - Create new `whitelist_v1` table with optimized schema - Migrate existing data (converting hex to binary for MySQL) - Drop old `whitelist` table - Update hardcoded table creation SQL in database drivers to match new schema - Update all whitelist queries to use `whitelist_v1` table name - Update MySQL driver to use binary storage with `UNHEX()`/bytes() for info_hash operations - Document the new schema in migrations README with storage format details for both databases The migration preserves all existing whitelist data while improving storage efficiency and query performance for whitelist lookups. --- Cargo.lock | 243 ++++++++++-------- packages/tracker-core/migrations/README.md | 9 +- ...30000_torrust_tracker_whitelist_binary.sql | 11 + ...orrust_tracker_whitelist_without_rowid.sql | 8 + .../src/databases/driver/mysql.rs | 34 +-- .../src/databases/driver/sqlite.rs | 19 +- 6 files changed, 181 insertions(+), 143 deletions(-) create mode 100644 packages/tracker-core/migrations/mysql/20260206130000_torrust_tracker_whitelist_binary.sql create mode 100644 packages/tracker-core/migrations/sqlite/20260206130000_torrust_tracker_whitelist_without_rowid.sql diff --git a/Cargo.lock b/Cargo.lock index 8916a6640..bcebf3db1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -134,9 +134,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.100" +version = "1.0.101" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a23eb6b1614318a8071c9b2521f36b424b2c83db5eb3a0fead4a6c0809af6e61" +checksum = "5f0e0fee31ef5ed1ba1316088939cea399010ed7731dba877ed44aeb407a75ea" [[package]] name = "approx" @@ -175,9 +175,9 @@ dependencies = [ [[package]] name = "arc-swap" -version = "1.8.0" +version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51d03449bb8ca2cc2ef70869af31463d1ae5ccc8fa3e334b307203fbf815207e" +checksum = "9ded5f9a03ac8f24d1b8a25101ee812cd32cdc8c50a4c50237de2c4915850e73" dependencies = [ "rustversion", ] @@ -751,7 +751,7 @@ dependencies = [ "bittorrent-udp-tracker-protocol", "bloom", "blowfish", - "cipher", + "cipher 0.5.0", "criterion 0.5.1", "futures", "lazy_static", @@ -831,14 +831,14 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e412e2cd0f2b2d93e02543ceae7917b3c70331573df19ee046bcbc35e45e87d7" dependencies = [ "byteorder", - "cipher", + "cipher 0.4.4", ] [[package]] name = "bollard" -version = "0.19.4" +version = "0.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87a52479c9237eb04047ddb94788c41ca0d26eaff8b697ecfbb4c32f7fdc3b1b" +checksum = "227aa051deec8d16bd9c34605e7aaf153f240e35483dd42f6f78903847934738" dependencies = [ "async-stream", "base64 0.22.1", @@ -846,7 +846,6 @@ dependencies = [ "bollard-buildkit-proto", "bollard-stubs", "bytes", - "chrono", "futures-core", "futures-util", "hex", @@ -864,14 +863,13 @@ dependencies = [ "rand 0.9.2", "rustls", "rustls-native-certs", - "rustls-pemfile", "rustls-pki-types", "serde", "serde_derive", "serde_json", - "serde_repr", "serde_urlencoded", "thiserror 2.0.18", + "time", "tokio", "tokio-stream", "tokio-util", @@ -896,19 +894,18 @@ dependencies = [ [[package]] name = "bollard-stubs" -version = "1.49.1-rc.28.4.0" +version = "1.52.1-rc.29.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5731fe885755e92beff1950774068e0cae67ea6ec7587381536fca84f1779623" +checksum = "0f0a8ca8799131c1837d1282c3f81f31e76ceb0ce426e04a7fe1ccee3287c066" dependencies = [ "base64 0.22.1", "bollard-buildkit-proto", "bytes", - "chrono", "prost", "serde", "serde_json", "serde_repr", - "serde_with", + "time", ] [[package]] @@ -1000,9 +997,9 @@ dependencies = [ [[package]] name = "bytemuck" -version = "1.24.0" +version = "1.25.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fbdf580320f38b612e485521afda1ee26d10cc9884efaaa750d383e13e3c5f4" +checksum = "c8efb64bd706a16a1bdde310ae86b351e4d21550d98d056f22f8a7f7a2183fec" [[package]] name = "byteorder" @@ -1012,9 +1009,9 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" -version = "1.11.0" +version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b35204fbdc0b3f4446b89fc1ac2cf84a8a68971995d0bf2e925ec7cd960f9cb3" +checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" [[package]] name = "camino" @@ -1042,9 +1039,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.54" +version = "1.2.55" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6354c81bbfd62d9cfa9cb3c773c2b7b2a3a482d569de977fd0e961f6e7c00583" +checksum = "47b26a0954ae34af09b50f0de26458fa95369a0d478d8236d3f93082b219bd29" dependencies = [ "find-msvc-tools", "jobserver", @@ -1124,8 +1121,18 @@ version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" dependencies = [ - "crypto-common", - "inout", + "crypto-common 0.1.7", + "inout 0.1.4", +] + +[[package]] +name = "cipher" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64727038c8c5e2bb503a15b9f5b9df50a1da9a33e83e1f93067d914f2c6604a5" +dependencies = [ + "crypto-common 0.2.0", + "inout 0.2.2", ] [[package]] @@ -1141,9 +1148,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.54" +version = "4.5.57" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6e6ff9dcd79cff5cd969a17a545d79e84ab086e444102a591e288a8aa3ce394" +checksum = "6899ea499e3fb9305a65d5ebf6e3d2248c5fab291f300ad0a704fbe142eae31a" dependencies = [ "clap_builder", "clap_derive", @@ -1151,9 +1158,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.54" +version = "4.5.57" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa42cf4d2b7a41bc8f663a7cab4031ebafa1bf3875705bfaf8466dc60ab52c00" +checksum = "7b12c8b680195a62a8364d16b8447b01b6c2c8f9aaf68bee653be34d4245e238" dependencies = [ "anstream", "anstyle", @@ -1163,9 +1170,9 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.5.49" +version = "4.5.55" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a0b5487afeab2deb2ff4e03a807ad1a03ac532ff5a2cee5d86884440c7f7671" +checksum = "a92793da1a46a5f2a02a6f4c46c6496b28c43638adea8306fcb0caa1634f24e5" dependencies = [ "heck", "proc-macro2", @@ -1329,16 +1336,16 @@ dependencies = [ [[package]] name = "criterion" -version = "0.8.1" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d883447757bb0ee46f233e9dc22eb84d93a9508c9b868687b274fc431d886bf" +checksum = "950046b2aa2492f9a536f5f4f9a3de7b9e2476e575e05bd6c333371add4d98f3" dependencies = [ "alloca", "anes", "cast", "ciborium", "clap", - "criterion-plot 0.8.1", + "criterion-plot 0.8.2", "itertools 0.13.0", "num-traits", "oorandom", @@ -1365,9 +1372,9 @@ dependencies = [ [[package]] name = "criterion-plot" -version = "0.8.1" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed943f81ea2faa8dcecbbfa50164acf95d555afec96a27871663b300e387b2e4" +checksum = "d8d80a2f4f5b554395e47b5d8305bc3d27813bacb73493eb1001e8f76dae29ea" dependencies = [ "cast", "itertools 0.13.0", @@ -1455,6 +1462,15 @@ dependencies = [ "typenum", ] +[[package]] +name = "crypto-common" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "211f05e03c7d03754740fd9e585de910a095d6b99f8bcfffdef8319fa02a8331" +dependencies = [ + "hybrid-array", +] + [[package]] name = "darling" version = "0.20.11" @@ -1627,7 +1643,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer", - "crypto-common", + "crypto-common 0.1.7", ] [[package]] @@ -1806,15 +1822,15 @@ dependencies = [ [[package]] name = "find-msvc-tools" -version = "0.1.8" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8591b0bcc8a98a64310a2fae1bb3e9b8564dd10e381e6e28010fde8e8e8568db" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" [[package]] name = "flate2" -version = "1.1.8" +version = "1.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b375d6465b98090a5f25b1c7703f3859783755aa9a80433b36e0379a3ec2f369" +checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" dependencies = [ "crc32fast", "libz-sys", @@ -2175,7 +2191,7 @@ checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" dependencies = [ "cfg-if", "crunchy", - "zerocopy 0.8.34", + "zerocopy 0.8.39", ] [[package]] @@ -2300,6 +2316,15 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" +[[package]] +name = "hybrid-array" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1b229d73f5803b562cc26e4da0396c8610a4ee209f4fac8fa4f8d709166dc45" +dependencies = [ + "typenum", +] + [[package]] name = "hyper" version = "1.8.1" @@ -2369,14 +2394,13 @@ dependencies = [ [[package]] name = "hyper-util" -version = "0.1.19" +version = "0.1.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "727805d60e7938b76b826a6ef209eb70eaa1812794f9424d4a4e2d740662df5f" +checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" dependencies = [ "base64 0.22.1", "bytes", "futures-channel", - "futures-core", "futures-util", "http", "http-body", @@ -2410,9 +2434,9 @@ dependencies = [ [[package]] name = "iana-time-zone" -version = "0.1.64" +version = "0.1.65" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33e57f83510bb73707521ebaffa789ec8caf86f9657cad665b092b581d40e9fb" +checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" dependencies = [ "android_system_properties", "core-foundation-sys", @@ -2578,6 +2602,15 @@ dependencies = [ "generic-array", ] +[[package]] +name = "inout" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4250ce6452e92010fdf7268ccc5d14faa80bb12fc741938534c58f16804e03c7" +dependencies = [ + "hybrid-array", +] + [[package]] name = "io-enum" version = "1.2.0" @@ -2785,9 +2818,9 @@ checksum = "6373607a59f0be73a39b6fe456b8192fcc3585f602af20751600e974dd455e77" [[package]] name = "local-ip-address" -version = "0.6.9" +version = "0.6.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92488bc8a0f99ee9f23577bdd06526d49657df8bd70504c61f812337cdad01ab" +checksum = "79ef8c257c92ade496781a32a581d43e3d512cf8ce714ecf04ea80f93ed0ff4a" dependencies = [ "libc", "neli", @@ -3047,9 +3080,9 @@ dependencies = [ [[package]] name = "neli" -version = "0.7.3" +version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e23bebbf3e157c402c4d5ee113233e5e0610cc27453b2f07eefce649c7365dcc" +checksum = "22f9786d56d972959e1408b6a93be6af13b9c1392036c5c1fafa08a1b0c6ee87" dependencies = [ "bitflags", "byteorder", @@ -3495,15 +3528,15 @@ dependencies = [ [[package]] name = "portable-atomic" -version = "1.13.0" +version = "1.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f89776e4d69bb58bc6993e99ffa1d11f228b839984854c7daeb5d37f87cbe950" +checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49" [[package]] name = "portable-atomic-util" -version = "0.2.4" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8a2f0d8d040d7848a709caf78912debcc3f33ee4b3cac47d73d1e1069e83507" +checksum = "7a9db96d7fa8782dd8c15ce32ffe8680bbd1e978a43bf51a34d39483540495f5" dependencies = [ "portable-atomic", ] @@ -3529,7 +3562,7 @@ version = "0.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" dependencies = [ - "zerocopy 0.8.34", + "zerocopy 0.8.39", ] [[package]] @@ -3912,9 +3945,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.12.2" +version = "1.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "843bc0191f75f3e22651ae5f1e72939ab2f72a4bc30fa80a066bd66edefc24d4" +checksum = "e10754a14b9137dd7b1e3e5b0493cc9171fdd105e0ab477f51b72e7f3ac0e276" dependencies = [ "aho-corasick", "memchr", @@ -3924,9 +3957,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.13" +version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5276caf25ac86c8d810222b3dbb938e512c55c6831a10f3e6ed1c93b84041f1c" +checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" dependencies = [ "aho-corasick", "memchr", @@ -3935,9 +3968,9 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.8.8" +version = "0.8.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a2d987857b319362043e95f5353c0535c1f58eec5336fdfcf626430af7def58" +checksum = "a96887878f22d7bad8a3b6dc5b7440e0ada9a245242924394987b21cf2210a4c" [[package]] name = "relative-path" @@ -4211,15 +4244,6 @@ dependencies = [ "security-framework 3.5.1", ] -[[package]] -name = "rustls-pemfile" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dce314e5fee3f39953d46bb63bb8a46d40c2f8fb7cc5a3b6cab2bde9721d6e50" -dependencies = [ - "rustls-pki-types", -] - [[package]] name = "rustls-pki-types" version = "1.14.0" @@ -4328,9 +4352,9 @@ dependencies = [ [[package]] name = "schemars" -version = "1.2.0" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54e910108742c57a770f492731f99be216a52fadd361b06c8fb59d74ccc267d2" +checksum = "a2b42f36aa1cd011945615b92222f6bf73c599a102a300334cd7f8dbeec726cc" dependencies = [ "dyn-clone", "ref-cast", @@ -4533,7 +4557,7 @@ dependencies = [ "indexmap 1.9.3", "indexmap 2.13.0", "schemars 0.9.0", - "schemars 1.2.0", + "schemars 1.2.1", "serde_core", "serde_json", "serde_with_macros", @@ -4613,15 +4637,15 @@ checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" [[package]] name = "siphasher" -version = "1.0.1" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56199f7ddabf13fe5074ce809e7d3f42b42ae711800501b5b16ea82ad029c39d" +checksum = "b2aa850e253778c88a04c3d7323b043aeda9d3e30d5971937c1855769763678e" [[package]] name = "slab" -version = "0.4.11" +version = "0.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a2ae44ef20feb57a68b23d846850f861394c2e02dc425a50098ae8c90267589" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" [[package]] name = "smallvec" @@ -4704,9 +4728,9 @@ dependencies = [ [[package]] name = "subprocess" -version = "0.2.13" +version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f75238edb5be30a9ea3035b945eb9c319dde80e879411cdc9a8978e1ac822960" +checksum = "2c56e8662b206b9892d7a5a3f2ecdbcb455d3d6b259111373b7e08b8055158a8" dependencies = [ "libc", "winapi", @@ -4783,9 +4807,9 @@ dependencies = [ [[package]] name = "system-configuration" -version = "0.6.1" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c879d448e9d986b661742763247d3693ed13609438cf3d006f51f5368a5ba6b" +checksum = "a13f3d0daba03132c0aa9767f98351b3488edc2c100cda2d2ec2b04f3d8d3c8b" dependencies = [ "bitflags", "core-foundation 0.9.4", @@ -4865,9 +4889,9 @@ checksum = "8f50febec83f5ee1df3015341d8bd429f2d1cc62bcba7ea2076759d315084683" [[package]] name = "testcontainers" -version = "0.26.3" +version = "0.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a81ec0158db5fbb9831e09d1813fe5ea9023a2b5e6e8e0a5fe67e2a820733629" +checksum = "c3fdcea723c64cc08dbc533b3761e345a15bf1222cbe6cb611de09b43f17a168" dependencies = [ "astral-tokio-tar", "async-trait", @@ -4878,6 +4902,7 @@ dependencies = [ "etcetera", "ferroid", "futures", + "http", "itertools 0.14.0", "log", "memchr", @@ -4954,9 +4979,9 @@ dependencies = [ [[package]] name = "time" -version = "0.3.46" +version = "0.3.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9da98b7d9b7dad93488a84b8248efc35352b0b2657397d4167e7ad67e5d535e5" +checksum = "743bd48c283afc0388f9b8827b976905fb217ad9e647fae3a379a9283c4def2c" dependencies = [ "deranged", "itoa", @@ -4975,9 +5000,9 @@ checksum = "7694e1cfe791f8d31026952abf09c69ca6f6fa4e1a1229e18988f06a04a12dca" [[package]] name = "time-macros" -version = "0.2.26" +version = "0.2.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78cc610bac2dcee56805c99642447d4c5dbde4d01f752ffea0199aee1f601dc4" +checksum = "2e70e4c5a0e0a8a4823ad65dfe1a6930e4f4d756dcd9dd7939022b5e8c501215" dependencies = [ "num-conv", "time-core", @@ -5173,9 +5198,9 @@ checksum = "ab16f14aed21ee8bfd8ec22513f7287cd4a91aa92e44edfe2c17ddd004e92607" [[package]] name = "tonic" -version = "0.14.2" +version = "0.14.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb7613188ce9f7df5bfe185db26c5814347d110db17920415cf2fbcad85e7203" +checksum = "a286e33f82f8a1ee2df63f4fa35c0becf4a85a0cb03091a15fd7bf0b402dc94a" dependencies = [ "async-trait", "axum", @@ -5202,9 +5227,9 @@ dependencies = [ [[package]] name = "tonic-prost" -version = "0.14.2" +version = "0.14.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "66bd50ad6ce1252d87ef024b3d64fe4c3cf54a86fb9ef4c631fdd0ded7aeaa67" +checksum = "d6c55a2d6a14174563de34409c9f92ff981d006f56da9c6ecd40d9d4a31500b0" dependencies = [ "bytes", "prost", @@ -5479,7 +5504,7 @@ dependencies = [ name = "torrust-tracker-contrib-bencode" version = "3.0.0-develop" dependencies = [ - "criterion 0.8.1", + "criterion 0.8.2", "thiserror 2.0.18", ] @@ -5543,7 +5568,7 @@ dependencies = [ "async-std", "bittorrent-primitives", "chrono", - "criterion 0.8.1", + "criterion 0.8.2", "crossbeam-skiplist", "futures", "mockall", @@ -5579,7 +5604,7 @@ dependencies = [ "aquatic_udp_protocol", "async-std", "bittorrent-primitives", - "criterion 0.8.1", + "criterion 0.8.2", "crossbeam-skiplist", "dashmap", "futures", @@ -5828,9 +5853,9 @@ checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" [[package]] name = "ureq" -version = "3.1.4" +version = "3.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d39cb1dbab692d82a977c0392ffac19e188bd9186a9f32806f0aaa859d75585a" +checksum = "fdc97a28575b85cfedf2a7e7d3cc64b3e11bd8ac766666318003abbacc7a21fc" dependencies = [ "base64 0.22.1", "log", @@ -5839,7 +5864,6 @@ dependencies = [ "rustls-pki-types", "ureq-proto", "utf-8", - "webpki-roots", ] [[package]] @@ -6036,18 +6060,9 @@ dependencies = [ [[package]] name = "webpki-root-certs" -version = "1.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36a29fc0408b113f68cf32637857ab740edfafdf460c326cd2afaa2d84cc05dc" -dependencies = [ - "rustls-pki-types", -] - -[[package]] -name = "webpki-roots" -version = "1.0.5" +version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12bed680863276c63889429bfd6cab3b99943659923822de1c8a39c49e4d722c" +checksum = "804f18a4ac2676ffb4e8b5b5fa9ae38af06df08162314f96a68d2a363e21a8ca" dependencies = [ "rustls-pki-types", ] @@ -6456,11 +6471,11 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.8.34" +version = "0.8.39" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "71ddd76bcebeed25db614f82bf31a9f4222d3fbba300e6fb6c00afa26cbd4d9d" +checksum = "db6d35d663eadb6c932438e763b262fe1a70987f9ae936e60158176d710cae4a" dependencies = [ - "zerocopy-derive 0.8.34", + "zerocopy-derive 0.8.39", ] [[package]] @@ -6476,9 +6491,9 @@ dependencies = [ [[package]] name = "zerocopy-derive" -version = "0.8.34" +version = "0.8.39" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8187381b52e32220d50b255276aa16a084ec0a9017a0ca2152a1f55c539758d" +checksum = "4122cd3169e94605190e77839c9a40d40ed048d305bfdc146e7df40ab0f3e517" dependencies = [ "proc-macro2", "quote", @@ -6547,9 +6562,9 @@ dependencies = [ [[package]] name = "zmij" -version = "1.0.17" +version = "1.0.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02aae0f83f69aafc94776e879363e9771d7ecbffe2c7fbb6c14c5e00dfe88439" +checksum = "3ff05f8caa9038894637571ae6b9e29466c1f4f829d26c9b28f869a29cbe3445" [[package]] name = "zstd" diff --git a/packages/tracker-core/migrations/README.md b/packages/tracker-core/migrations/README.md index 0682ea9c3..089b16909 100644 --- a/packages/tracker-core/migrations/README.md +++ b/packages/tracker-core/migrations/README.md @@ -8,14 +8,15 @@ The migrations in this folder were introduced to add some new changes (permanent The tracker uses 4 tables: -### 1. `whitelist` +### 1. `whitelist_v1` Stores whitelisted torrent infohashes for private/whitelisted mode. | Column | SQLite Type | MySQL Type | Description | |--------|-------------|------------|-------------| -| `id` | INTEGER PRIMARY KEY AUTOINCREMENT | integer PRIMARY KEY AUTO_INCREMENT | Auto-increment ID | -| `info_hash` | TEXT NOT NULL UNIQUE | VARCHAR(40) NOT NULL UNIQUE | BitTorrent V1 infohash (40-char hex string) | +| `info_hash` | TEXT PRIMARY KEY NOT NULL | BINARY(20) PRIMARY KEY NOT NULL | BitTorrent V1 infohash (SQLite: 40-char hex string, MySQL: 20-byte binary) | + +> **Note**: SQLite uses `WITHOUT ROWID` optimization. MySQL stores as binary for efficiency. ### 2. `torrents` @@ -57,6 +58,7 @@ Stores global/aggregate metrics not tied to specific torrents (e.g., total downl | `20240730183500_torrust_tracker_keys_valid_until_nullable.sql` | Makes `valid_until` column nullable in `keys` table (for permanent keys) | | `20250527093000_torrust_tracker_new_torrent_aggregate_metrics_table.sql` | Creates `torrent_aggregate_metrics` table for global metrics | | `20260206120000_torrust_tracker_torrents_completed_non_zero.sql` | Removes rows with completed=0 and adds CHECK constraint (completed >= 1) | +| `20260206130000_torrust_tracker_whitelist_without_rowid.sql` | Optimizes whitelist table: removes `id` column, uses `info_hash` as PRIMARY KEY with WITHOUT ROWID | ### MySQL @@ -66,3 +68,4 @@ Stores global/aggregate metrics not tied to specific torrents (e.g., total downl | `20240730183500_torrust_tracker_keys_valid_until_nullable.sql` | Makes `valid_until` column nullable in `keys` table (for permanent keys) | | `20250527093000_torrust_tracker_new_torrent_aggregate_metrics_table.sql` | Creates `torrent_aggregate_metrics` table for global metrics | | `20260206120000_torrust_tracker_torrents_completed_non_zero.sql` | Removes rows with completed=0 and adds CHECK constraint (completed >= 1) | +| `20260206130000_torrust_tracker_whitelist_binary.sql` | Optimizes whitelist table: removes `id` column, uses BINARY(20) `info_hash` as PRIMARY KEY | diff --git a/packages/tracker-core/migrations/mysql/20260206130000_torrust_tracker_whitelist_binary.sql b/packages/tracker-core/migrations/mysql/20260206130000_torrust_tracker_whitelist_binary.sql new file mode 100644 index 000000000..612f3599a --- /dev/null +++ b/packages/tracker-core/migrations/mysql/20260206130000_torrust_tracker_whitelist_binary.sql @@ -0,0 +1,11 @@ +-- Migrate whitelist table to whitelist_v1 with BINARY(20) PRIMARY KEY for efficient storage +-- BINARY(20) stores the raw 20-byte infohash instead of 40-char hex string + +CREATE TABLE IF NOT EXISTS whitelist_v1 ( + info_hash BINARY(20) PRIMARY KEY NOT NULL +); + +-- Convert existing hex strings to binary +INSERT INTO whitelist_v1 (info_hash) SELECT UNHEX(info_hash) FROM whitelist; + +DROP TABLE whitelist; diff --git a/packages/tracker-core/migrations/sqlite/20260206130000_torrust_tracker_whitelist_without_rowid.sql b/packages/tracker-core/migrations/sqlite/20260206130000_torrust_tracker_whitelist_without_rowid.sql new file mode 100644 index 000000000..148084697 --- /dev/null +++ b/packages/tracker-core/migrations/sqlite/20260206130000_torrust_tracker_whitelist_without_rowid.sql @@ -0,0 +1,8 @@ +-- Migrate whitelist table to whitelist_v1 with info_hash as PRIMARY KEY and WITHOUT ROWID optimization +CREATE TABLE IF NOT EXISTS whitelist_v1 ( + info_hash TEXT PRIMARY KEY NOT NULL +) WITHOUT ROWID; + +INSERT INTO whitelist_v1 (info_hash) SELECT info_hash FROM whitelist; + +DROP TABLE whitelist; diff --git a/packages/tracker-core/src/databases/driver/mysql.rs b/packages/tracker-core/src/databases/driver/mysql.rs index 92eac520c..67c347d6a 100644 --- a/packages/tracker-core/src/databases/driver/mysql.rs +++ b/packages/tracker-core/src/databases/driver/mysql.rs @@ -73,9 +73,8 @@ impl Database for Mysql { /// Refer to [`databases::Database::create_database_tables`](crate::core::databases::Database::create_database_tables). fn create_database_tables(&self) -> Result<(), Error> { let create_whitelist_table = " - CREATE TABLE IF NOT EXISTS whitelist ( - id integer PRIMARY KEY AUTO_INCREMENT, - info_hash VARCHAR(40) NOT NULL UNIQUE + CREATE TABLE IF NOT EXISTS whitelist_v1 ( + info_hash BINARY(20) PRIMARY KEY NOT NULL );" .to_string(); @@ -123,7 +122,7 @@ impl Database for Mysql { /// Refer to [`databases::Database::drop_database_tables`](crate::core::databases::Database::drop_database_tables). fn drop_database_tables(&self) -> Result<(), Error> { let drop_whitelist_table = " - DROP TABLE `whitelist`;" + DROP TABLE `whitelist_v1`;" .to_string(); let drop_torrents_table = " @@ -137,7 +136,7 @@ impl Database for Mysql { let mut conn = self.pool.get().map_err(|e| (e, DRIVER))?; conn.query_drop(&drop_whitelist_table) - .expect("Could not drop `whitelist` table."); + .expect("Could not drop `whitelist_v1` table."); conn.query_drop(&drop_torrents_table) .expect("Could not drop `torrents` table."); conn.query_drop(&drop_keys_table).expect("Could not drop `keys` table."); @@ -248,8 +247,8 @@ impl Database for Mysql { fn load_whitelist(&self) -> Result, Error> { let mut conn = self.pool.get().map_err(|e| (e, DRIVER))?; - let info_hashes = conn.query_map("SELECT info_hash FROM whitelist", |info_hash: String| { - InfoHash::from_str(&info_hash).unwrap() + let info_hashes = conn.query_map("SELECT info_hash FROM whitelist_v1", |info_hash_bytes: Vec| { + InfoHash::from_bytes(&info_hash_bytes) })?; Ok(info_hashes) @@ -259,12 +258,12 @@ impl Database for Mysql { fn get_info_hash_from_whitelist(&self, info_hash: InfoHash) -> Result, Error> { let mut conn = self.pool.get().map_err(|e| (e, DRIVER))?; - let select = conn.exec_first::( - "SELECT info_hash FROM whitelist WHERE info_hash = :info_hash", - params! { "info_hash" => info_hash.to_hex_string() }, + let select = conn.exec_first::, _, _>( + "SELECT info_hash FROM whitelist_v1 WHERE info_hash = :info_hash", + params! { "info_hash" => info_hash.bytes() }, )?; - let info_hash = select.map(|f| InfoHash::from_str(&f).expect("Failed to decode InfoHash String from DB!")); + let info_hash = select.map(|bytes| InfoHash::from_bytes(&bytes)); Ok(info_hash) } @@ -273,11 +272,11 @@ impl Database for Mysql { fn add_info_hash_to_whitelist(&self, info_hash: InfoHash) -> Result { let mut conn = self.pool.get().map_err(|e| (e, DRIVER))?; - let info_hash_str = info_hash.to_string(); + let info_hash_bytes = info_hash.bytes(); conn.exec_drop( - "INSERT INTO whitelist (info_hash) VALUES (:info_hash_str)", - params! { info_hash_str }, + "INSERT INTO whitelist_v1 (info_hash) VALUES (:info_hash_bytes)", + params! { info_hash_bytes }, )?; Ok(1) @@ -287,9 +286,12 @@ impl Database for Mysql { fn remove_info_hash_from_whitelist(&self, info_hash: InfoHash) -> Result { let mut conn = self.pool.get().map_err(|e| (e, DRIVER))?; - let info_hash = info_hash.to_string(); + let info_hash_bytes = info_hash.bytes(); - conn.exec_drop("DELETE FROM whitelist WHERE info_hash = :info_hash", params! { info_hash })?; + conn.exec_drop( + "DELETE FROM whitelist_v1 WHERE info_hash = :info_hash_bytes", + params! { info_hash_bytes }, + )?; Ok(1) } diff --git a/packages/tracker-core/src/databases/driver/sqlite.rs b/packages/tracker-core/src/databases/driver/sqlite.rs index 4142ae4b1..7681adf4e 100644 --- a/packages/tracker-core/src/databases/driver/sqlite.rs +++ b/packages/tracker-core/src/databases/driver/sqlite.rs @@ -88,11 +88,10 @@ impl Database for Sqlite { /// Refer to [`databases::Database::create_database_tables`](crate::core::databases::Database::create_database_tables). fn create_database_tables(&self) -> Result<(), Error> { let create_whitelist_table = " - CREATE TABLE IF NOT EXISTS whitelist ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - info_hash TEXT NOT NULL UNIQUE - );" - .to_string(); + CREATE TABLE IF NOT EXISTS whitelist_v1 ( + info_hash TEXT PRIMARY KEY NOT NULL + ) WITHOUT ROWID;" + .to_string(); let create_torrents_table = " CREATE TABLE IF NOT EXISTS torrents ( @@ -131,7 +130,7 @@ impl Database for Sqlite { /// Refer to [`databases::Database::drop_database_tables`](crate::core::databases::Database::drop_database_tables). fn drop_database_tables(&self) -> Result<(), Error> { let drop_whitelist_table = " - DROP TABLE whitelist;" + DROP TABLE whitelist_v1;" .to_string(); let drop_torrents_table = " @@ -269,7 +268,7 @@ impl Database for Sqlite { fn load_whitelist(&self) -> Result, Error> { let conn = self.pool.get().map_err(|e| (e, DRIVER))?; - let mut stmt = conn.prepare("SELECT info_hash FROM whitelist")?; + let mut stmt = conn.prepare("SELECT info_hash FROM whitelist_v1")?; let info_hash_iter = stmt.query_map([], |row| { let info_hash: String = row.get(0)?; @@ -286,7 +285,7 @@ impl Database for Sqlite { fn get_info_hash_from_whitelist(&self, info_hash: InfoHash) -> Result, Error> { let conn = self.pool.get().map_err(|e| (e, DRIVER))?; - let mut stmt = conn.prepare("SELECT info_hash FROM whitelist WHERE info_hash = ?")?; + let mut stmt = conn.prepare("SELECT info_hash FROM whitelist_v1 WHERE info_hash = ?")?; let mut rows = stmt.query([info_hash.to_hex_string()])?; @@ -299,7 +298,7 @@ impl Database for Sqlite { fn add_info_hash_to_whitelist(&self, info_hash: InfoHash) -> Result { let conn = self.pool.get().map_err(|e| (e, DRIVER))?; - let insert = conn.execute("INSERT INTO whitelist (info_hash) VALUES (?)", [info_hash.to_string()])?; + let insert = conn.execute("INSERT INTO whitelist_v1 (info_hash) VALUES (?)", [info_hash.to_string()])?; if insert == 0 { Err(Error::InsertFailed { @@ -315,7 +314,7 @@ impl Database for Sqlite { fn remove_info_hash_from_whitelist(&self, info_hash: InfoHash) -> Result { let conn = self.pool.get().map_err(|e| (e, DRIVER))?; - let deleted = conn.execute("DELETE FROM whitelist WHERE info_hash = ?", [info_hash.to_string()])?; + let deleted = conn.execute("DELETE FROM whitelist_v1 WHERE info_hash = ?", [info_hash.to_string()])?; if deleted == 1 { // should only remove a single record. From c5cdd26d7244a6e4bea516ece90eff83e54877ef Mon Sep 17 00:00:00 2001 From: Cameron Garnham Date: Fri, 6 Feb 2026 13:14:18 +0100 Subject: [PATCH 3/3] refactor: migrate torrents table to completed_v1 with optimized schema Rename the `torrents` table to `completed_v1` with improved storage efficiency and semantic clarity. This change removes the unnecessary auto-increment `id` column, uses `info_hash` directly as the PRIMARY KEY, and renames the `completed` column to `count` for better clarity. Database-specific optimizations: - **SQLite**: Use `WITHOUT ROWID` optimization since info_hash is already a suitable primary key (40-char hex string). This eliminates the hidden rowid column and reduces storage overhead. - **MySQL**: Store info_hash as `BINARY(20)` instead of `VARCHAR(40)`, reducing storage from 40 bytes to 20 bytes per entry and improving index performance. Changes: - Add migration scripts for both SQLite and MySQL to: - Create new `completed_v1` table with optimized schema - Migrate existing data (converting hex to binary for MySQL) - Drop old `torrents` table - Update hardcoded table creation SQL in database drivers to match new schema - Update all torrent download queries to use `completed_v1` table name and `count` column name - Update MySQL driver to use binary storage with `UNHEX()`/bytes() for info_hash operations - Document the new schema in migrations README with storage format details for both databases The migration preserves all existing torrent download data while improving storage efficiency and query performance. The new table name `completed_v1` better reflects its purpose of tracking download completion counts. --- packages/tracker-core/migrations/README.md | 13 +++-- ...206140000_torrust_tracker_completed_v1.sql | 13 +++++ ...206140000_torrust_tracker_completed_v1.sql | 10 ++++ .../src/databases/driver/mysql.rs | 47 +++++++++---------- .../src/databases/driver/sqlite.rs | 33 +++++++------ 5 files changed, 70 insertions(+), 46 deletions(-) create mode 100644 packages/tracker-core/migrations/mysql/20260206140000_torrust_tracker_completed_v1.sql create mode 100644 packages/tracker-core/migrations/sqlite/20260206140000_torrust_tracker_completed_v1.sql diff --git a/packages/tracker-core/migrations/README.md b/packages/tracker-core/migrations/README.md index 089b16909..048c150ec 100644 --- a/packages/tracker-core/migrations/README.md +++ b/packages/tracker-core/migrations/README.md @@ -18,15 +18,16 @@ Stores whitelisted torrent infohashes for private/whitelisted mode. > **Note**: SQLite uses `WITHOUT ROWID` optimization. MySQL stores as binary for efficiency. -### 2. `torrents` +### 2. `completed_v1` -Stores per-torrent metrics (completed download count). +Stores per-torrent download completion counts. | Column | SQLite Type | MySQL Type | Description | |--------|-------------|------------|-------------| -| `id` | INTEGER PRIMARY KEY AUTOINCREMENT | integer PRIMARY KEY AUTO_INCREMENT | Auto-increment ID | -| `info_hash` | TEXT NOT NULL UNIQUE | VARCHAR(40) NOT NULL UNIQUE | BitTorrent V1 infohash (40-char hex string) | -| `completed` | INTEGER DEFAULT 1 NOT NULL CHECK (completed >= 1) | INTEGER DEFAULT 1 NOT NULL CHECK (completed >= 1) | Number of times the torrent has been fully downloaded (minimum 1) | +| `info_hash` | TEXT PRIMARY KEY NOT NULL | BINARY(20) PRIMARY KEY NOT NULL | BitTorrent V1 infohash (SQLite: 40-char hex string, MySQL: 20-byte binary) | +| `count` | INTEGER DEFAULT 1 NOT NULL CHECK (count >= 1) | INTEGER DEFAULT 1 NOT NULL CHECK (count >= 1) | Number of times the torrent has been fully downloaded (minimum 1) | + +> **Note**: SQLite uses `WITHOUT ROWID` optimization. MySQL stores info_hash as binary for efficiency. ### 3. `keys` @@ -59,6 +60,7 @@ Stores global/aggregate metrics not tied to specific torrents (e.g., total downl | `20250527093000_torrust_tracker_new_torrent_aggregate_metrics_table.sql` | Creates `torrent_aggregate_metrics` table for global metrics | | `20260206120000_torrust_tracker_torrents_completed_non_zero.sql` | Removes rows with completed=0 and adds CHECK constraint (completed >= 1) | | `20260206130000_torrust_tracker_whitelist_without_rowid.sql` | Optimizes whitelist table: removes `id` column, uses `info_hash` as PRIMARY KEY with WITHOUT ROWID | +| `20260206140000_torrust_tracker_completed_v1.sql` | Migrates `torrents` to `completed_v1`: removes `id` column, renames `completed` to `count`, uses `info_hash` as PRIMARY KEY with WITHOUT ROWID | ### MySQL @@ -69,3 +71,4 @@ Stores global/aggregate metrics not tied to specific torrents (e.g., total downl | `20250527093000_torrust_tracker_new_torrent_aggregate_metrics_table.sql` | Creates `torrent_aggregate_metrics` table for global metrics | | `20260206120000_torrust_tracker_torrents_completed_non_zero.sql` | Removes rows with completed=0 and adds CHECK constraint (completed >= 1) | | `20260206130000_torrust_tracker_whitelist_binary.sql` | Optimizes whitelist table: removes `id` column, uses BINARY(20) `info_hash` as PRIMARY KEY | +| `20260206140000_torrust_tracker_completed_v1.sql` | Migrates `torrents` to `completed_v1`: removes `id` column, renames `completed` to `count`, uses BINARY(20) `info_hash` as PRIMARY KEY | diff --git a/packages/tracker-core/migrations/mysql/20260206140000_torrust_tracker_completed_v1.sql b/packages/tracker-core/migrations/mysql/20260206140000_torrust_tracker_completed_v1.sql new file mode 100644 index 000000000..cd83b043a --- /dev/null +++ b/packages/tracker-core/migrations/mysql/20260206140000_torrust_tracker_completed_v1.sql @@ -0,0 +1,13 @@ +-- Migrate torrents table to completed_v1 with BINARY(20) info_hash as PRIMARY KEY +-- Rename 'completed' column to 'count' +-- BINARY(20) stores the raw 20-byte infohash instead of 40-char hex string + +CREATE TABLE IF NOT EXISTS completed_v1 ( + info_hash BINARY(20) PRIMARY KEY NOT NULL, + count INTEGER DEFAULT 1 NOT NULL CHECK (count >= 1) +); + +-- Convert existing hex strings to binary +INSERT INTO completed_v1 (info_hash, count) SELECT UNHEX(info_hash), completed FROM torrents; + +DROP TABLE torrents; diff --git a/packages/tracker-core/migrations/sqlite/20260206140000_torrust_tracker_completed_v1.sql b/packages/tracker-core/migrations/sqlite/20260206140000_torrust_tracker_completed_v1.sql new file mode 100644 index 000000000..8ba45e24f --- /dev/null +++ b/packages/tracker-core/migrations/sqlite/20260206140000_torrust_tracker_completed_v1.sql @@ -0,0 +1,10 @@ +-- Migrate torrents table to completed_v1 with info_hash as PRIMARY KEY and WITHOUT ROWID optimization +-- Rename 'completed' column to 'count' +CREATE TABLE IF NOT EXISTS completed_v1 ( + info_hash TEXT PRIMARY KEY NOT NULL, + count INTEGER DEFAULT 1 NOT NULL CHECK (count >= 1) +) WITHOUT ROWID; + +INSERT INTO completed_v1 (info_hash, count) SELECT info_hash, completed FROM torrents; + +DROP TABLE torrents; diff --git a/packages/tracker-core/src/databases/driver/mysql.rs b/packages/tracker-core/src/databases/driver/mysql.rs index 67c347d6a..d2c41d5ca 100644 --- a/packages/tracker-core/src/databases/driver/mysql.rs +++ b/packages/tracker-core/src/databases/driver/mysql.rs @@ -5,7 +5,6 @@ //! based on a URL, creates the necessary tables (for torrent metrics, torrent //! whitelist, and authentication keys), and implements all CRUD operations //! required by the persistence layer. -use std::str::FromStr; use std::time::Duration; use bittorrent_primitives::info_hash::InfoHash; @@ -78,11 +77,10 @@ impl Database for Mysql { );" .to_string(); - let create_torrents_table = " - CREATE TABLE IF NOT EXISTS torrents ( - id integer PRIMARY KEY AUTO_INCREMENT, - info_hash VARCHAR(40) NOT NULL UNIQUE, - completed INTEGER DEFAULT 1 NOT NULL CHECK (completed >= 1) + let create_completed_table = " + CREATE TABLE IF NOT EXISTS completed_v1 ( + info_hash BINARY(20) PRIMARY KEY NOT NULL, + count INTEGER DEFAULT 1 NOT NULL CHECK (count >= 1) );" .to_string(); @@ -108,8 +106,8 @@ impl Database for Mysql { let mut conn = self.pool.get().map_err(|e| (e, DRIVER))?; - conn.query_drop(&create_torrents_table) - .expect("Could not create torrents table."); + conn.query_drop(&create_completed_table) + .expect("Could not create completed_v1 table."); conn.query_drop(&create_torrent_aggregate_metrics_table) .expect("Could not create create_torrent_aggregate_metrics_table table."); conn.query_drop(&create_keys_table).expect("Could not create keys table."); @@ -125,8 +123,8 @@ impl Database for Mysql { DROP TABLE `whitelist_v1`;" .to_string(); - let drop_torrents_table = " - DROP TABLE `torrents`;" + let drop_completed_table = " + DROP TABLE `completed_v1`;" .to_string(); let drop_keys_table = " @@ -137,8 +135,8 @@ impl Database for Mysql { conn.query_drop(&drop_whitelist_table) .expect("Could not drop `whitelist_v1` table."); - conn.query_drop(&drop_torrents_table) - .expect("Could not drop `torrents` table."); + conn.query_drop(&drop_completed_table) + .expect("Could not drop `completed_v1` table."); conn.query_drop(&drop_keys_table).expect("Could not drop `keys` table."); Ok(()) @@ -149,10 +147,10 @@ impl Database for Mysql { let mut conn = self.pool.get().map_err(|e| (e, DRIVER))?; let torrents = conn.query_map( - "SELECT info_hash, completed FROM torrents", - |(info_hash_string, completed): (String, u32)| { - let info_hash = InfoHash::from_str(&info_hash_string).unwrap(); - (info_hash, completed) + "SELECT info_hash, count FROM completed_v1", + |(info_hash_bytes, count): (Vec, u32)| { + let info_hash = InfoHash::from_bytes(&info_hash_bytes); + (info_hash, count) }, )?; @@ -164,8 +162,8 @@ impl Database for Mysql { let mut conn = self.pool.get().map_err(|e| (e, DRIVER))?; let query = conn.exec_first::( - "SELECT completed FROM torrents WHERE info_hash = :info_hash", - params! { "info_hash" => info_hash.to_hex_string() }, + "SELECT count FROM completed_v1 WHERE info_hash = :info_hash", + params! { "info_hash" => info_hash.bytes() }, ); let persistent_torrent = query?; @@ -175,24 +173,25 @@ impl Database for Mysql { /// Refer to [`databases::Database::save_persistent_torrent`](crate::core::databases::Database::save_persistent_torrent). fn save_torrent_downloads(&self, info_hash: &InfoHash, completed: u32) -> Result<(), Error> { - const COMMAND : &str = "INSERT INTO torrents (info_hash, completed) VALUES (:info_hash_str, :completed) ON DUPLICATE KEY UPDATE completed = VALUES(completed)"; + const COMMAND: &str = "INSERT INTO completed_v1 (info_hash, count) VALUES (:info_hash_bytes, :count) ON DUPLICATE KEY UPDATE count = VALUES(count)"; let mut conn = self.pool.get().map_err(|e| (e, DRIVER))?; - let info_hash_str = info_hash.to_string(); + let info_hash_bytes = info_hash.bytes(); + let count = completed; - Ok(conn.exec_drop(COMMAND, params! { info_hash_str, completed })?) + Ok(conn.exec_drop(COMMAND, params! { info_hash_bytes, count })?) } /// Refer to [`databases::Database::increase_number_of_downloads`](crate::core::databases::Database::increase_number_of_downloads). fn increase_downloads_for_torrent(&self, info_hash: &InfoHash) -> Result<(), Error> { let mut conn = self.pool.get().map_err(|e| (e, DRIVER))?; - let info_hash_str = info_hash.to_string(); + let info_hash_bytes = info_hash.bytes(); conn.exec_drop( - "UPDATE torrents SET completed = completed + 1 WHERE info_hash = :info_hash_str", - params! { info_hash_str }, + "UPDATE completed_v1 SET count = count + 1 WHERE info_hash = :info_hash_bytes", + params! { info_hash_bytes }, )?; Ok(()) diff --git a/packages/tracker-core/src/databases/driver/sqlite.rs b/packages/tracker-core/src/databases/driver/sqlite.rs index 7681adf4e..6491c6cf0 100644 --- a/packages/tracker-core/src/databases/driver/sqlite.rs +++ b/packages/tracker-core/src/databases/driver/sqlite.rs @@ -93,13 +93,12 @@ impl Database for Sqlite { ) WITHOUT ROWID;" .to_string(); - let create_torrents_table = " - CREATE TABLE IF NOT EXISTS torrents ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - info_hash TEXT NOT NULL UNIQUE, - completed INTEGER DEFAULT 1 NOT NULL CHECK (completed >= 1) - );" - .to_string(); + let create_completed_table = " + CREATE TABLE IF NOT EXISTS completed_v1 ( + info_hash TEXT PRIMARY KEY NOT NULL, + count INTEGER DEFAULT 1 NOT NULL CHECK (count >= 1) + ) WITHOUT ROWID;" + .to_string(); let create_torrent_aggregate_metrics_table = " CREATE TABLE IF NOT EXISTS torrent_aggregate_metrics ( @@ -121,7 +120,7 @@ impl Database for Sqlite { conn.execute(&create_whitelist_table, [])?; conn.execute(&create_keys_table, [])?; - conn.execute(&create_torrents_table, [])?; + conn.execute(&create_completed_table, [])?; conn.execute(&create_torrent_aggregate_metrics_table, [])?; Ok(()) @@ -133,8 +132,8 @@ impl Database for Sqlite { DROP TABLE whitelist_v1;" .to_string(); - let drop_torrents_table = " - DROP TABLE torrents;" + let drop_completed_table = " + DROP TABLE completed_v1;" .to_string(); let drop_keys_table = " @@ -144,7 +143,7 @@ impl Database for Sqlite { let conn = self.pool.get().map_err(|e| (e, DRIVER))?; conn.execute(&drop_whitelist_table, []) - .and_then(|_| conn.execute(&drop_torrents_table, [])) + .and_then(|_| conn.execute(&drop_completed_table, [])) .and_then(|_| conn.execute(&drop_keys_table, []))?; Ok(()) @@ -154,13 +153,13 @@ impl Database for Sqlite { fn load_all_torrents_downloads(&self) -> Result { let conn = self.pool.get().map_err(|e| (e, DRIVER))?; - let mut stmt = conn.prepare("SELECT info_hash, completed FROM torrents")?; + let mut stmt = conn.prepare("SELECT info_hash, count FROM completed_v1")?; let torrent_iter = stmt.query_map([], |row| { let info_hash_string: String = row.get(0)?; let info_hash = InfoHash::from_str(&info_hash_string).unwrap(); - let completed: u32 = row.get(1)?; - Ok((info_hash, completed)) + let count: u32 = row.get(1)?; + Ok((info_hash, count)) })?; Ok(torrent_iter.filter_map(std::result::Result::ok).collect()) @@ -170,7 +169,7 @@ impl Database for Sqlite { fn load_torrent_downloads(&self, info_hash: &InfoHash) -> Result, Error> { let conn = self.pool.get().map_err(|e| (e, DRIVER))?; - let mut stmt = conn.prepare("SELECT completed FROM torrents WHERE info_hash = ?")?; + let mut stmt = conn.prepare("SELECT count FROM completed_v1 WHERE info_hash = ?")?; let mut rows = stmt.query([info_hash.to_hex_string()])?; @@ -187,7 +186,7 @@ impl Database for Sqlite { let conn = self.pool.get().map_err(|e| (e, DRIVER))?; let insert = conn.execute( - "INSERT INTO torrents (info_hash, completed) VALUES (?1, ?2) ON CONFLICT(info_hash) DO UPDATE SET completed = ?2", + "INSERT INTO completed_v1 (info_hash, count) VALUES (?1, ?2) ON CONFLICT(info_hash) DO UPDATE SET count = ?2", [info_hash.to_string(), completed.to_string()], )?; @@ -206,7 +205,7 @@ impl Database for Sqlite { let conn = self.pool.get().map_err(|e| (e, DRIVER))?; let _ = conn.execute( - "UPDATE torrents SET completed = completed + 1 WHERE info_hash = ?", + "UPDATE completed_v1 SET count = count + 1 WHERE info_hash = ?", [info_hash.to_string()], )?;