diff --git a/Cargo.lock b/Cargo.lock index a1b386c09..c8cda6a25 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1937,7 +1937,6 @@ dependencies = [ "futures-util", "getrandom 0.3.4", "grass", - "hex", "hostname", "http 1.4.0", "http-body-util", @@ -1965,7 +1964,6 @@ dependencies = [ "serde_json", "slug", "sqlx", - "strum", "syntect", "sysinfo", "tempfile", @@ -2005,8 +2003,14 @@ dependencies = [ "docs_rs_env_vars", "docs_rs_opentelemetry", "futures-util", + "hex", "opentelemetry", + "rand 0.9.2", + "serde", + "serde_json", "sqlx", + "strum", + "test-case", "thiserror 2.0.17", "tokio", "tracing", diff --git a/Cargo.toml b/Cargo.toml index d7799a907..7a1f79838 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -93,12 +93,10 @@ opentelemetry_sdk = { workspace = true } rustwide = { version = "0.21.0", features = ["unstable-toolchain-ci", "unstable"] } hostname = "0.4.0" base64 = "0.22" -strum = { workspace = true } lol_html = "2.0.0" font-awesome-as-a-crate = { path = "crates/lib/font-awesome-as-a-crate" } getrandom = "0.3.1" itertools = { workspace = true } -hex = "0.4.3" derive_more = { workspace = true } sysinfo = { version = "0.37.2", default-features = false, features = ["system"] } derive_builder = "0.20.2" diff --git a/crates/lib/docs_rs_database/.sqlx/query-4a6887c2d436121cb2ba6a9c5069455b8f222d929672dc1ff810fa49c2940e2c.json b/crates/lib/docs_rs_database/.sqlx/query-4a6887c2d436121cb2ba6a9c5069455b8f222d929672dc1ff810fa49c2940e2c.json new file mode 100644 index 000000000..530c4d879 --- /dev/null +++ b/crates/lib/docs_rs_database/.sqlx/query-4a6887c2d436121cb2ba6a9c5069455b8f222d929672dc1ff810fa49c2940e2c.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO config (name, value)\n VALUES ($1, $2)\n ON CONFLICT (name) DO UPDATE SET value = $2;", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Json" + ] + }, + "nullable": [] + }, + "hash": "4a6887c2d436121cb2ba6a9c5069455b8f222d929672dc1ff810fa49c2940e2c" +} diff --git a/.sqlx/query-4ec79e1fa4249f4e1b085d1bbdffa537d1e3160d642e4a3325f4aea6c21974eb.json b/crates/lib/docs_rs_database/.sqlx/query-4ec79e1fa4249f4e1b085d1bbdffa537d1e3160d642e4a3325f4aea6c21974eb.json similarity index 100% rename from .sqlx/query-4ec79e1fa4249f4e1b085d1bbdffa537d1e3160d642e4a3325f4aea6c21974eb.json rename to crates/lib/docs_rs_database/.sqlx/query-4ec79e1fa4249f4e1b085d1bbdffa537d1e3160d642e4a3325f4aea6c21974eb.json diff --git a/crates/lib/docs_rs_database/.sqlx/query-5ad9cd6cd9d444d258f7486fda178d4dd071cf43cb0ea950574af8e2f37b4a21.json b/crates/lib/docs_rs_database/.sqlx/query-5ad9cd6cd9d444d258f7486fda178d4dd071cf43cb0ea950574af8e2f37b4a21.json new file mode 100644 index 000000000..264e19fd2 --- /dev/null +++ b/crates/lib/docs_rs_database/.sqlx/query-5ad9cd6cd9d444d258f7486fda178d4dd071cf43cb0ea950574af8e2f37b4a21.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT value FROM config WHERE name = $1;", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "value", + "type_info": "Json" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "5ad9cd6cd9d444d258f7486fda178d4dd071cf43cb0ea950574af8e2f37b4a21" +} diff --git a/crates/lib/docs_rs_database/.sqlx/query-96b68919f9016705a1a36ef11a5a659e7fb431beb0017fbcfd21132f105ce722.json b/crates/lib/docs_rs_database/.sqlx/query-96b68919f9016705a1a36ef11a5a659e7fb431beb0017fbcfd21132f105ce722.json new file mode 100644 index 000000000..984eff3ac --- /dev/null +++ b/crates/lib/docs_rs_database/.sqlx/query-96b68919f9016705a1a36ef11a5a659e7fb431beb0017fbcfd21132f105ce722.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT relname\n FROM pg_class\n INNER JOIN pg_namespace ON\n pg_class.relnamespace = pg_namespace.oid\n WHERE pg_class.relkind = 'S'\n AND pg_namespace.nspname = $1\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "relname", + "type_info": "Name" + } + ], + "parameters": { + "Left": [ + "Name" + ] + }, + "nullable": [ + false + ] + }, + "hash": "96b68919f9016705a1a36ef11a5a659e7fb431beb0017fbcfd21132f105ce722" +} diff --git a/crates/lib/docs_rs_database/Cargo.toml b/crates/lib/docs_rs_database/Cargo.toml index c4ad4fa4f..594d8c60c 100644 --- a/crates/lib/docs_rs_database/Cargo.toml +++ b/crates/lib/docs_rs_database/Cargo.toml @@ -2,17 +2,31 @@ name = "docs_rs_database" version = "0.1.0" edition = "2024" +build = "build.rs" [dependencies] anyhow = { workspace = true } docs_rs_env_vars = { path = "../docs_rs_env_vars" } docs_rs_opentelemetry = { path = "../docs_rs_opentelemetry" } futures-util = { workspace = true } +hex = "0.4.3" opentelemetry = { workspace = true } +rand = { workspace = true, optional = true } +serde = { workspace = true } +serde_json = { workspace = true } sqlx = { workspace = true } +strum = { workspace = true } thiserror = { workspace = true } tokio = { workspace = true } tracing = { workspace = true } +[dev-dependencies] +rand = { workspace = true } +docs_rs_opentelemetry = { path = "../docs_rs_opentelemetry", features = ["testing"] } +test-case = { workspace = true } + [features] -testing = [] +testing = [ + "dep:rand", + "docs_rs_opentelemetry/testing", +] diff --git a/crates/lib/docs_rs_database/build.rs b/crates/lib/docs_rs_database/build.rs new file mode 100644 index 000000000..25c393629 --- /dev/null +++ b/crates/lib/docs_rs_database/build.rs @@ -0,0 +1,4 @@ +fn main() { + // trigger recompilation when a new migration is added + println!("cargo:rerun-if-changed=migrations"); +} diff --git a/migrations/20231021111635_initial.down.sql b/crates/lib/docs_rs_database/migrations/20231021111635_initial.down.sql similarity index 100% rename from migrations/20231021111635_initial.down.sql rename to crates/lib/docs_rs_database/migrations/20231021111635_initial.down.sql diff --git a/migrations/20231021111635_initial.up.sql b/crates/lib/docs_rs_database/migrations/20231021111635_initial.up.sql similarity index 100% rename from migrations/20231021111635_initial.up.sql rename to crates/lib/docs_rs_database/migrations/20231021111635_initial.up.sql diff --git a/migrations/20240221104457_drop_releases_build_status.down.sql b/crates/lib/docs_rs_database/migrations/20240221104457_drop_releases_build_status.down.sql similarity index 100% rename from migrations/20240221104457_drop_releases_build_status.down.sql rename to crates/lib/docs_rs_database/migrations/20240221104457_drop_releases_build_status.down.sql diff --git a/migrations/20240221104457_drop_releases_build_status.up.sql b/crates/lib/docs_rs_database/migrations/20240221104457_drop_releases_build_status.up.sql similarity index 100% rename from migrations/20240221104457_drop_releases_build_status.up.sql rename to crates/lib/docs_rs_database/migrations/20240221104457_drop_releases_build_status.up.sql diff --git a/migrations/20240221113734_drop_releases_rustc_version.down.sql b/crates/lib/docs_rs_database/migrations/20240221113734_drop_releases_rustc_version.down.sql similarity index 100% rename from migrations/20240221113734_drop_releases_rustc_version.down.sql rename to crates/lib/docs_rs_database/migrations/20240221113734_drop_releases_rustc_version.down.sql diff --git a/migrations/20240221113734_drop_releases_rustc_version.up.sql b/crates/lib/docs_rs_database/migrations/20240221113734_drop_releases_rustc_version.up.sql similarity index 100% rename from migrations/20240221113734_drop_releases_rustc_version.up.sql rename to crates/lib/docs_rs_database/migrations/20240221113734_drop_releases_rustc_version.up.sql diff --git a/migrations/20240221114302_ensure_no_buildless_releases_exist.down.sql b/crates/lib/docs_rs_database/migrations/20240221114302_ensure_no_buildless_releases_exist.down.sql similarity index 100% rename from migrations/20240221114302_ensure_no_buildless_releases_exist.down.sql rename to crates/lib/docs_rs_database/migrations/20240221114302_ensure_no_buildless_releases_exist.down.sql diff --git a/migrations/20240221114302_ensure_no_buildless_releases_exist.up.sql b/crates/lib/docs_rs_database/migrations/20240221114302_ensure_no_buildless_releases_exist.up.sql similarity index 100% rename from migrations/20240221114302_ensure_no_buildless_releases_exist.up.sql rename to crates/lib/docs_rs_database/migrations/20240221114302_ensure_no_buildless_releases_exist.up.sql diff --git a/migrations/20240221124844_multi_stage_build_status.down.sql b/crates/lib/docs_rs_database/migrations/20240221124844_multi_stage_build_status.down.sql similarity index 100% rename from migrations/20240221124844_multi_stage_build_status.down.sql rename to crates/lib/docs_rs_database/migrations/20240221124844_multi_stage_build_status.down.sql diff --git a/migrations/20240221124844_multi_stage_build_status.up.sql b/crates/lib/docs_rs_database/migrations/20240221124844_multi_stage_build_status.up.sql similarity index 100% rename from migrations/20240221124844_multi_stage_build_status.up.sql rename to crates/lib/docs_rs_database/migrations/20240221124844_multi_stage_build_status.up.sql diff --git a/migrations/20240227040753_add_owner_kind.down.sql b/crates/lib/docs_rs_database/migrations/20240227040753_add_owner_kind.down.sql similarity index 100% rename from migrations/20240227040753_add_owner_kind.down.sql rename to crates/lib/docs_rs_database/migrations/20240227040753_add_owner_kind.down.sql diff --git a/migrations/20240227040753_add_owner_kind.up.sql b/crates/lib/docs_rs_database/migrations/20240227040753_add_owner_kind.up.sql similarity index 100% rename from migrations/20240227040753_add_owner_kind.up.sql rename to crates/lib/docs_rs_database/migrations/20240227040753_add_owner_kind.up.sql diff --git a/migrations/20240309082057_release_status_view.sql.down.sql b/crates/lib/docs_rs_database/migrations/20240309082057_release_status_view.sql.down.sql similarity index 100% rename from migrations/20240309082057_release_status_view.sql.down.sql rename to crates/lib/docs_rs_database/migrations/20240309082057_release_status_view.sql.down.sql diff --git a/migrations/20240309082057_release_status_view.sql.up.sql b/crates/lib/docs_rs_database/migrations/20240309082057_release_status_view.sql.up.sql similarity index 100% rename from migrations/20240309082057_release_status_view.sql.up.sql rename to crates/lib/docs_rs_database/migrations/20240309082057_release_status_view.sql.up.sql diff --git a/migrations/20240311202914_release_status_materialized.down.sql b/crates/lib/docs_rs_database/migrations/20240311202914_release_status_materialized.down.sql similarity index 100% rename from migrations/20240311202914_release_status_materialized.down.sql rename to crates/lib/docs_rs_database/migrations/20240311202914_release_status_materialized.down.sql diff --git a/migrations/20240311202914_release_status_materialized.up.sql b/crates/lib/docs_rs_database/migrations/20240311202914_release_status_materialized.up.sql similarity index 100% rename from migrations/20240311202914_release_status_materialized.up.sql rename to crates/lib/docs_rs_database/migrations/20240311202914_release_status_materialized.up.sql diff --git a/migrations/20240313103708_make_release_fields_optional.down.sql b/crates/lib/docs_rs_database/migrations/20240313103708_make_release_fields_optional.down.sql similarity index 100% rename from migrations/20240313103708_make_release_fields_optional.down.sql rename to crates/lib/docs_rs_database/migrations/20240313103708_make_release_fields_optional.down.sql diff --git a/migrations/20240313103708_make_release_fields_optional.up.sql b/crates/lib/docs_rs_database/migrations/20240313103708_make_release_fields_optional.up.sql similarity index 100% rename from migrations/20240313103708_make_release_fields_optional.up.sql rename to crates/lib/docs_rs_database/migrations/20240313103708_make_release_fields_optional.up.sql diff --git a/migrations/20240313182623_make_build_fields_optional.down.sql b/crates/lib/docs_rs_database/migrations/20240313182623_make_build_fields_optional.down.sql similarity index 100% rename from migrations/20240313182623_make_build_fields_optional.down.sql rename to crates/lib/docs_rs_database/migrations/20240313182623_make_build_fields_optional.down.sql diff --git a/migrations/20240313182623_make_build_fields_optional.up.sql b/crates/lib/docs_rs_database/migrations/20240313182623_make_build_fields_optional.up.sql similarity index 100% rename from migrations/20240313182623_make_build_fields_optional.up.sql rename to crates/lib/docs_rs_database/migrations/20240313182623_make_build_fields_optional.up.sql diff --git a/migrations/20240313184911_build_errors.down.sql b/crates/lib/docs_rs_database/migrations/20240313184911_build_errors.down.sql similarity index 100% rename from migrations/20240313184911_build_errors.down.sql rename to crates/lib/docs_rs_database/migrations/20240313184911_build_errors.down.sql diff --git a/migrations/20240313184911_build_errors.up.sql b/crates/lib/docs_rs_database/migrations/20240313184911_build_errors.up.sql similarity index 100% rename from migrations/20240313184911_build_errors.up.sql rename to crates/lib/docs_rs_database/migrations/20240313184911_build_errors.up.sql diff --git a/migrations/20240519141105_crate-version-name-field-length.down.sql b/crates/lib/docs_rs_database/migrations/20240519141105_crate-version-name-field-length.down.sql similarity index 100% rename from migrations/20240519141105_crate-version-name-field-length.down.sql rename to crates/lib/docs_rs_database/migrations/20240519141105_crate-version-name-field-length.down.sql diff --git a/migrations/20240519141105_crate-version-name-field-length.up.sql b/crates/lib/docs_rs_database/migrations/20240519141105_crate-version-name-field-length.up.sql similarity index 100% rename from migrations/20240519141105_crate-version-name-field-length.up.sql rename to crates/lib/docs_rs_database/migrations/20240519141105_crate-version-name-field-length.up.sql diff --git a/migrations/20240624085737_build-status-idx.down.sql b/crates/lib/docs_rs_database/migrations/20240624085737_build-status-idx.down.sql similarity index 100% rename from migrations/20240624085737_build-status-idx.down.sql rename to crates/lib/docs_rs_database/migrations/20240624085737_build-status-idx.down.sql diff --git a/migrations/20240624085737_build-status-idx.up.sql b/crates/lib/docs_rs_database/migrations/20240624085737_build-status-idx.up.sql similarity index 100% rename from migrations/20240624085737_build-status-idx.up.sql rename to crates/lib/docs_rs_database/migrations/20240624085737_build-status-idx.up.sql diff --git a/migrations/20241015054153_buildqueue-queue-crate-version-name-field-length.down.sql b/crates/lib/docs_rs_database/migrations/20241015054153_buildqueue-queue-crate-version-name-field-length.down.sql similarity index 100% rename from migrations/20241015054153_buildqueue-queue-crate-version-name-field-length.down.sql rename to crates/lib/docs_rs_database/migrations/20241015054153_buildqueue-queue-crate-version-name-field-length.down.sql diff --git a/migrations/20241015054153_buildqueue-queue-crate-version-name-field-length.up.sql b/crates/lib/docs_rs_database/migrations/20241015054153_buildqueue-queue-crate-version-name-field-length.up.sql similarity index 100% rename from migrations/20241015054153_buildqueue-queue-crate-version-name-field-length.up.sql rename to crates/lib/docs_rs_database/migrations/20241015054153_buildqueue-queue-crate-version-name-field-length.up.sql diff --git a/migrations/20241018031600_documentation_size.down.sql b/crates/lib/docs_rs_database/migrations/20241018031600_documentation_size.down.sql similarity index 100% rename from migrations/20241018031600_documentation_size.down.sql rename to crates/lib/docs_rs_database/migrations/20241018031600_documentation_size.down.sql diff --git a/migrations/20241018031600_documentation_size.up.sql b/crates/lib/docs_rs_database/migrations/20241018031600_documentation_size.up.sql similarity index 100% rename from migrations/20241018031600_documentation_size.up.sql rename to crates/lib/docs_rs_database/migrations/20241018031600_documentation_size.up.sql diff --git a/migrations/20241018052241_builds-rustc-nightly-date.down.sql b/crates/lib/docs_rs_database/migrations/20241018052241_builds-rustc-nightly-date.down.sql similarity index 100% rename from migrations/20241018052241_builds-rustc-nightly-date.down.sql rename to crates/lib/docs_rs_database/migrations/20241018052241_builds-rustc-nightly-date.down.sql diff --git a/migrations/20241018052241_builds-rustc-nightly-date.up.sql b/crates/lib/docs_rs_database/migrations/20241018052241_builds-rustc-nightly-date.up.sql similarity index 100% rename from migrations/20241018052241_builds-rustc-nightly-date.up.sql rename to crates/lib/docs_rs_database/migrations/20241018052241_builds-rustc-nightly-date.up.sql diff --git a/migrations/20241021050229_builds-started-finished.down.sql b/crates/lib/docs_rs_database/migrations/20241021050229_builds-started-finished.down.sql similarity index 100% rename from migrations/20241021050229_builds-started-finished.down.sql rename to crates/lib/docs_rs_database/migrations/20241021050229_builds-started-finished.down.sql diff --git a/migrations/20241021050229_builds-started-finished.up.sql b/crates/lib/docs_rs_database/migrations/20241021050229_builds-started-finished.up.sql similarity index 100% rename from migrations/20241021050229_builds-started-finished.up.sql rename to crates/lib/docs_rs_database/migrations/20241021050229_builds-started-finished.up.sql diff --git a/migrations/20241106085600_releases-rustdoc-status-idx.down.sql b/crates/lib/docs_rs_database/migrations/20241106085600_releases-rustdoc-status-idx.down.sql similarity index 100% rename from migrations/20241106085600_releases-rustdoc-status-idx.down.sql rename to crates/lib/docs_rs_database/migrations/20241106085600_releases-rustdoc-status-idx.down.sql diff --git a/migrations/20241106085600_releases-rustdoc-status-idx.up.sql b/crates/lib/docs_rs_database/migrations/20241106085600_releases-rustdoc-status-idx.up.sql similarity index 100% rename from migrations/20241106085600_releases-rustdoc-status-idx.up.sql rename to crates/lib/docs_rs_database/migrations/20241106085600_releases-rustdoc-status-idx.up.sql diff --git a/migrations/20241219091521_owner-avatar-longer.down.sql b/crates/lib/docs_rs_database/migrations/20241219091521_owner-avatar-longer.down.sql similarity index 100% rename from migrations/20241219091521_owner-avatar-longer.down.sql rename to crates/lib/docs_rs_database/migrations/20241219091521_owner-avatar-longer.down.sql diff --git a/migrations/20241219091521_owner-avatar-longer.up.sql b/crates/lib/docs_rs_database/migrations/20241219091521_owner-avatar-longer.up.sql similarity index 100% rename from migrations/20241219091521_owner-avatar-longer.up.sql rename to crates/lib/docs_rs_database/migrations/20241219091521_owner-avatar-longer.up.sql diff --git a/migrations/20251202020754_remove-file-public.down.sql b/crates/lib/docs_rs_database/migrations/20251202020754_remove-file-public.down.sql similarity index 100% rename from migrations/20251202020754_remove-file-public.down.sql rename to crates/lib/docs_rs_database/migrations/20251202020754_remove-file-public.down.sql diff --git a/migrations/20251202020754_remove-file-public.up.sql b/crates/lib/docs_rs_database/migrations/20251202020754_remove-file-public.up.sql similarity index 100% rename from migrations/20251202020754_remove-file-public.up.sql rename to crates/lib/docs_rs_database/migrations/20251202020754_remove-file-public.up.sql diff --git a/migrations/20251202040858_remove-cdn-invalidation-queue.down.sql b/crates/lib/docs_rs_database/migrations/20251202040858_remove-cdn-invalidation-queue.down.sql similarity index 100% rename from migrations/20251202040858_remove-cdn-invalidation-queue.down.sql rename to crates/lib/docs_rs_database/migrations/20251202040858_remove-cdn-invalidation-queue.down.sql diff --git a/migrations/20251202040858_remove-cdn-invalidation-queue.up.sql b/crates/lib/docs_rs_database/migrations/20251202040858_remove-cdn-invalidation-queue.up.sql similarity index 100% rename from migrations/20251202040858_remove-cdn-invalidation-queue.up.sql rename to crates/lib/docs_rs_database/migrations/20251202040858_remove-cdn-invalidation-queue.up.sql diff --git a/migrations/20251220165302_remove-files.down.sql b/crates/lib/docs_rs_database/migrations/20251220165302_remove-files.down.sql similarity index 100% rename from migrations/20251220165302_remove-files.down.sql rename to crates/lib/docs_rs_database/migrations/20251220165302_remove-files.down.sql diff --git a/migrations/20251220165302_remove-files.up.sql b/crates/lib/docs_rs_database/migrations/20251220165302_remove-files.up.sql similarity index 100% rename from migrations/20251220165302_remove-files.up.sql rename to crates/lib/docs_rs_database/migrations/20251220165302_remove-files.up.sql diff --git a/crates/lib/docs_rs_database/src/lib.rs b/crates/lib/docs_rs_database/src/lib.rs index 7e4c466bf..568814d16 100644 --- a/crates/lib/docs_rs_database/src/lib.rs +++ b/crates/lib/docs_rs_database/src/lib.rs @@ -1,8 +1,13 @@ mod config; mod errors; mod metrics; +mod migrations; mod pool; +pub mod service_config; +#[cfg(any(test, feature = "testing"))] +pub mod testing; pub use config::Config; pub use errors::PoolError; +pub use migrations::migrate; pub use pool::{AsyncPoolClient, Pool}; diff --git a/crates/lib/docs_rs_database/src/migrations.rs b/crates/lib/docs_rs_database/src/migrations.rs new file mode 100644 index 000000000..9f08301cd --- /dev/null +++ b/crates/lib/docs_rs_database/src/migrations.rs @@ -0,0 +1,60 @@ +use anyhow::Result; +use sqlx::migrate::{Migrate, Migrator}; + +pub static MIGRATOR: Migrator = sqlx::migrate!(); // defaults to "./migrations" + +pub async fn migrate(conn: &mut sqlx::PgConnection, target: Option) -> Result<()> { + conn.ensure_migrations_table().await?; + + // `database_versions` is the table that tracked the old `schemamama` migrations. + // If we find the table, and it contains records, we insert a fake record + // into the `_sqlx_migrations` table so the big initial migration isn't executed. + if sqlx::query( + "SELECT table_name + FROM information_schema.tables + WHERE table_schema = 'public' AND table_name = 'database_versions'", + ) + .fetch_optional(&mut *conn) + .await? + .is_some() + { + let max_version: Option = + sqlx::query_scalar("SELECT max(version) FROM database_versions") + .fetch_one(&mut *conn) + .await?; + + if max_version != Some(39) { + anyhow::bail!( + "database_versions table has unexpected version: {:?}", + max_version + ); + } + + sqlx::query( + "INSERT INTO _sqlx_migrations ( version, description, success, checksum, execution_time ) + VALUES ( $1, $2, TRUE, $3, -1 )", + ) + // the next two parameters relate to the filename of the initial migration file + .bind(20231021111635i64) + .bind("initial") + // this is the hash of the initial migration file, as sqlx requires it. + // if the initial migration file changes, this has to be updated with the new value, + // easiest to get from the `_sqlx_migrations` table when the migration was normally + // executed. + .bind(hex::decode("df802e0ec416063caadd1c06b13348cd885583c44962998886b929d5fe6ef3b70575d5101c5eb31daa989721df08d806").unwrap()) + .execute(&mut *conn) + .await?; + + sqlx::query("DROP TABLE database_versions") + .execute(&mut *conn) + .await?; + } + + // when we find records + if let Some(target) = target { + MIGRATOR.undo(conn, target).await?; + } else { + MIGRATOR.run(conn).await?; + } + Ok(()) +} diff --git a/crates/lib/docs_rs_database/src/pool.rs b/crates/lib/docs_rs_database/src/pool.rs index 21ec35501..b4bc926f8 100644 --- a/crates/lib/docs_rs_database/src/pool.rs +++ b/crates/lib/docs_rs_database/src/pool.rs @@ -30,7 +30,7 @@ impl Pool { Self::new_inner(config, DEFAULT_SCHEMA, otel_meter_provider).await } - #[cfg(feature = "testing")] + #[cfg(any(test, feature = "testing"))] pub async fn new_with_schema( config: &Config, schema: &str, diff --git a/crates/lib/docs_rs_database/src/service_config.rs b/crates/lib/docs_rs_database/src/service_config.rs new file mode 100644 index 000000000..cb99aefbe --- /dev/null +++ b/crates/lib/docs_rs_database/src/service_config.rs @@ -0,0 +1,109 @@ +use anyhow::Result; +use serde::{Serialize, de::DeserializeOwned}; + +#[derive(strum::IntoStaticStr)] +#[strum(serialize_all = "snake_case")] +pub enum ConfigName { + RustcVersion, + LastSeenIndexReference, + QueueLocked, + Toolchain, +} + +pub async fn set_config( + conn: &mut sqlx::PgConnection, + name: ConfigName, + value: impl Serialize, +) -> anyhow::Result<()> { + let name: &'static str = name.into(); + sqlx::query!( + "INSERT INTO config (name, value) + VALUES ($1, $2) + ON CONFLICT (name) DO UPDATE SET value = $2;", + name, + &serde_json::to_value(value)?, + ) + .execute(conn) + .await?; + Ok(()) +} + +pub async fn get_config(conn: &mut sqlx::PgConnection, name: ConfigName) -> Result> +where + T: DeserializeOwned, +{ + let name: &'static str = name.into(); + Ok( + match sqlx::query!("SELECT value FROM config WHERE name = $1;", name) + .fetch_optional(conn) + .await? + { + Some(row) => serde_json::from_value(row.value)?, + None => None, + }, + ) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{Config, testing::TestDatabase}; + use docs_rs_opentelemetry::testing::TestMetrics; + use serde_json::Value; + use test_case::test_case; + + #[test_case(ConfigName::RustcVersion, "rustc_version")] + #[test_case(ConfigName::QueueLocked, "queue_locked")] + #[test_case(ConfigName::LastSeenIndexReference, "last_seen_index_reference")] + fn test_configname_variants(variant: ConfigName, expected: &'static str) { + let name: &'static str = variant.into(); + assert_eq!(name, expected); + } + + #[tokio::test(flavor = "multi_thread")] + async fn test_get_config_empty() -> anyhow::Result<()> { + let test_metrics = TestMetrics::new(); + let db = TestDatabase::new(&Config::test_config()?, test_metrics.provider()).await?; + + let mut conn = db.async_conn().await; + sqlx::query!("DELETE FROM config") + .execute(&mut *conn) + .await?; + + assert!( + get_config::(&mut conn, ConfigName::RustcVersion) + .await? + .is_none() + ); + Ok(()) + } + + #[tokio::test(flavor = "multi_thread")] + async fn test_set_and_get_config_() -> anyhow::Result<()> { + let test_metrics = TestMetrics::new(); + let db = TestDatabase::new(&Config::test_config()?, test_metrics.provider()).await?; + + let mut conn = db.async_conn().await; + sqlx::query!("DELETE FROM config") + .execute(&mut *conn) + .await?; + + assert!( + get_config::(&mut conn, ConfigName::RustcVersion) + .await? + .is_none() + ); + + set_config( + &mut conn, + ConfigName::RustcVersion, + Value::String("some value".into()), + ) + .await?; + assert_eq!( + get_config(&mut conn, ConfigName::RustcVersion).await?, + Some("some value".to_string()) + ); + Ok(()) + } +} diff --git a/crates/lib/docs_rs_database/src/testing/mod.rs b/crates/lib/docs_rs_database/src/testing/mod.rs new file mode 100644 index 000000000..b26953c2d --- /dev/null +++ b/crates/lib/docs_rs_database/src/testing/mod.rs @@ -0,0 +1,3 @@ +mod test_env; + +pub use test_env::TestDatabase; diff --git a/crates/lib/docs_rs_database/src/testing/test_env.rs b/crates/lib/docs_rs_database/src/testing/test_env.rs new file mode 100644 index 000000000..e0a747d41 --- /dev/null +++ b/crates/lib/docs_rs_database/src/testing/test_env.rs @@ -0,0 +1,110 @@ +use crate::{AsyncPoolClient, Config, Pool, migrations}; +use anyhow::{Context as _, Result}; +use docs_rs_opentelemetry::AnyMeterProvider; +use futures_util::TryStreamExt as _; +use sqlx::Connection as _; +use tokio::{runtime, task::block_in_place}; +use tracing::error; + +#[derive(Debug)] +pub struct TestDatabase { + pool: Pool, + schema: String, + runtime: runtime::Handle, +} + +impl TestDatabase { + pub async fn new(config: &Config, otel_meter_provider: &AnyMeterProvider) -> Result { + // A random schema name is generated and used for the current connection. This allows each + // test to create a fresh instance of the database to run within. + let schema = format!("docs_rs_test_schema_{}", rand::random::()); + + let pool = Pool::new_with_schema(config, &schema, otel_meter_provider).await?; + + let mut conn = sqlx::PgConnection::connect(&config.database_url).await?; + sqlx::query(&format!("CREATE SCHEMA {schema}")) + .execute(&mut conn) + .await + .context("error creating schema")?; + sqlx::query(&format!("SET search_path TO {schema}, public")) + .execute(&mut conn) + .await + .context("error setting search path")?; + migrations::migrate(&mut conn, None) + .await + .context("error running migrations")?; + + // Move all sequence start positions 10000 apart to avoid overlapping primary keys + let sequence_names: Vec<_> = sqlx::query!( + "SELECT relname + FROM pg_class + INNER JOIN pg_namespace ON + pg_class.relnamespace = pg_namespace.oid + WHERE pg_class.relkind = 'S' + AND pg_namespace.nspname = $1 + ", + schema, + ) + .fetch(&mut conn) + .map_ok(|row| row.relname) + .try_collect() + .await?; + + for (i, sequence) in sequence_names.into_iter().enumerate() { + let offset = (i + 1) * 10000; + sqlx::query(&format!( + r#"ALTER SEQUENCE "{sequence}" RESTART WITH {offset};"# + )) + .execute(&mut conn) + .await?; + } + + Ok(TestDatabase { + pool, + schema, + runtime: runtime::Handle::current(), + }) + } + + pub fn pool(&self) -> &Pool { + &self.pool + } + + pub async fn async_conn(&self) -> AsyncPoolClient { + self.pool + .get_async() + .await + .expect("failed to get a connection out of the pool") + } +} + +impl Drop for TestDatabase { + fn drop(&mut self) { + let pool = self.pool.clone(); + let schema = self.schema.clone(); + let runtime = self.runtime.clone(); + + block_in_place(move || { + runtime.block_on(async move { + let Ok(mut conn) = pool.get_async().await else { + error!("error in drop impl"); + return; + }; + + let migration_result = migrations::migrate(&mut conn, Some(0)).await; + + if let Err(e) = sqlx::query(format!("DROP SCHEMA {} CASCADE;", schema).as_str()) + .execute(&mut *conn) + .await + { + error!("failed to drop test schema {}: {}", schema, e); + return; + } + + if let Err(err) = migration_result { + error!(?err, "error reverting migrations"); + } + }) + }); + } +} diff --git a/crates/lib/docs_rs_repository_stats/.sqlx/query-440b3d15186a21435c470279a5c482315132d6baff4c517df246bc39638f5527.json b/crates/lib/docs_rs_repository_stats/.sqlx/query-440b3d15186a21435c470279a5c482315132d6baff4c517df246bc39638f5527.json new file mode 100644 index 000000000..a88c69335 --- /dev/null +++ b/crates/lib/docs_rs_repository_stats/.sqlx/query-440b3d15186a21435c470279a5c482315132d6baff4c517df246bc39638f5527.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT host_id\n FROM repositories\n WHERE host = $1 AND updated_at < NOW() - INTERVAL '1 day';", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "host_id", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "440b3d15186a21435c470279a5c482315132d6baff4c517df246bc39638f5527" +} diff --git a/crates/lib/docs_rs_repository_stats/.sqlx/query-4a6887c2d436121cb2ba6a9c5069455b8f222d929672dc1ff810fa49c2940e2c.json b/crates/lib/docs_rs_repository_stats/.sqlx/query-4a6887c2d436121cb2ba6a9c5069455b8f222d929672dc1ff810fa49c2940e2c.json new file mode 100644 index 000000000..530c4d879 --- /dev/null +++ b/crates/lib/docs_rs_repository_stats/.sqlx/query-4a6887c2d436121cb2ba6a9c5069455b8f222d929672dc1ff810fa49c2940e2c.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO config (name, value)\n VALUES ($1, $2)\n ON CONFLICT (name) DO UPDATE SET value = $2;", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Json" + ] + }, + "nullable": [] + }, + "hash": "4a6887c2d436121cb2ba6a9c5069455b8f222d929672dc1ff810fa49c2940e2c" +} diff --git a/crates/lib/docs_rs_repository_stats/.sqlx/query-5ad9cd6cd9d444d258f7486fda178d4dd071cf43cb0ea950574af8e2f37b4a21.json b/crates/lib/docs_rs_repository_stats/.sqlx/query-5ad9cd6cd9d444d258f7486fda178d4dd071cf43cb0ea950574af8e2f37b4a21.json new file mode 100644 index 000000000..264e19fd2 --- /dev/null +++ b/crates/lib/docs_rs_repository_stats/.sqlx/query-5ad9cd6cd9d444d258f7486fda178d4dd071cf43cb0ea950574af8e2f37b4a21.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT value FROM config WHERE name = $1;", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "value", + "type_info": "Json" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "5ad9cd6cd9d444d258f7486fda178d4dd071cf43cb0ea950574af8e2f37b4a21" +} diff --git a/crates/lib/docs_rs_repository_stats/.sqlx/query-5b401b6a191f7364be11110c23228933120dc7c39d0ef436ececc8bee9695c05.json b/crates/lib/docs_rs_repository_stats/.sqlx/query-5b401b6a191f7364be11110c23228933120dc7c39d0ef436ececc8bee9695c05.json new file mode 100644 index 000000000..cc79bd27a --- /dev/null +++ b/crates/lib/docs_rs_repository_stats/.sqlx/query-5b401b6a191f7364be11110c23228933120dc7c39d0ef436ececc8bee9695c05.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE releases SET repository_id = $1 WHERE id = $2;", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int4", + "Int4" + ] + }, + "nullable": [] + }, + "hash": "5b401b6a191f7364be11110c23228933120dc7c39d0ef436ececc8bee9695c05" +} diff --git a/crates/lib/docs_rs_repository_stats/.sqlx/query-718576e299a41495b28c843737921e3493a61c0629a9d9a5d04066d443663965.json b/crates/lib/docs_rs_repository_stats/.sqlx/query-718576e299a41495b28c843737921e3493a61c0629a9d9a5d04066d443663965.json new file mode 100644 index 000000000..032419e27 --- /dev/null +++ b/crates/lib/docs_rs_repository_stats/.sqlx/query-718576e299a41495b28c843737921e3493a61c0629a9d9a5d04066d443663965.json @@ -0,0 +1,29 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO repositories (\n host, host_id, name, description, last_commit, stars, forks, issues, updated_at\n ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, NOW())\n ON CONFLICT (host, host_id) DO\n UPDATE SET\n name = $3,\n description = $4,\n last_commit = $5,\n stars = $6,\n forks = $7,\n issues = $8,\n updated_at = NOW()\n RETURNING id;", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Varchar", + "Varchar", + "Timestamptz", + "Int4", + "Int4", + "Int4" + ] + }, + "nullable": [ + false + ] + }, + "hash": "718576e299a41495b28c843737921e3493a61c0629a9d9a5d04066d443663965" +} diff --git a/crates/lib/docs_rs_repository_stats/.sqlx/query-97a9b51028cbf8e585e120382efdff87417d99c179c695ded1bdb6cd584a7323.json b/crates/lib/docs_rs_repository_stats/.sqlx/query-97a9b51028cbf8e585e120382efdff87417d99c179c695ded1bdb6cd584a7323.json new file mode 100644 index 000000000..33a64a917 --- /dev/null +++ b/crates/lib/docs_rs_repository_stats/.sqlx/query-97a9b51028cbf8e585e120382efdff87417d99c179c695ded1bdb6cd584a7323.json @@ -0,0 +1,40 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT releases.id, crates.name, releases.version, releases.repository_url\n FROM releases\n INNER JOIN crates ON (crates.id = releases.crate_id)\n WHERE repository_id IS NULL AND repository_url LIKE $1;", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Int4" + }, + { + "ordinal": 1, + "name": "name", + "type_info": "Text" + }, + { + "ordinal": 2, + "name": "version", + "type_info": "Text" + }, + { + "ordinal": 3, + "name": "repository_url", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false, + false, + false, + true + ] + }, + "hash": "97a9b51028cbf8e585e120382efdff87417d99c179c695ded1bdb6cd584a7323" +} diff --git a/crates/lib/docs_rs_repository_stats/.sqlx/query-ce3c4ceec1fee051b9c9716d7e3dc94b213dc5449083dc109c1d935164e647e2.json b/crates/lib/docs_rs_repository_stats/.sqlx/query-ce3c4ceec1fee051b9c9716d7e3dc94b213dc5449083dc109c1d935164e647e2.json new file mode 100644 index 000000000..4200b5330 --- /dev/null +++ b/crates/lib/docs_rs_repository_stats/.sqlx/query-ce3c4ceec1fee051b9c9716d7e3dc94b213dc5449083dc109c1d935164e647e2.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT id FROM repositories WHERE name = $1 AND host = $2 LIMIT 1;", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "ce3c4ceec1fee051b9c9716d7e3dc94b213dc5449083dc109c1d935164e647e2" +} diff --git a/crates/lib/docs_rs_repository_stats/.sqlx/query-cf83cc24df4c285fe67808f1e1a2b56f850014b975a7506597a4ae3c5c6851aa.json b/crates/lib/docs_rs_repository_stats/.sqlx/query-cf83cc24df4c285fe67808f1e1a2b56f850014b975a7506597a4ae3c5c6851aa.json new file mode 100644 index 000000000..6bfe26455 --- /dev/null +++ b/crates/lib/docs_rs_repository_stats/.sqlx/query-cf83cc24df4c285fe67808f1e1a2b56f850014b975a7506597a4ae3c5c6851aa.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM repositories WHERE host_id = $1 AND host = $2;", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "cf83cc24df4c285fe67808f1e1a2b56f850014b975a7506597a4ae3c5c6851aa" +} diff --git a/crates/lib/docs_rs_utils/Cargo.toml b/crates/lib/docs_rs_utils/Cargo.toml index 39b6545a9..6c2ff888e 100644 --- a/crates/lib/docs_rs_utils/Cargo.toml +++ b/crates/lib/docs_rs_utils/Cargo.toml @@ -8,7 +8,7 @@ build = "build.rs" anyhow = { workspace = true } chrono = { workspace = true } regex = { workspace = true } -tokio = { workspace = true } +tokio = { workspace = true, features = ["time"] } tracing = { workspace = true } [build-dependencies] diff --git a/dockerfiles/Dockerfile b/dockerfiles/Dockerfile index 5e073580e..787fe17c6 100644 --- a/dockerfiles/Dockerfile +++ b/dockerfiles/Dockerfile @@ -184,8 +184,6 @@ RUN apt-get update \ && rm -rf /var/lib/apt/lists/* WORKDIR /srv/docsrs -# copy migrations so we can run them via CLI script -COPY migrations migrations/ ENTRYPOINT ["/usr/bin/tini", "/usr/local/bin/cratesfyi", "--"] diff --git a/justfiles/testing.just b/justfiles/testing.just index a87959818..327d120fe 100644 --- a/justfiles/testing.just +++ b/justfiles/testing.just @@ -12,14 +12,25 @@ bench-rustdoc-page host="http://127.0.0.1:8888": # crates that have it. [group('testing')] [group('sqlx')] -[no-cd] sqlx-prepare *args: _ensure_db_and_s3_are_running - cargo sqlx prepare \ + #!/usr/bin/env bash + set -euo pipefail + + cargo sqlx prepare \ --database-url $DOCSRS_DATABASE_URL \ --workspace \ {{ args }} \ -- --all-targets --all-features + for subcrate in crates/lib/*; do + # NOTE: potential optimization: only run this command when `sqlx` + # is in the dependencies of the subcrate. + (cd "${subcrate}" && cargo sqlx prepare \ + --database-url $DOCSRS_DATABASE_URL \ + {{ args }} \ + -- --all-targets --all-features) + done + [group('testing')] [group('sqlx')] @@ -29,7 +40,9 @@ sqlx-check: [group('testing')] [group('sqlx')] sqlx-migrate-run: _ensure_db_and_s3_are_running - cargo sqlx migrate run --database-url $DOCSRS_DATABASE_URL + cargo sqlx migrate run \ + --database-url $DOCSRS_DATABASE_URL \ + --source ./crates/lib/docs_rs_database/migrations [group('testing')] [group('sqlx')] @@ -37,6 +50,7 @@ sqlx-migrate-revert target="0": _ensure_db_and_s3_are_running # --target 0 means "revert everything" cargo sqlx migrate revert \ --database-url $DOCSRS_DATABASE_URL \ + --source ./crates/lib/docs_rs_database/migrations \ --target-version {{ target }} diff --git a/src/bin/cratesfyi.rs b/src/bin/cratesfyi.rs index 5596583be..b00ef4234 100644 --- a/src/bin/cratesfyi.rs +++ b/src/bin/cratesfyi.rs @@ -6,11 +6,11 @@ use docs_rs::{ db::{self, Overrides}, queue_rebuilds_faulty_rustdoc, start_web_server, utils::{ - ConfigName, daemon::start_background_service_metric_collector, get_config, - get_crate_pattern_and_priority, list_crate_priorities, queue_builder, - remove_crate_priority, set_config, set_crate_priority, + daemon::start_background_service_metric_collector, get_crate_pattern_and_priority, + list_crate_priorities, queue_builder, remove_crate_priority, set_crate_priority, }, }; +use docs_rs_database::service_config::{ConfigName, get_config, set_config}; use docs_rs_storage::add_path_into_database; use docs_rs_types::{CrateId, Version}; use futures_util::StreamExt; @@ -513,7 +513,7 @@ impl DatabaseSubcommand { .runtime .block_on(async { let mut conn = ctx.pool.get_async().await?; - db::migrate(&mut conn, version).await + docs_rs_database::migrate(&mut conn, version).await }) .context("Failed to run database migrations")?, diff --git a/src/build_queue.rs b/src/build_queue.rs index 5615ad3d1..104bd7674 100644 --- a/src/build_queue.rs +++ b/src/build_queue.rs @@ -3,12 +3,15 @@ use crate::{ db::{delete_crate, delete_version, update_latest_version_id}, docbuilder::{BuilderMetrics, PackageKind}, error::Result, - utils::{ConfigName, get_config, get_crate_priority, report_error, set_config}, + utils::{get_crate_priority, report_error}, }; use anyhow::Context as _; use chrono::NaiveDate; use crates_index_diff::{Change, CrateVersion}; -use docs_rs_database::{AsyncPoolClient, Pool}; +use docs_rs_database::{ + AsyncPoolClient, Pool, + service_config::{ConfigName, get_config, set_config}, +}; use docs_rs_fastly::{Cdn, CdnBehaviour as _}; use docs_rs_opentelemetry::AnyMeterProvider; use docs_rs_storage::AsyncStorage; diff --git a/src/db/mod.rs b/src/db/mod.rs index c2ce468b8..33a900fbc 100644 --- a/src/db/mod.rs +++ b/src/db/mod.rs @@ -1,7 +1,4 @@ //! Database operations -use anyhow::Result; -use sqlx::migrate::{Migrate, Migrator}; - pub use self::add_package::update_latest_version_id; pub(crate) use self::add_package::{ add_doc_coverage, finish_build, finish_release, initialize_build, initialize_crate, @@ -17,61 +14,3 @@ mod add_package; pub mod blacklist; pub mod delete; mod overrides; - -static MIGRATOR: Migrator = sqlx::migrate!(); - -pub async fn migrate(conn: &mut sqlx::PgConnection, target: Option) -> Result<()> { - conn.ensure_migrations_table().await?; - - // `database_versions` is the table that tracked the old `schemamama` migrations. - // If we find the table, and it contains records, we insert a fake record - // into the `_sqlx_migrations` table so the big initial migration isn't executed. - if sqlx::query( - "SELECT table_name - FROM information_schema.tables - WHERE table_schema = 'public' AND table_name = 'database_versions'", - ) - .fetch_optional(&mut *conn) - .await? - .is_some() - { - let max_version: Option = - sqlx::query_scalar("SELECT max(version) FROM database_versions") - .fetch_one(&mut *conn) - .await?; - - if max_version != Some(39) { - anyhow::bail!( - "database_versions table has unexpected version: {:?}", - max_version - ); - } - - sqlx::query( - "INSERT INTO _sqlx_migrations ( version, description, success, checksum, execution_time ) - VALUES ( $1, $2, TRUE, $3, -1 )", - ) - // the next two parameters relate to the filename of the initial migration file - .bind(20231021111635i64) - .bind("initial") - // this is the hash of the initial migration file, as sqlx requires it. - // if the initial migration file changes, this has to be updated with the new value, - // easiest to get from the `_sqlx_migrations` table when the migration was normally - // executed. - .bind(hex::decode("df802e0ec416063caadd1c06b13348cd885583c44962998886b929d5fe6ef3b70575d5101c5eb31daa989721df08d806").unwrap()) - .execute(&mut *conn) - .await?; - - sqlx::query("DROP TABLE database_versions") - .execute(&mut *conn) - .await?; - } - - // when we find records - if let Some(target) = target { - MIGRATOR.undo(conn, target).await?; - } else { - MIGRATOR.run(conn).await?; - } - Ok(()) -} diff --git a/src/docbuilder/rustwide_builder.rs b/src/docbuilder/rustwide_builder.rs index 86a6f734f..2df87de03 100644 --- a/src/docbuilder/rustwide_builder.rs +++ b/src/docbuilder/rustwide_builder.rs @@ -8,11 +8,14 @@ use crate::{ docbuilder::Limits, error::Result, metrics::{BUILD_TIME_HISTOGRAM_BUCKETS, DOCUMENTATION_SIZE_BUCKETS}, - utils::{ConfigName, copy_dir_all, get_config, report_error, set_config}, + utils::{copy_dir_all, report_error}, }; use anyhow::{Context as _, Error, anyhow, bail}; use docs_rs_cargo_metadata::{CargoMetadata, MetadataPackage}; -use docs_rs_database::Pool; +use docs_rs_database::{ + Pool, + service_config::{ConfigName, get_config, set_config}, +}; use docs_rs_opentelemetry::AnyMeterProvider; use docs_rs_registry_api::RegistryApi; use docs_rs_repository_stats::RepositoryStatsUpdater; diff --git a/src/test/fakes.rs b/src/test/fakes.rs index 5a0b864eb..5133647af 100644 --- a/src/test/fakes.rs +++ b/src/test/fakes.rs @@ -1,4 +1,3 @@ -use super::TestDatabase; use crate::{ db::{initialize_build, initialize_crate, initialize_release, update_build_status}, docbuilder::{DocCoverage, RUSTDOC_JSON_COMPRESSION_ALGORITHMS}, @@ -8,6 +7,7 @@ use anyhow::{Context, bail}; use base64::{Engine, engine::general_purpose::STANDARD as b64}; use chrono::{DateTime, Utc}; use docs_rs_cargo_metadata::{Dependency, MetadataPackage, Target}; +use docs_rs_database::testing::TestDatabase; use docs_rs_registry_api::{CrateData, CrateOwner, ReleaseData}; use docs_rs_storage::{ AsyncStorage, CompressionAlgorithm, FileEntry, RustdocJsonFormatVersion, diff --git a/src/test/mod.rs b/src/test/mod.rs index 81cc72a0d..75dae9758 100644 --- a/src/test/mod.rs +++ b/src/test/mod.rs @@ -5,7 +5,6 @@ pub(crate) use self::fakes::{FakeBuild, fake_release_that_failed_before_build}; use crate::{ AsyncBuildQueue, BuildQueue, Config, Context, config::ConfigBuilder, - db, error::Result, web::{build_axum_app, cache, page::TemplateData}, }; @@ -13,28 +12,22 @@ use anyhow::{Context as _, anyhow}; use axum::body::Bytes; use axum::{Router, body::Body, http::Request, response::Response as AxumResponse}; use axum_extra::headers::{ETag, HeaderMapExt as _}; -use docs_rs_database::{AsyncPoolClient, Pool}; +use docs_rs_database::testing::TestDatabase; use docs_rs_fastly::Cdn; use docs_rs_headers::{IfNoneMatch, SURROGATE_CONTROL, SurrogateKeys}; -use docs_rs_opentelemetry::{ - AnyMeterProvider, - testing::{CollectedMetrics, TestMetrics}, -}; +use docs_rs_opentelemetry::testing::{CollectedMetrics, TestMetrics}; use docs_rs_storage::{AsyncStorage, Storage, StorageKind, testing::TestStorage}; use docs_rs_types::Version; use fn_error_context::context; -use futures_util::stream::TryStreamExt; use http::{ HeaderMap, HeaderName, HeaderValue, StatusCode, header::{CACHE_CONTROL, CONTENT_TYPE}, }; use http_body_util::BodyExt; use serde::de::DeserializeOwned; -use sqlx::Connection as _; use std::{collections::HashMap, fs, future::Future, panic, rc::Rc, sync::Arc}; -use tokio::{runtime, task::block_in_place}; +use tokio::runtime; use tower::ServiceExt; -use tracing::error; // testing krate name constants pub(crate) const KRATE: &str = "krate"; @@ -453,8 +446,7 @@ impl TestEnvironment { fs::create_dir_all(config.registry_index_path.clone())?; let test_metrics = TestMetrics::new(); - - let test_db = TestDatabase::new(&config, test_metrics.provider()) + let test_db = TestDatabase::new(&config.database, test_metrics.provider()) .await .context("can't initialize test database")?; @@ -543,108 +535,3 @@ impl TestEnvironment { fakes::FakeRelease::new(self.async_db(), self.context.async_storage.clone()) } } - -#[derive(Debug)] -pub(crate) struct TestDatabase { - pool: Pool, - schema: String, - runtime: runtime::Handle, -} - -impl TestDatabase { - async fn new(config: &Config, otel_meter_provider: &AnyMeterProvider) -> Result { - // A random schema name is generated and used for the current connection. This allows each - // test to create a fresh instance of the database to run within. - let schema = format!("docs_rs_test_schema_{}", rand::random::()); - - let config = &config.database; - - let pool = Pool::new_with_schema(config, &schema, otel_meter_provider).await?; - - let mut conn = sqlx::PgConnection::connect(&config.database_url).await?; - sqlx::query(&format!("CREATE SCHEMA {schema}")) - .execute(&mut conn) - .await - .context("error creating schema")?; - sqlx::query(&format!("SET search_path TO {schema}, public")) - .execute(&mut conn) - .await - .context("error setting search path")?; - db::migrate(&mut conn, None) - .await - .context("error running migrations")?; - - // Move all sequence start positions 10000 apart to avoid overlapping primary keys - let sequence_names: Vec<_> = sqlx::query!( - "SELECT relname - FROM pg_class - INNER JOIN pg_namespace ON - pg_class.relnamespace = pg_namespace.oid - WHERE pg_class.relkind = 'S' - AND pg_namespace.nspname = $1 - ", - schema, - ) - .fetch(&mut conn) - .map_ok(|row| row.relname) - .try_collect() - .await?; - - for (i, sequence) in sequence_names.into_iter().enumerate() { - let offset = (i + 1) * 10000; - sqlx::query(&format!( - r#"ALTER SEQUENCE "{sequence}" RESTART WITH {offset};"# - )) - .execute(&mut conn) - .await?; - } - - Ok(TestDatabase { - pool, - schema, - runtime: runtime::Handle::current(), - }) - } - - pub(crate) fn pool(&self) -> &Pool { - &self.pool - } - - pub(crate) async fn async_conn(&self) -> AsyncPoolClient { - self.pool - .get_async() - .await - .expect("failed to get a connection out of the pool") - } -} - -impl Drop for TestDatabase { - fn drop(&mut self) { - let pool = self.pool.clone(); - let schema = self.schema.clone(); - let runtime = self.runtime.clone(); - - block_in_place(move || { - runtime.block_on(async move { - let Ok(mut conn) = pool.get_async().await else { - error!("error in drop impl"); - return; - }; - - let migration_result = db::migrate(&mut conn, Some(0)).await; - - if let Err(e) = sqlx::query(format!("DROP SCHEMA {} CASCADE;", schema).as_str()) - .execute(&mut *conn) - .await - { - error!("failed to drop test schema {}: {}", schema, e); - return; - } - - if let Err(err) = migration_result { - error!(?err, "error reverting migrations"); - } - }) - }); - } -} diff --git a/src/utils/mod.rs b/src/utils/mod.rs index e1c3fac50..70e8b6b83 100644 --- a/src/utils/mod.rs +++ b/src/utils/mod.rs @@ -17,8 +17,6 @@ mod html; mod queue; pub(crate) mod queue_builder; -use anyhow::Result; -use serde::{Serialize, de::DeserializeOwned}; use tracing::error; pub(crate) fn report_error(err: &anyhow::Error) { @@ -30,107 +28,3 @@ pub(crate) fn report_error(err: &anyhow::Error) { error!("{err:?}"); } } - -#[derive(strum::IntoStaticStr)] -#[strum(serialize_all = "snake_case")] -pub enum ConfigName { - RustcVersion, - LastSeenIndexReference, - QueueLocked, - Toolchain, -} - -pub async fn set_config( - conn: &mut sqlx::PgConnection, - name: ConfigName, - value: impl Serialize, -) -> anyhow::Result<()> { - let name: &'static str = name.into(); - sqlx::query!( - "INSERT INTO config (name, value) - VALUES ($1, $2) - ON CONFLICT (name) DO UPDATE SET value = $2;", - name, - &serde_json::to_value(value)?, - ) - .execute(conn) - .await?; - Ok(()) -} - -pub async fn get_config(conn: &mut sqlx::PgConnection, name: ConfigName) -> Result> -where - T: DeserializeOwned, -{ - let name: &'static str = name.into(); - Ok( - match sqlx::query!("SELECT value FROM config WHERE name = $1;", name) - .fetch_optional(conn) - .await? - { - Some(row) => serde_json::from_value(row.value)?, - None => None, - }, - ) -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::test::async_wrapper; - use serde_json::Value; - use test_case::test_case; - - #[test_case(ConfigName::RustcVersion, "rustc_version")] - #[test_case(ConfigName::QueueLocked, "queue_locked")] - #[test_case(ConfigName::LastSeenIndexReference, "last_seen_index_reference")] - fn test_configname_variants(variant: ConfigName, expected: &'static str) { - let name: &'static str = variant.into(); - assert_eq!(name, expected); - } - - #[test] - fn test_get_config_empty() { - async_wrapper(|env| async move { - let mut conn = env.async_db().async_conn().await; - sqlx::query!("DELETE FROM config") - .execute(&mut *conn) - .await?; - - assert!( - get_config::(&mut conn, ConfigName::RustcVersion) - .await? - .is_none() - ); - Ok(()) - }); - } - - #[test] - fn test_set_and_get_config_() { - async_wrapper(|env| async move { - let mut conn = env.async_db().async_conn().await; - sqlx::query!("DELETE FROM config") - .execute(&mut *conn) - .await?; - - assert!( - get_config::(&mut conn, ConfigName::RustcVersion) - .await? - .is_none() - ); - - set_config( - &mut conn, - ConfigName::RustcVersion, - Value::String("some value".into()), - ) - .await?; - assert_eq!( - get_config(&mut conn, ConfigName::RustcVersion).await?, - Some("some value".to_string()) - ); - Ok(()) - }); - } -} diff --git a/src/web/crate_details.rs b/src/web/crate_details.rs index 0efd2e391..c35ce78cc 100644 --- a/src/web/crate_details.rs +++ b/src/web/crate_details.rs @@ -723,10 +723,11 @@ mod tests { use super::*; use crate::db::update_build_status; use crate::test::{ - AxumResponseTestExt, AxumRouterTestExt, FakeBuild, TestDatabase, TestEnvironment, - async_wrapper, fake_release_that_failed_before_build, + AxumResponseTestExt, AxumRouterTestExt, FakeBuild, TestEnvironment, async_wrapper, + fake_release_that_failed_before_build, }; use anyhow::Error; + use docs_rs_database::testing::TestDatabase; use docs_rs_registry_api::CrateOwner; use docs_rs_types::KrateName; use http::StatusCode; diff --git a/src/web/mod.rs b/src/web/mod.rs index 28473a264..312cdf48d 100644 --- a/src/web/mod.rs +++ b/src/web/mod.rs @@ -628,9 +628,9 @@ mod test { use super::*; use crate::docbuilder::DocCoverage; use crate::test::{ - AxumResponseTestExt, AxumRouterTestExt, FakeBuild, TestDatabase, TestEnvironment, - async_wrapper, + AxumResponseTestExt, AxumRouterTestExt, FakeBuild, TestEnvironment, async_wrapper, }; + use docs_rs_database::testing::TestDatabase; use docs_rs_types::ReleaseId; use kuchikiki::traits::TendrilSink; use pretty_assertions::assert_eq; diff --git a/src/web/sitemap.rs b/src/web/sitemap.rs index faa14acd2..f4fe332a2 100644 --- a/src/web/sitemap.rs +++ b/src/web/sitemap.rs @@ -2,7 +2,7 @@ use crate::{ Config, docbuilder::Limits, impl_axum_webpage, - utils::{ConfigName, get_config, report_error}, + utils::report_error, web::{ AxumErrorPage, error::{AxumNope, AxumResult}, @@ -21,6 +21,7 @@ use axum::{ }; use axum_extra::{TypedHeader, headers::ContentType}; use chrono::{TimeZone, Utc}; +use docs_rs_database::service_config::{ConfigName, get_config}; use docs_rs_mimes as mimes; use futures_util::{StreamExt as _, pin_mut}; use std::sync::Arc;