extract registry api to subcrate

Author: syphar

Cargo.lock

@@ -46,11 +46,12 @@ tracing = "0.1.37"
 url = { version = "2.1.1", features = ["serde"] }
 
 [dependencies]
-docs_rs_fastly = { path = "crates/lib/docs_rs_fastly" }
 docs_rs_env_vars = { path = "crates/lib/docs_rs_env_vars" }
+docs_rs_fastly = { path = "crates/lib/docs_rs_fastly" }
 docs_rs_headers = { path = "crates/lib/docs_rs_headers" }
 docs_rs_logging = { path = "crates/lib/docs_rs_logging" }
 docs_rs_opentelemetry = { path = "crates/lib/docs_rs_opentelemetry" }
+docs_rs_registry_api = { path = "crates/lib/docs_rs_registry_api" }
 docs_rs_types = { path = "crates/lib/docs_rs_types" }
 docs_rs_uri = { path = "crates/lib/docs_rs_uri" }
 docs_rs_utils = { path = "crates/lib/docs_rs_utils" }

Cargo.toml

@@ -0,0 +1,17 @@
+[package]
+name = "docs_rs_registry_api"
+version = "0.1.0"
+edition = "2024"
+
+[dependencies]
+anyhow = { workspace = true }
+bincode = { workspace = true }
+chrono = { workspace = true }
+docs_rs_env_vars = { path = "../docs_rs_env_vars" }
+docs_rs_types = { path = "../docs_rs_types" }
+docs_rs_utils = { path = "../docs_rs_utils" }
+reqwest = { workspace = true }
+serde = { workspace = true }
+sqlx = { workspace = true }
+tracing = { workspace = true }
+url = { workspace = true }

crates/lib/docs_rs_registry_api/Cargo.toml

@@ -1,11 +1,13 @@
-use crate::{APP_USER_AGENT, error::Result};
-use anyhow::{Context, anyhow, bail};
+use crate::{
+    Config,
+    models::{CrateData, CrateOwner, OwnerKind, ReleaseData, Search, SearchCrate, SearchMeta},
+};
+use anyhow::{Context, Result, anyhow, bail};
 use chrono::{DateTime, Utc};
 use docs_rs_types::Version;
-use docs_rs_utils::retry_async;
+use docs_rs_utils::{APP_USER_AGENT, retry_async};
 use reqwest::header::{ACCEPT, HeaderValue, USER_AGENT};
-use serde::{Deserialize, Serialize};
-use std::fmt;
+use serde::Deserialize;
 use tracing::instrument;
 use url::Url;
 
@@ -16,84 +18,14 @@ pub struct RegistryApi {
     client: reqwest::Client,
 }
 
-#[derive(Debug)]
-pub struct CrateData {
-    pub(crate) owners: Vec<CrateOwner>,
-}
-
-#[derive(Debug)]
-pub(crate) struct ReleaseData {
-    pub(crate) release_time: DateTime<Utc>,
-    pub(crate) yanked: bool,
-    pub(crate) downloads: i32,
-}
-
-impl Default for ReleaseData {
-    fn default() -> ReleaseData {
-        ReleaseData {
-            release_time: Utc::now(),
-            yanked: false,
-            downloads: 0,
-        }
-    }
-}
-
-#[derive(Debug, Clone)]
-pub struct CrateOwner {
-    pub(crate) avatar: String,
-    pub(crate) login: String,
-    pub(crate) kind: OwnerKind,
-}
-
-#[derive(
-    Debug,
-    Clone,
-    Copy,
-    PartialEq,
-    Eq,
-    PartialOrd,
-    Ord,
-    Serialize,
-    Deserialize,
-    sqlx::Type,
-    bincode::Encode,
-)]
-#[sqlx(type_name = "owner_kind", rename_all = "lowercase")]
-#[serde(rename_all = "lowercase")]
-pub enum OwnerKind {
-    User,
-    Team,
-}
-
-impl fmt::Display for OwnerKind {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        match self {
-            Self::User => f.write_str("user"),
-            Self::Team => f.write_str("team"),
-        }
+impl RegistryApi {
+    pub fn from_config(config: &Config) -> Result<Self> {
+        Self::new(
+            config.registry_api_host.clone(),
+            config.crates_io_api_call_retries,
+        )
     }
-}
-
-#[derive(Deserialize, Debug)]
 
-pub(crate) struct SearchCrate {
-    pub(crate) name: String,
-}
-
-#[derive(Deserialize, Debug)]
-
-pub(crate) struct SearchMeta {
-    pub(crate) next_page: Option<String>,
-    pub(crate) prev_page: Option<String>,
-}
-
-#[derive(Deserialize, Debug)]
-pub(crate) struct Search {
-    pub(crate) crates: Vec<SearchCrate>,
-    pub(crate) meta: SearchMeta,
-}
-
-impl RegistryApi {
     pub fn new(api_base: Url, max_retries: u32) -> Result<Self> {
         let headers = vec![
             (USER_AGENT, HeaderValue::from_static(APP_USER_AGENT)),
@@ -124,11 +56,7 @@ impl RegistryApi {
     }
 
     #[instrument(skip(self))]
-    pub(crate) async fn get_release_data(
-        &self,
-        name: &str,
-        version: &Version,
-    ) -> Result<ReleaseData> {
+    pub async fn get_release_data(&self, name: &str, version: &Version) -> Result<ReleaseData> {
         let (release_time, yanked, downloads) = self
             .get_release_time_yanked_downloads(name, version)
             .await
@@ -256,7 +184,7 @@ impl RegistryApi {
     }
 
     /// Fetch crates from the registry's API
-    pub(crate) async fn search(&self, query_params: &str) -> Result<Search> {
+    pub async fn search(&self, query_params: &str) -> Result<Search> {
         #[derive(Deserialize, Debug)]
         struct SearchError {
             detail: String,

src/registry_api.rs …ates/lib/docs_rs_registry_api/src/api.rssrc/registry_api.rs renamed to crates/lib/docs_rs_registry_api/src/api.rs src/registry_api.rs renamed to crates/lib/docs_rs_registry_api/src/api.rs

@@ -0,0 +1,22 @@
+use docs_rs_env_vars::env;
+use url::Url;
+
+#[derive(Debug)]
+pub struct Config {
+    pub registry_api_host: Url,
+
+    // amount of retries for external API calls, mostly crates.io
+    pub crates_io_api_call_retries: u32,
+}
+
+impl Config {
+    pub fn from_environment() -> anyhow::Result<Self> {
+        Ok(Self {
+            crates_io_api_call_retries: env("DOCSRS_CRATESIO_API_CALL_RETRIES", 3u32)?,
+            registry_api_host: env(
+                "DOCSRS_REGISTRY_API_HOST",
+                "https://crates.io".parse().unwrap(),
+            )?,
+        })
+    }
+}

crates/lib/docs_rs_registry_api/src/config.rs

@@ -0,0 +1,7 @@
+mod api;
+mod config;
+mod models;
+
+pub use api::RegistryApi;
+pub use config::Config;
+pub use models::{CrateData, CrateOwner, OwnerKind, ReleaseData, Search};

crates/lib/docs_rs_registry_api/src/lib.rs

@@ -0,0 +1,78 @@
+use chrono::{DateTime, Utc};
+use serde::{Deserialize, Serialize};
+use std::fmt;
+
+#[derive(Debug)]
+pub struct CrateData {
+    pub owners: Vec<CrateOwner>,
+}
+
+#[derive(Debug)]
+pub struct ReleaseData {
+    pub release_time: DateTime<Utc>,
+    pub yanked: bool,
+    pub downloads: i32,
+}
+
+impl Default for ReleaseData {
+    fn default() -> ReleaseData {
+        ReleaseData {
+            release_time: Utc::now(),
+            yanked: false,
+            downloads: 0,
+        }
+    }
+}
+
+#[derive(Debug, Clone)]
+pub struct CrateOwner {
+    pub avatar: String,
+    pub login: String,
+    pub kind: OwnerKind,
+}
+
+#[derive(
+    Debug,
+    Clone,
+    Copy,
+    PartialEq,
+    Eq,
+    PartialOrd,
+    Ord,
+    Serialize,
+    Deserialize,
+    sqlx::Type,
+    bincode::Encode,
+)]
+#[sqlx(type_name = "owner_kind", rename_all = "lowercase")]
+#[serde(rename_all = "lowercase")]
+pub enum OwnerKind {
+    User,
+    Team,
+}
+
+impl fmt::Display for OwnerKind {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match self {
+            Self::User => f.write_str("user"),
+            Self::Team => f.write_str("team"),
+        }
+    }
+}
+
+#[derive(Deserialize, Debug)]
+pub struct SearchCrate {
+    pub name: String,
+}
+
+#[derive(Deserialize, Debug)]
+pub struct SearchMeta {
+    pub next_page: Option<String>,
+    pub prev_page: Option<String>,
+}
+
+#[derive(Deserialize, Debug)]
+pub struct Search {
+    pub crates: Vec<SearchCrate>,
+    pub meta: SearchMeta,
+}

crates/lib/docs_rs_registry_api/src/models.rs

@@ -6,15 +6,13 @@ use std::{
     path::{self, Path, PathBuf},
     time::Duration,
 };
-use url::Url;
 
 #[derive(Debug, derive_builder::Builder)]
 #[builder(pattern = "owned")]
 pub struct Config {
     pub prefix: PathBuf,
     pub registry_index_path: PathBuf,
     pub registry_url: Option<String>,
-    pub registry_api_host: Url,
 
     /// How long to wait between registry checks
     pub(crate) delay_between_registry_fetches: Duration,
@@ -53,9 +51,6 @@ pub struct Config {
     // constant_time_eq for comparisons!)
     pub(crate) cratesio_token: Option<String>,
 
-    // amount of retries for external API calls, mostly crates.io
-    pub crates_io_api_call_retries: u32,
-
     // request timeout in seconds
     pub(crate) request_timeout: Option<Duration>,
     pub(crate) report_request_timeouts: bool,
@@ -135,6 +130,7 @@ pub struct Config {
 
     pub(crate) fastly: docs_rs_fastly::Config,
     pub(crate) opentelemetry: docs_rs_opentelemetry::Config,
+    pub(crate) registry_api: docs_rs_registry_api::Config,
 }
 
 impl Config {
@@ -171,13 +167,8 @@ impl Config {
                 "DOCSRS_DELAY_BETWEEN_REGISTRY_FETCHES",
                 60,
             )?))
-            .crates_io_api_call_retries(env("DOCSRS_CRATESIO_API_CALL_RETRIES", 3u32)?)
             .registry_index_path(env("REGISTRY_INDEX_PATH", prefix.join("crates.io-index"))?)
             .registry_url(maybe_env("REGISTRY_URL")?)
-            .registry_api_host(env(
-                "DOCSRS_REGISTRY_API_HOST",
-                "https://crates.io".parse().unwrap(),
-            )?)
             .prefix(prefix.clone())
             .database_url(require_env("DOCSRS_DATABASE_URL")?)
             .max_pool_size(env("DOCSRS_MAX_POOL_SIZE", 90u32)?)
@@ -237,7 +228,8 @@ impl Config {
                 docs_rs_fastly::Config::from_environment()
                     .context("error reading fastly config from environment")?,
             )
-            .opentelemetry(docs_rs_opentelemetry::Config::from_environment()?))
+            .opentelemetry(docs_rs_opentelemetry::Config::from_environment()?)
+            .registry_api(docs_rs_registry_api::Config::from_environment()?))
     }
 
     pub fn max_file_size_for(&self, path: impl AsRef<Path>) -> usize {

src/config.rs

@@ -1,10 +1,11 @@
 use crate::{
-    AsyncBuildQueue, AsyncStorage, BuildQueue, Config, RegistryApi, Storage, db::Pool,
+    AsyncBuildQueue, AsyncStorage, BuildQueue, Config, Storage, db::Pool,
     repositories::RepositoryStatsUpdater,
 };
 use anyhow::Result;
 use docs_rs_fastly::Cdn;
 use docs_rs_opentelemetry::{AnyMeterProvider, get_meter_provider};
+use docs_rs_registry_api::RegistryApi;
 use std::sync::Arc;
 use tokio::runtime;
 
@@ -82,10 +83,7 @@ impl Context {
             async_storage,
             cdn,
             pool: pool.clone(),
-            registry_api: Arc::new(RegistryApi::new(
-                config.registry_api_host.clone(),
-                config.crates_io_api_call_retries,
-            )?),
+            registry_api: Arc::new(RegistryApi::from_config(&config.registry_api)?),
             repository_stats_updater: Arc::new(RepositoryStatsUpdater::new(&config, pool)),
             runtime,
             config,