extract cargo-metadata logic into subcrate

Author: syphar

Cargo.lock

@@ -46,6 +46,7 @@ tracing = "0.1.37"
 url = { version = "2.1.1", features = ["serde"] }
 
 [dependencies]
+docs_rs_cargo_metadata = { path = "crates/lib/docs_rs_cargo_metadata" }
 docs_rs_env_vars = { path = "crates/lib/docs_rs_env_vars" }
 docs_rs_fastly = { path = "crates/lib/docs_rs_fastly" }
 docs_rs_headers = { path = "crates/lib/docs_rs_headers" }
@@ -138,6 +139,7 @@ chrono = { workspace = true }
 constant_time_eq = "0.4.2"
 
 [dev-dependencies]
+docs_rs_cargo_metadata = { path = "crates/lib/docs_rs_cargo_metadata", features = ["testing"] }
 docs_rs_fastly = { path = "crates/lib/docs_rs_fastly", features = ["testing"] }
 docs_rs_headers = { path = "crates/lib/docs_rs_headers", features = ["testing"] }
 docs_rs_opentelemetry = { path = "crates/lib/docs_rs_opentelemetry", features = ["testing"] }

Cargo.toml

@@ -0,0 +1,18 @@
+[package]
+name = "docs_rs_cargo_metadata"
+version = "0.1.0"
+edition = "2024"
+
+[dependencies]
+anyhow = { workspace = true }
+bincode = { workspace = true }
+derive_more = { workspace = true }
+docs_rs_types = { path = "../docs_rs_types" }
+serde = { workspace = true }
+serde_json = { workspace = true }
+
+[dev-dependencies]
+test-case = { workspace = true }
+
+[features]
+testing = []

crates/lib/docs_rs_cargo_metadata/Cargo.toml

@@ -0,0 +1,5 @@
+mod metadata;
+mod release_dependency;
+
+pub use metadata::{CargoMetadata, Dependency, Package as MetadataPackage, Target};
+pub use release_dependency::{ReleaseDependency, ReleaseDependencyList};

crates/lib/docs_rs_cargo_metadata/src/lib.rs

@@ -1,47 +1,28 @@
-use crate::{error::Result, web::extractors::rustdoc::RustdocParams};
-use anyhow::{Context, bail};
-use docs_rs_types::{KrateName, ReqVersion, Version, VersionReq};
-use rustwide::{Toolchain, Workspace, cmd::Command};
+use anyhow::{Context, Result};
+use docs_rs_types::{Version, VersionReq};
 use serde::{Deserialize, Serialize};
 use std::collections::HashMap;
-use std::path::Path;
 
-pub(crate) struct CargoMetadata {
+pub struct CargoMetadata {
     root: Package,
 }
 
 impl CargoMetadata {
-    pub(crate) fn load_from_rustwide(
-        workspace: &Workspace,
-        toolchain: &Toolchain,
-        source_dir: &Path,
-    ) -> Result<Self> {
-        let res = Command::new(workspace, toolchain.cargo())
-            .args(&["metadata", "--format-version", "1"])
-            .cd(source_dir)
-            .log_output(false)
-            .run_capture()?;
-        let [metadata] = res.stdout_lines() else {
-            bail!("invalid output returned by `cargo metadata`")
-        };
-        Self::load_from_metadata(metadata)
-    }
-
-    #[cfg(test)]
-    pub(crate) fn load_from_host_path(source_dir: &Path) -> Result<Self> {
+    #[cfg(feature = "testing")]
+    pub fn load_from_host_path(source_dir: &std::path::Path) -> Result<Self> {
         let res = std::process::Command::new("cargo")
             .args(["metadata", "--format-version", "1", "--offline"])
             .current_dir(source_dir)
             .output()?;
         let status = res.status;
         if !status.success() {
             let stderr = std::str::from_utf8(&res.stderr).unwrap_or("");
-            bail!("error returned by `cargo metadata`: {status}\n{stderr}")
+            anyhow::bail!("error returned by `cargo metadata`: {status}\n{stderr}")
         }
         Self::load_from_metadata(std::str::from_utf8(&res.stdout)?)
     }
 
-    pub(crate) fn load_from_metadata(metadata: &str) -> Result<Self> {
+    pub fn load_from_metadata(metadata: &str) -> Result<Self> {
         let metadata = serde_json::from_str::<DeserializedMetadata>(metadata)?;
         let root = metadata.resolve.root;
         Ok(CargoMetadata {
@@ -53,26 +34,26 @@ impl CargoMetadata {
         })
     }
 
-    pub(crate) fn root(&self) -> &Package {
+    pub fn root(&self) -> &Package {
         &self.root
     }
 }
 
 #[derive(Debug, Deserialize, Serialize)]
-pub(crate) struct Package {
-    pub(crate) id: String,
-    pub(crate) name: String,
-    pub(crate) version: Version,
-    pub(crate) license: Option<String>,
-    pub(crate) repository: Option<String>,
-    pub(crate) homepage: Option<String>,
-    pub(crate) description: Option<String>,
-    pub(crate) documentation: Option<String>,
-    pub(crate) dependencies: Vec<Dependency>,
-    pub(crate) targets: Vec<Target>,
-    pub(crate) readme: Option<String>,
-    pub(crate) keywords: Vec<String>,
-    pub(crate) features: HashMap<String, Vec<String>>,
+pub struct Package {
+    pub id: String,
+    pub name: String,
+    pub version: Version,
+    pub license: Option<String>,
+    pub repository: Option<String>,
+    pub homepage: Option<String>,
+    pub description: Option<String>,
+    pub documentation: Option<String>,
+    pub dependencies: Vec<Dependency>,
+    pub targets: Vec<Target>,
+    pub readme: Option<String>,
+    pub keywords: Vec<String>,
+    pub features: HashMap<String, Vec<String>>,
 }
 
 impl Package {
@@ -82,15 +63,15 @@ impl Package {
             .find(|target| target.crate_types.iter().any(|kind| kind != "bin"))
     }
 
-    pub(crate) fn is_library(&self) -> bool {
+    pub fn is_library(&self) -> bool {
         self.library_target().is_some()
     }
 
     fn normalize_package_name(&self, name: &str) -> String {
         name.replace('-', "_")
     }
 
-    pub(crate) fn package_name(&self) -> String {
+    pub fn package_name(&self) -> String {
         self.library_name().unwrap_or_else(|| {
             self.targets
                 .first()
@@ -99,25 +80,25 @@ impl Package {
         })
     }
 
-    pub(crate) fn library_name(&self) -> Option<String> {
+    pub fn library_name(&self) -> Option<String> {
         self.library_target()
             .map(|target| self.normalize_package_name(&target.name))
     }
 }
 
 #[derive(Debug, Deserialize, Serialize)]
-pub(crate) struct Target {
-    pub(crate) name: String,
-    #[cfg(not(test))]
+pub struct Target {
+    pub name: String,
+    #[cfg(not(feature = "testing"))]
     crate_types: Vec<String>,
-    #[cfg(test)]
-    pub(crate) crate_types: Vec<String>,
-    pub(crate) src_path: Option<String>,
+    #[cfg(feature = "testing")]
+    pub crate_types: Vec<String>,
+    pub src_path: Option<String>,
 }
 
 impl Target {
-    #[cfg(test)]
-    pub(crate) fn dummy_lib(name: String, src_path: Option<String>) -> Self {
+    #[cfg(feature = "testing")]
+    pub fn dummy_lib(name: String, src_path: Option<String>) -> Self {
         Target {
             name,
             crate_types: vec!["lib".into()],
@@ -127,25 +108,12 @@ impl Target {
 }
 
 #[derive(Debug, Clone, Deserialize, Serialize, PartialEq)]
-pub(crate) struct Dependency {
-    pub(crate) name: String,
-    pub(crate) req: VersionReq,
-    pub(crate) kind: Option<String>,
-    pub(crate) rename: Option<String>,
-    pub(crate) optional: bool,
-}
-
-impl Dependency {
-    pub(crate) fn rustdoc_params(&self) -> RustdocParams {
-        RustdocParams::new(
-            // I validated in the database, which makes me assume that renames are
-            // handled before storing the deps into the column.
-            self.name
-                .parse::<KrateName>()
-                .expect("we validated that the dep name is always a valid KrateName"),
-        )
-        .with_req_version(ReqVersion::Semver(self.req.clone()))
-    }
+pub struct Dependency {
+    pub name: String,
+    pub req: VersionReq,
+    pub kind: Option<String>,
+    pub rename: Option<String>,
+    pub optional: bool,
 }
 
 impl bincode::Encode for Dependency {
@@ -172,7 +140,7 @@ impl bincode::Encode for Dependency {
 }
 
 impl Dependency {
-    #[cfg(test)]
+    #[cfg(feature = "testing")]
     pub fn new(name: String, req: VersionReq) -> Dependency {
         Dependency {
             name,
@@ -183,7 +151,7 @@ impl Dependency {
         }
     }
 
-    #[cfg(test)]
+    #[cfg(feature = "testing")]
     pub fn set_optional(mut self, optional: bool) -> Self {
         self.optional = optional;
         self

src/utils/cargo_metadata.rs …b/docs_rs_cargo_metadata/src/metadata.rssrc/utils/cargo_metadata.rs renamed to crates/lib/docs_rs_cargo_metadata/src/metadata.rs src/utils/cargo_metadata.rs renamed to crates/lib/docs_rs_cargo_metadata/src/metadata.rs

@@ -1,4 +1,4 @@
-use crate::utils::Dependency;
+use super::Dependency;
 use derive_more::Deref;
 use docs_rs_types::VersionReq;
 use serde::{Deserialize, Serialize};
@@ -7,7 +7,7 @@ const DEFAULT_KIND: &str = "normal";
 
 /// A crate dependency in our internal representation for releases.dependencies json.
 #[derive(Debug, Clone, PartialEq, Deref)]
-pub(crate) struct ReleaseDependency(Dependency);
+pub struct ReleaseDependency(Dependency);
 
 impl<'de> Deserialize<'de> for ReleaseDependency {
     fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
@@ -67,7 +67,7 @@ impl From<ReleaseDependency> for Dependency {
     }
 }
 
-pub(crate) type ReleaseDependencyList = Vec<ReleaseDependency>;
+pub type ReleaseDependencyList = Vec<ReleaseDependency>;
 
 #[cfg(test)]
 mod tests {

src/db/types/dependencies.rs …cargo_metadata/src/release_dependency.rssrc/db/types/dependencies.rs renamed to crates/lib/docs_rs_cargo_metadata/src/release_dependency.rs src/db/types/dependencies.rs renamed to crates/lib/docs_rs_cargo_metadata/src/release_dependency.rs

@@ -1,12 +1,11 @@
 use crate::{
-    db::types::dependencies::ReleaseDependencyList,
     docbuilder::DocCoverage,
     error::Result,
     storage::CompressionAlgorithm,
-    utils::MetadataPackage,
     web::crate_details::{latest_release, releases_for_crate},
 };
 use anyhow::{Context, anyhow};
+use docs_rs_cargo_metadata::{MetadataPackage, ReleaseDependencyList};
 use docs_rs_registry_api::{CrateData, CrateOwner, ReleaseData};
 use docs_rs_types::{BuildId, BuildStatus, CrateId, Feature, ReleaseId, Version};
 use docs_rs_utils::rustc_version::parse_rustc_date;
@@ -623,8 +622,8 @@ where
 mod test {
     use super::*;
     use crate::test::*;
-    use crate::utils::CargoMetadata;
     use chrono::NaiveDate;
+    use docs_rs_cargo_metadata::CargoMetadata;
     use docs_rs_registry_api::OwnerKind;
     use std::slice;
     use test_case::test_case;

src/db/add_package.rs

@@ -22,7 +22,6 @@ pub(crate) mod file;
 pub(crate) mod mimes;
 mod overrides;
 mod pool;
-pub mod types;
 
 static MIGRATOR: Migrator = sqlx::migrate!();
 

src/db/mod.rs

@@ -15,12 +15,10 @@ use crate::{
         CompressionAlgorithm, RustdocJsonFormatVersion, compress, get_file_list,
         rustdoc_archive_path, rustdoc_json_path, source_archive_path,
     },
-    utils::{
-        CargoMetadata, ConfigName, MetadataPackage, copy_dir_all, get_config, report_error,
-        set_config,
-    },
+    utils::{ConfigName, copy_dir_all, get_config, report_error, set_config},
 };
 use anyhow::{Context as _, Error, anyhow, bail};
+use docs_rs_cargo_metadata::{CargoMetadata, MetadataPackage};
 use docs_rs_opentelemetry::AnyMeterProvider;
 use docs_rs_registry_api::RegistryApi;
 use docs_rs_types::{BuildId, BuildStatus, CrateId, ReleaseId, Version};
@@ -110,6 +108,22 @@ fn build_workspace(context: &Context) -> Result<Workspace> {
     Ok(workspace)
 }
 
+fn load_metadata_from_rustwide(
+    workspace: &Workspace,
+    toolchain: &Toolchain,
+    source_dir: &Path,
+) -> Result<CargoMetadata> {
+    let res = Command::new(workspace, toolchain.cargo())
+        .args(&["metadata", "--format-version", "1"])
+        .cd(source_dir)
+        .log_output(false)
+        .run_capture()?;
+    let [metadata] = res.stdout_lines() else {
+        bail!("invalid output returned by `cargo metadata`")
+    };
+    CargoMetadata::load_from_metadata(metadata)
+}
+
 #[derive(Debug)]
 pub enum PackageKind<'a> {
     Local(&'a Path),
@@ -529,7 +543,7 @@ impl RustwideBuilder {
     }
 
     pub fn build_local_package(&mut self, path: &Path) -> Result<BuildPackageSummary> {
-        let metadata = CargoMetadata::load_from_rustwide(&self.workspace, &self.toolchain, path)
+        let metadata = load_metadata_from_rustwide(&self.workspace, &self.toolchain, path)
             .map_err(|err| {
                 err.context(format!("failed to load local package {}", path.display()))
             })?;
@@ -1146,7 +1160,7 @@ impl RustwideBuilder {
         create_essential_files: bool,
         collect_metrics: bool,
     ) -> Result<FullBuildResult> {
-        let cargo_metadata = CargoMetadata::load_from_rustwide(
+        let cargo_metadata = load_metadata_from_rustwide(
             &self.workspace,
             &self.toolchain,
             &build.host_source_dir(),