extract opentelemetry logic into subcrate

Author: syphar

Cargo.lock

@@ -22,10 +22,17 @@ exclude = [
 
 [workspace.dependencies]
 anyhow = { version = "1.0.42", features = ["backtrace"]}
+derive_more = { version = "2.0.0", features = ["display", "deref", "from", "into", "from_str"] }
+opentelemetry = "0.31.0"
+opentelemetry-otlp = { version = "0.31.0", features = ["grpc-tonic",  "metrics"] }
+opentelemetry-resource-detectors = "0.10.0"
+opentelemetry_sdk = { version = "0.31.0", features = ["rt-tokio"] }
 tracing = "0.1.37"
+url = { version = "2.1.1", features = ["serde"] }
 
 [dependencies]
 docs_rs_env_vars = { path = "crates/lib/docs_rs_env_vars" }
+docs_rs_opentelemetry = { path = "crates/lib/docs_rs_opentelemetry" }
 sentry = { version = "0.46.0", features = ["panic", "tracing", "tower-http", "anyhow", "backtrace"] }
 log = "0.4"
 tracing = { workspace = true }
@@ -41,17 +48,15 @@ reqwest = { version = "0.12", features = ["json", "gzip"] }
 semver = { version = "1.0.4", features = ["serde"] }
 slug = "0.1.1"
 sqlx = { version = "0.8", features = [ "runtime-tokio", "postgres", "sqlite", "chrono" ] }
-url = { version = "2.1.1", features = ["serde"] }
+url = { workspace = true }
 docsrs-metadata = { path = "crates/lib/metadata" }
 anyhow = { workspace = true }
 thiserror = "2.0.3"
 comrak = { version = "0.49.0", default-features = false }
 syntect = { version = "5.0.0", default-features = false, features = ["parsing", "html", "dump-load", "regex-onig"] }
 toml = "0.9.2"
-opentelemetry = "0.31.0"
-opentelemetry-otlp = { version = "0.31.0", features = ["grpc-tonic",  "metrics"] }
-opentelemetry-resource-detectors = "0.10.0"
-opentelemetry_sdk = { version = "0.31.0", features = ["rt-tokio"] }
+opentelemetry = { workspace = true }
+opentelemetry_sdk = { workspace = true }
 rustwide = { version = "0.21.0", features = ["unstable-toolchain-ci", "unstable"] }
 mime_guess = "2"
 zstd = "0.13.0"
@@ -68,7 +73,7 @@ bzip2 = "0.6.0"
 getrandom = "0.3.1"
 itertools = { version = "0.14.0" }
 hex = "0.4.3"
-derive_more = { version = "2.0.0", features = ["display", "deref", "from", "into", "from_str"] }
+derive_more = { workspace = true }
 sysinfo = { version = "0.37.2", default-features = false, features = ["system"] }
 derive_builder = "0.20.2"
 
@@ -117,6 +122,7 @@ md5 = "0.8.0"
 crates_io_validation = { path = "crates/lib/crates_io_validation" }
 
 [dev-dependencies]
+docs_rs_opentelemetry = { path = "crates/lib/docs_rs_opentelemetry", features = ["testing"] }
 criterion = "0.8.0"
 kuchikiki = "0.8"
 http-body-util = "0.1.0"

Cargo.toml

@@ -0,0 +1,18 @@
+[package]
+name = "docs_rs_opentelemetry"
+version = "0.1.0"
+edition = "2024"
+
+[dependencies]
+anyhow = { workspace = true }
+derive_more = { workspace = true, optional = true }
+docs_rs_env_vars = { path = "../docs_rs_env_vars" }
+opentelemetry = { workspace = true }
+opentelemetry-otlp = { version = "0.31.0", features = ["grpc-tonic",  "metrics"] }
+opentelemetry-resource-detectors = "0.10.0"
+opentelemetry_sdk = { workspace = true }
+tracing = { workspace = true }
+url = { workspace = true }
+
+[features]
+testing = ["dep:derive_more"]

crates/lib/docs_rs_opentelemetry/Cargo.toml

@@ -0,0 +1,16 @@
+use docs_rs_env_vars::maybe_env;
+use url::Url;
+
+#[derive(Debug)]
+pub struct Config {
+    // opentelemetry endpoint to send OTLP to
+    pub endpoint: Option<Url>,
+}
+
+impl Config {
+    pub fn from_environment() -> anyhow::Result<Self> {
+        Ok(Self {
+            endpoint: maybe_env("OTEL_EXPORTER_OTLP_ENDPOINT")?,
+        })
+    }
+}

crates/lib/docs_rs_opentelemetry/src/config.rs

@@ -1,4 +1,8 @@
-use crate::Config;
+mod config;
+#[cfg(feature = "testing")]
+pub mod testing;
+pub use config::Config;
+
 use anyhow::Result;
 use opentelemetry::{
     InstrumentationScope,
@@ -26,8 +30,8 @@ impl MeterProviderWithExt for opentelemetry_sdk::metrics::SdkMeterProvider {
 
 /// opentelemetry metric provider setup,
 /// if no endpoint is configured, use a no-op provider
-pub(crate) fn get_meter_provider(config: &Config) -> Result<AnyMeterProvider> {
-    if let Some(ref endpoint) = config.opentelemetry_endpoint {
+pub fn get_meter_provider(config: &config::Config) -> Result<AnyMeterProvider> {
+    if let Some(ref endpoint) = config.endpoint {
         let endpoint = endpoint.to_string();
         info!(endpoint, "setting up OpenTelemetry metrics exporter");
 
@@ -61,7 +65,7 @@ pub(crate) fn get_meter_provider(config: &Config) -> Result<AnyMeterProvider> {
 /// For now, copy/paste from opentelemetry-sdk, see
 /// https://github.com/open-telemetry/opentelemetry-rust/pull/3111
 #[derive(Debug, Default)]
-pub(crate) struct NoopMeterProvider {
+pub struct NoopMeterProvider {
     _private: (),
 }
 
@@ -86,13 +90,13 @@ impl MeterProviderWithExt for NoopMeterProvider {
 
 /// A no-op instance of a `Meter`
 #[derive(Debug, Default)]
-pub(crate) struct NoopMeter {
+pub struct NoopMeter {
     _private: (),
 }
 
 impl NoopMeter {
     /// Create a new no-op meter core.
-    pub(crate) fn new() -> Self {
+    pub fn new() -> Self {
         NoopMeter { _private: () }
     }
 }

src/metrics/otel.rs …tes/lib/docs_rs_opentelemetry/src/lib.rssrc/metrics/otel.rs renamed to crates/lib/docs_rs_opentelemetry/src/lib.rs src/metrics/otel.rs renamed to crates/lib/docs_rs_opentelemetry/src/lib.rs

@@ -1,5 +1,6 @@
 use std::sync::Arc;
 
+use crate::AnyMeterProvider;
 use anyhow::{Result, anyhow};
 use derive_more::Deref;
 use opentelemetry_sdk::metrics::{
@@ -9,11 +10,9 @@ use opentelemetry_sdk::metrics::{
     },
 };
 
-use crate::metrics::otel::AnyMeterProvider;
-
 /// set up a standalone InMemoryMetricExporter and MeterProvider for testing purposes.
 /// For when you want to collect metrics, and then inspect what was collected.
-pub(crate) fn setup_test_meter_provider() -> (InMemoryMetricExporter, AnyMeterProvider) {
+pub fn setup_test_meter_provider() -> (InMemoryMetricExporter, AnyMeterProvider) {
     let metric_exporter = InMemoryMetricExporter::default();
 
     (
@@ -29,10 +28,10 @@ pub(crate) fn setup_test_meter_provider() -> (InMemoryMetricExporter, AnyMeterPr
 /// small wrapper around the collected result of the InMemoryMetricExporter.
 /// For convenience in tests.
 #[derive(Debug)]
-pub(crate) struct CollectedMetrics(pub(crate) Vec<ResourceMetrics>);
+pub struct CollectedMetrics(pub Vec<ResourceMetrics>);
 
 impl CollectedMetrics {
-    pub(crate) fn get_metric<'a>(
+    pub fn get_metric<'a>(
         &'a self,
         scope: impl AsRef<str>,
         name: impl AsRef<str>,
@@ -70,10 +69,10 @@ impl CollectedMetrics {
 }
 
 #[derive(Debug, Deref)]
-pub(crate) struct CollectedMetric<'a>(&'a Metric);
+pub struct CollectedMetric<'a>(&'a Metric);
 
 impl<'a> CollectedMetric<'a> {
-    pub(crate) fn get_u64_counter(&'a self) -> &'a SumDataPoint<u64> {
+    pub fn get_u64_counter(&'a self) -> &'a SumDataPoint<u64> {
         let AggregatedMetrics::U64(metric_data) = self.data() else {
             panic!("Expected U64 metric data, got: {:?}", self.data());
         };
@@ -93,7 +92,7 @@ impl<'a> CollectedMetric<'a> {
         result
     }
 
-    pub(crate) fn get_f64_histogram(&'a self) -> &'a HistogramDataPoint<f64> {
+    pub fn get_f64_histogram(&'a self) -> &'a HistogramDataPoint<f64> {
         let AggregatedMetrics::F64(metric_data) = self.data() else {
             panic!("Expected F64 metric data, got: {:?}", self.data());
         };

src/test/test_metrics.rs …lib/docs_rs_opentelemetry/src/testing.rssrc/test/test_metrics.rs renamed to crates/lib/docs_rs_opentelemetry/src/testing.rs src/test/test_metrics.rs renamed to crates/lib/docs_rs_opentelemetry/src/testing.rs

@@ -9,13 +9,13 @@ use crate::{
     },
     docbuilder::{BuilderMetrics, PackageKind},
     error::Result,
-    metrics::otel::AnyMeterProvider,
     storage::AsyncStorage,
     utils::{ConfigName, get_config, get_crate_priority, report_error, retry, set_config},
 };
 use anyhow::Context as _;
 use chrono::NaiveDate;
 use crates_index_diff::{Change, CrateVersion};
+use docs_rs_opentelemetry::AnyMeterProvider;
 use fn_error_context::context;
 use futures_util::{StreamExt, stream::TryStreamExt};
 use opentelemetry::metrics::Counter;

src/build_queue.rs

@@ -175,8 +175,9 @@ where
 #[cfg(test)]
 mod tests {
     use super::*;
-    use crate::test::{TestEnvironment, setup_test_meter_provider};
+    use crate::test::TestEnvironment;
     use chrono::TimeZone;
+    use docs_rs_opentelemetry::testing::setup_test_meter_provider;
     use std::str::FromStr as _;
 
     #[test]

src/cdn/fastly.rs

@@ -1,8 +1,6 @@
-use crate::{
-    Config, db::types::krate_name::KrateName, metrics::otel::AnyMeterProvider,
-    web::headers::SurrogateKey,
-};
+use crate::{Config, db::types::krate_name::KrateName, web::headers::SurrogateKey};
 use anyhow::Result;
+use docs_rs_opentelemetry::AnyMeterProvider;
 use opentelemetry::metrics::{Counter, Gauge};
 use tracing::{error, info, instrument};
 

src/cdn/mod.rs

@@ -142,8 +142,7 @@ pub struct Config {
     // automatic rebuild configuration
     pub(crate) max_queued_rebuilds: Option<u16>,
 
-    // opentelemetry endpoint to send OTLP to
-    pub(crate) opentelemetry_endpoint: Option<Url>,
+    pub(crate) opentelemetry: docs_rs_opentelemetry::Config,
 }
 
 impl Config {
@@ -187,7 +186,6 @@ impl Config {
                 "DOCSRS_REGISTRY_API_HOST",
                 "https://crates.io".parse().unwrap(),
             )?)
-            .opentelemetry_endpoint(maybe_env("OTEL_EXPORTER_OTLP_ENDPOINT")?)
             .prefix(prefix.clone())
             .database_url(require_env("DOCSRS_DATABASE_URL")?)
             .max_pool_size(env("DOCSRS_MAX_POOL_SIZE", 90u32)?)
@@ -248,7 +246,8 @@ impl Config {
                 "DOCSRS_BUILD_WORKSPACE_REINITIALIZATION_INTERVAL",
                 86400,
             )?))
-            .max_queued_rebuilds(maybe_env("DOCSRS_MAX_QUEUED_REBUILDS")?))
+            .max_queued_rebuilds(maybe_env("DOCSRS_MAX_QUEUED_REBUILDS")?)
+            .opentelemetry(docs_rs_opentelemetry::Config::from_environment()?))
     }
 
     pub fn max_file_size_for(&self, path: impl AsRef<Path>) -> usize {