diff --git a/quickwit/Cargo.lock b/quickwit/Cargo.lock index c8294c03eb6..de55281a520 100644 --- a/quickwit/Cargo.lock +++ b/quickwit/Cargo.lock @@ -5580,6 +5580,18 @@ dependencies = [ "tracing", ] +[[package]] +name = "opentelemetry-appender-tracing" +version = "0.31.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef6a1ac5ca3accf562b8c306fa8483c85f4390f768185ab775f242f7fe8fdcc2" +dependencies = [ + "opentelemetry", + "tracing", + "tracing-core", + "tracing-subscriber", +] + [[package]] name = "opentelemetry-http" version = "0.31.0" @@ -6828,6 +6840,7 @@ dependencies = [ "once_cell", "openssl-probe 0.1.6", "opentelemetry", + "opentelemetry-appender-tracing", "opentelemetry-otlp", "opentelemetry_sdk", "predicates", diff --git a/quickwit/Cargo.toml b/quickwit/Cargo.toml index eefbac9159f..5ff9b9df760 100644 --- a/quickwit/Cargo.toml +++ b/quickwit/Cargo.toml @@ -170,6 +170,7 @@ oneshot = "0.1" openssl = { version = "0.10", default-features = false } openssl-probe = "0.1" opentelemetry = "0.31" +opentelemetry-appender-tracing = "0.31" opentelemetry_sdk = { version = "0.31", features = ["rt-tokio"] } opentelemetry-otlp = { version = "0.31", features = ["grpc-tonic"] } ouroboros = "0.18" diff --git a/quickwit/quickwit-cli/Cargo.toml b/quickwit/quickwit-cli/Cargo.toml index c595cb7e90a..2b61a5d2e6b 100644 --- a/quickwit/quickwit-cli/Cargo.toml +++ b/quickwit/quickwit-cli/Cargo.toml @@ -37,6 +37,7 @@ numfmt = { workspace = true } once_cell = { workspace = true } openssl-probe = { workspace = true, optional = true } opentelemetry = { workspace = true } +opentelemetry-appender-tracing = { workspace = true } opentelemetry_sdk = { workspace = true } opentelemetry-otlp = { workspace = true } reqwest = { workspace = true } diff --git a/quickwit/quickwit-cli/src/logger.rs b/quickwit/quickwit-cli/src/logger.rs index 6388d4ed78d..e1e60a14f93 100644 --- a/quickwit/quickwit-cli/src/logger.rs +++ b/quickwit/quickwit-cli/src/logger.rs @@ -18,6 +18,8 @@ use std::{env, fmt}; use anyhow::Context; use opentelemetry::trace::TracerProvider; use opentelemetry::{KeyValue, global}; +use opentelemetry_appender_tracing::layer::OpenTelemetryTracingBridge; +use opentelemetry_sdk::logs::SdkLoggerProvider; use opentelemetry_sdk::propagation::TraceContextPropagator; use opentelemetry_sdk::trace::{BatchConfigBuilder, SdkTracerProvider}; use opentelemetry_sdk::{Resource, trace}; @@ -56,7 +58,10 @@ pub fn setup_logging_and_tracing( level: Level, ansi_colors: bool, build_info: &BuildInfo, -) -> anyhow::Result<(EnvFilterReloadFn, Option)> { +) -> anyhow::Result<( + EnvFilterReloadFn, + Option<(SdkTracerProvider, SdkLoggerProvider)>, +)> { #[cfg(feature = "tokio-console")] { if get_bool_from_env(QW_ENABLE_TOKIO_CONSOLE_ENV_KEY, false) { @@ -93,11 +98,11 @@ pub fn setup_logging_and_tracing( // Note on disabling ANSI characters: setting the ansi boolean on event format is insufficient. // It is thus set on layers, see https://github.com/tokio-rs/tracing/issues/1817 let provider_opt = if get_bool_from_env(QW_ENABLE_OPENTELEMETRY_OTLP_EXPORTER_ENV_KEY, false) { - let otlp_exporter = opentelemetry_otlp::SpanExporter::builder() + let span_exporter = opentelemetry_otlp::SpanExporter::builder() .with_tonic() .build() .context("failed to initialize OpenTelemetry OTLP exporter")?; - let batch_processor = trace::BatchSpanProcessor::builder(otlp_exporter) + let span_processor = trace::BatchSpanProcessor::builder(span_exporter) .with_batch_config( BatchConfigBuilder::default() // Quickwit can generate a lot of spans, especially in debug mode, and the @@ -112,17 +117,33 @@ pub fn setup_logging_and_tracing( .with_attribute(KeyValue::new("service.version", build_info.version.clone())) .build(); - let provider = opentelemetry_sdk::trace::SdkTracerProvider::builder() - .with_span_processor(batch_processor) + let logs_exporter = opentelemetry_otlp::LogExporter::builder() + .with_tonic() + .build() + .context("failed to initialize OpenTelemetry OTLP logs")?; + + let logger_provider = SdkLoggerProvider::builder() + .with_resource(resource.clone()) + .with_batch_exporter(logs_exporter) + .build(); + + let tracing_provider = opentelemetry_sdk::trace::SdkTracerProvider::builder() + .with_span_processor(span_processor) .with_resource(resource) .build(); - let tracer = provider.tracer("quickwit"); + + let tracer = tracing_provider.tracer("quickwit"); let telemetry_layer = tracing_opentelemetry::layer().with_tracer(tracer); + + // Bridge between tracing logs and otel tracing events + let logs_otel_layer = OpenTelemetryTracingBridge::new(&logger_provider); + registry .with(telemetry_layer) + .with(logs_otel_layer) .try_init() .context("failed to register tracing subscriber")?; - Some(provider) + Some((tracing_provider, logger_provider)) } else { registry .try_init() diff --git a/quickwit/quickwit-cli/src/main.rs b/quickwit/quickwit-cli/src/main.rs index 518cf6518e4..95456007df3 100644 --- a/quickwit/quickwit-cli/src/main.rs +++ b/quickwit/quickwit-cli/src/main.rs @@ -113,8 +113,11 @@ async fn main_impl() -> anyhow::Result<()> { 0 }; - if let Some(provider) = tracer_provider_opt { - provider + if let Some((trace_provider, logs_provider)) = tracer_provider_opt { + trace_provider + .shutdown() + .context("failed to shutdown OpenTelemetry tracer provider")?; + logs_provider .shutdown() .context("failed to shutdown OpenTelemetry tracer provider")?; }