feat(obs): export internal logs with otlp exporter

quickwit/Cargo.toml

@@ -170,6 +170,7 @@ oneshot = "0.1"
 openssl = { version = "0.10", default-features = false }
 openssl-probe = "0.1"
 opentelemetry = "0.31"
+opentelemetry-appender-tracing = "0.31"
 opentelemetry_sdk = { version = "0.31", features = ["rt-tokio"] }
 opentelemetry-otlp = { version = "0.31", features = ["grpc-tonic"] }
 ouroboros = "0.18"

quickwit/quickwit-cli/Cargo.toml

@@ -37,6 +37,7 @@ numfmt = { workspace = true }
 once_cell = { workspace = true }
 openssl-probe = { workspace = true, optional = true }
 opentelemetry = { workspace = true }
+opentelemetry-appender-tracing = { workspace = true }
 opentelemetry_sdk = { workspace = true }
 opentelemetry-otlp = { workspace = true }
 reqwest = { workspace = true }

quickwit/quickwit-cli/src/logger.rs

@@ -18,6 +18,8 @@ use std::{env, fmt};
 use anyhow::Context;
 use opentelemetry::trace::TracerProvider;
 use opentelemetry::{KeyValue, global};
+use opentelemetry_appender_tracing::layer::OpenTelemetryTracingBridge;
+use opentelemetry_sdk::logs::SdkLoggerProvider;
 use opentelemetry_sdk::propagation::TraceContextPropagator;
 use opentelemetry_sdk::trace::{BatchConfigBuilder, SdkTracerProvider};
 use opentelemetry_sdk::{Resource, trace};
@@ -56,7 +58,10 @@ pub fn setup_logging_and_tracing(
     level: Level,
     ansi_colors: bool,
     build_info: &BuildInfo,
-) -> anyhow::Result<(EnvFilterReloadFn, Option<SdkTracerProvider>)> {
+) -> anyhow::Result<(
+    EnvFilterReloadFn,
+    Option<(SdkTracerProvider, SdkLoggerProvider)>,
+)> {
     #[cfg(feature = "tokio-console")]
     {
         if get_bool_from_env(QW_ENABLE_TOKIO_CONSOLE_ENV_KEY, false) {
@@ -93,11 +98,11 @@ pub fn setup_logging_and_tracing(
     // Note on disabling ANSI characters: setting the ansi boolean on event format is insufficient.
     // It is thus set on layers, see https://github.com/tokio-rs/tracing/issues/1817
     let provider_opt = if get_bool_from_env(QW_ENABLE_OPENTELEMETRY_OTLP_EXPORTER_ENV_KEY, false) {
-        let otlp_exporter = opentelemetry_otlp::SpanExporter::builder()
+        let span_exporter = opentelemetry_otlp::SpanExporter::builder()
             .with_tonic()
             .build()
             .context("failed to initialize OpenTelemetry OTLP exporter")?;
-        let batch_processor = trace::BatchSpanProcessor::builder(otlp_exporter)
+        let span_processor = trace::BatchSpanProcessor::builder(span_exporter)
             .with_batch_config(
                 BatchConfigBuilder::default()
                     // Quickwit can generate a lot of spans, especially in debug mode, and the
@@ -112,17 +117,33 @@ pub fn setup_logging_and_tracing(
             .with_attribute(KeyValue::new("service.version", build_info.version.clone()))
             .build();
 
-        let provider = opentelemetry_sdk::trace::SdkTracerProvider::builder()
-            .with_span_processor(batch_processor)
+        let logs_exporter = opentelemetry_otlp::LogExporter::builder()
+            .with_tonic()
+            .build()
+            .context("failed to initialize OpenTelemetry OTLP logs")?;
+
+        let logger_provider = SdkLoggerProvider::builder()
+            .with_resource(resource.clone())
+            .with_batch_exporter(logs_exporter)
+            .build();
+
+        let tracing_provider = opentelemetry_sdk::trace::SdkTracerProvider::builder()
+            .with_span_processor(span_processor)
             .with_resource(resource)
             .build();
-        let tracer = provider.tracer("quickwit");
+
+        let tracer = tracing_provider.tracer("quickwit");
         let telemetry_layer = tracing_opentelemetry::layer().with_tracer(tracer);
+
+        // Bridge between tracing logs and otel tracing events
+        let logs_otel_layer = OpenTelemetryTracingBridge::new(&logger_provider);
+
         registry
             .with(telemetry_layer)
+            .with(logs_otel_layer)
             .try_init()
             .context("failed to register tracing subscriber")?;
-        Some(provider)
+        Some((tracing_provider, logger_provider))
     } else {
         registry
             .try_init()

quickwit/quickwit-cli/src/main.rs

@@ -113,8 +113,11 @@ async fn main_impl() -> anyhow::Result<()> {
         0
     };
 
-    if let Some(provider) = tracer_provider_opt {
-        provider
+    if let Some((trace_provider, logs_provider)) = tracer_provider_opt {
+        trace_provider
+            .shutdown()
+            .context("failed to shutdown OpenTelemetry tracer provider")?;
+        logs_provider
             .shutdown()
             .context("failed to shutdown OpenTelemetry tracer provider")?;
     }