feat: integrate Langfuse LLM observability via OTEL

.env.example

@@ -2707,22 +2707,121 @@ PLUGINS_CLI_MARKUP_MODE=rich
 # Copy resource attributes to span attributes (for Arize compatibility)
 # Some observability backends like Arize require certain attributes as span attributes
 # rather than resource attributes. Enable this to copy arize.project.name and model_id.
-# Direct env read (mcpgateway/observability.py)
+# Read via mcpgateway/config.py
 # OTEL_COPY_RESOURCE_ATTRS_TO_SPANS=false
 
 # Deployment environment label for observability resource attributes
-# Direct env read (mcpgateway/observability.py)
+# Read via mcpgateway/config.py
 # DEPLOYMENT_ENV=development
 
 # Jaeger exporter auth (only used when OTEL_TRACES_EXPORTER=jaeger)
-# Direct env read (mcpgateway/observability.py)
+# Read via mcpgateway/config.py
 # OTEL_EXPORTER_JAEGER_USER=
 # OTEL_EXPORTER_JAEGER_PASSWORD=
 
 # Test mode for observability (disables tracing when set to 1)
 # Direct env read (mcpgateway/observability.py)
 # MCP_TESTING=0
 
+# =============================================================================
+# Langfuse LLM Observability Integration
+# =============================================================================
+# Langfuse provides trace visualization, prompt management, evaluations,
+# cost tracking, and LLM analytics. Integrates via OTLP/HTTP.
+#
+# Quick start:  make langfuse-up
+# Access:       http://localhost:3100
+# Combined:     make langfuse-monitoring-up (Langfuse + Grafana/Tempo; gateway traces still go to Langfuse by default)
+#
+# Usage: docker compose -f docker-compose.yml -f docker-compose.with-langfuse.yml up -d
+
+# Langfuse OTLP endpoint override for the gateway.
+# Defaults to the local compose service when unset in the overlay.
+# LANGFUSE_OTEL_ENDPOINT=http://localhost:3100/api/public/otel/v1/traces
+
+# Langfuse API keys used by ContextForge to connect to Langfuse via OTLP.
+# For the local self-hosted compose overlay, unset values fall back to the
+# compose-local dev defaults `pk-lf-contextforge` / `sk-lf-contextforge`.
+# Set these when you want a different local project or when connecting to an
+# external Langfuse instance.
+# LANGFUSE_PUBLIC_KEY=pk-lf-<optional-override>
+# LANGFUSE_SECRET_KEY=sk-lf-<optional-override>
+
+# Optional OTEL auth override: base64("publicKey:secretKey")
+# When LANGFUSE_PUBLIC_KEY and LANGFUSE_SECRET_KEY are set, the gateway can
+# derive the Authorization header automatically and this override is not needed.
+# LANGFUSE_OTEL_AUTH=<base64-of-publicKey:secretKey>
+
+# Langfuse UI host port (default: 3100 to avoid Grafana conflict on 3000)
+# LANGFUSE_PORT=3100
+
+# Langfuse worker metrics port (localhost only)
+# LANGFUSE_WORKER_PORT=3130
+
+# Langfuse UI URL (used for NEXTAUTH_URL and CORS)
+# LANGFUSE_URL=http://localhost:3100
+
+# Auto-provisioned admin user override for the local self-hosted Langfuse overlay
+# LANGFUSE_INIT_USER_EMAIL=admin@example.com
+# LANGFUSE_INIT_USER_PASSWORD=<optional-override>
+
+# Auto-provisioned organization and project
+# LANGFUSE_INIT_ORG_ID=contextforge
+# LANGFUSE_INIT_ORG_NAME=ContextForge
+# LANGFUSE_INIT_PROJECT_ID=contextforge-gateway
+# LANGFUSE_INIT_PROJECT_NAME=ContextForge Gateway
+
+# Optional overrides for the local self-hosted Langfuse overlay only.
+# ContextForge does not read these. They are used only by docker-compose.with-langfuse.yml.
+# If unset, the overlay uses local-only defaults defined in that compose file.
+# LANGFUSE_POSTGRES_PASSWORD=<optional-override>
+# LANGFUSE_CLICKHOUSE_USER=clickhouse
+# LANGFUSE_CLICKHOUSE_PASSWORD=<optional-override>
+# LANGFUSE_MINIO_USER=minio
+# LANGFUSE_MINIO_PASSWORD=<optional-override>
+# LANGFUSE_REDIS_AUTH=<optional-override>
+# LANGFUSE_NEXTAUTH_SECRET=<optional-override>
+# LANGFUSE_SALT=<optional-override>
+# LANGFUSE_ENCRYPTION_KEY=<optional-override>
+
+# Langfuse optional features
+# LANGFUSE_TELEMETRY_ENABLED=true
+# LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES=false
+
+# Monitoring stack host port overrides for `make monitoring-up` and
+# `make langfuse-monitoring-up`. These are compose-only and are not read by
+# mcpgateway/config.py.
+# NGINX_PORT=8080
+# GRAFANA_PORT=3000
+# LOKI_PORT=3101
+# PROMETHEUS_PORT=9090
+# TEMPO_PORT=3200
+# TEMPO_OTLP_GRPC_PORT=4317
+# TEMPO_OTLP_HTTP_PORT=4318
+# TEMPO_IMAGE_TAG=2.10.0
+# PGADMIN_PORT=5050
+# REDIS_COMMANDER_PORT=8081
+# POSTGRES_EXPORTER_PORT=9187
+# REDIS_EXPORTER_PORT=9121
+# PGBOUNCER_EXPORTER_PORT=9127
+# NGINX_EXPORTER_PORT=9113
+# CADVISOR_PORT=8085
+
+# OTEL trace controls for Langfuse and other OTLP backends
+# Langfuse-specific attributes auto-enable when LANGFUSE_OTEL_ENDPOINT points to
+# Langfuse. Set these explicitly only when you want to override that behavior.
+# OTEL_EMIT_LANGFUSE_ATTRIBUTES=
+# OTEL_CAPTURE_IDENTITY_ATTRIBUTES=
+#
+# Payload capture is allowlist-based. By default the gateway does not capture
+# observation input or output payloads unless the relevant span names are listed.
+# The local `docker-compose.with-langfuse.yml` overlay sets a dev-friendly input
+# allowlist for `tool.invoke,prompt.render,llm.proxy,a2a.invoke`.
+# OTEL_REDACT_FIELDS=password,secret,token,api_key,authorization,credential,auth_value,access_token,refresh_token,auth_token,client_secret,cookie,set-cookie,private_key
+# OTEL_MAX_TRACE_PAYLOAD_SIZE=32768
+# OTEL_CAPTURE_INPUT_SPANS=tool.invoke,prompt.render
+# OTEL_CAPTURE_OUTPUT_SPANS=
+
 # --- Auxiliary tools and CLIs (non-gateway runtime) --------------------------
 
 # These are used by helper tools, CLIs, and SDK wrappers (not the main gateway server).

.gitignore

@@ -77,6 +77,8 @@ pip-delete-this-directory.txt
 # ========================================
 # Testing & Coverage
 # ========================================
+output
+output/
 htmlcov
 htmlcov/
 .tox/