feat(orchestrator): ship bitcoind/monerod/electrs logs via promtail

swap-orchestrator/README.md

@@ -40,7 +40,7 @@ If you're not compiling the `orchestrator` from source you can grab the latest [
 
 Run the command below to start the wizard. It’ll guide you through a bunch of questions to generate the `config.toml` file and the `docker-compose.yml` file based on your needs. You can always modify the `config.toml` later on to modify specific things about your `asb` like the minimum swap amount or the configured markup.
 
-To also ship `asb` tracing logs to a Loki endpoint, set `PROMTAIL_LOKI_PUSH_URL`, `PROMTAIL_LOKI_PUSH_TOKEN`, and `PROMTAIL_INSTANCE` before running the orchestrator — this adds a `promtail` service to the generated `docker-compose.yml`.
+To also ship the `asb` tracing logs and the `bitcoind`/`monerod`/`electrs` container logs to a Loki endpoint, set `PROMTAIL_LOKI_PUSH_URL`, `PROMTAIL_LOKI_PUSH_TOKEN`, and `PROMTAIL_INSTANCE` before running the orchestrator — this adds `promtail` and `docker-socket-proxy` services to the generated `docker-compose.yml`. All streams are labelled with `host=<instance>`; the daemon logs additionally carry `job=node` and `container=<name>`.
 
 ```bash
 ./orchestrator

swap-orchestrator/src/compose.rs

@@ -45,11 +45,12 @@ pub struct CloudflaredConfig {
 
 /// Promtail log-shipper configuration.
 ///
-/// When set, the orchestrator adds a `promtail` service to the compose file
-/// and writes a `promtail.yml` next to `docker-compose.yml`. The shipper
-/// tails the JSON tracing logs from the `asb-data` volume (mounted
-/// read-only) and pushes them to a Loki endpoint over HTTPS with a bearer
-/// token.
+/// When set, the orchestrator adds `promtail` and `docker-socket-proxy`
+/// services to the compose file and writes a `promtail.yml` next to
+/// `docker-compose.yml`. The shipper tails the JSON tracing logs from the
+/// `asb-data` volume (mounted read-only) and the stdout of the
+/// `bitcoind`/`monerod`/`electrs` containers (read via the socket proxy),
+/// then pushes everything to a Loki endpoint over HTTPS with a bearer token.
 #[derive(Clone)]
 pub struct PromtailConfig {
     /// Loki push endpoint, e.g.
@@ -59,7 +60,8 @@ pub struct PromtailConfig {
     /// `promtail.yml` only — never written to `docker-compose.yml`.
     pub loki_push_token: String,
     /// Short identifier for this host (e.g. `asb-de-1`). Exported as the
-    /// `host` Loki label so operators can filter by deployment in Grafana.
+    /// `host` Loki label on both the asb and node log streams so operators
+    /// can filter a whole deployment in Grafana.
     pub instance: String,
 }
 
@@ -84,6 +86,7 @@ pub struct OrchestratorImages<T: IntoImageAttribute> {
     pub rendezvous_node: T,
     pub cloudflared: T,
     pub promtail: T,
+    pub docker_socket_proxy: T,
 }
 
 pub struct OrchestratorPorts {
@@ -321,20 +324,40 @@ fn build(input: OrchestratorInput) -> String {
         // The promtail config file lives next to docker-compose.yml on the
         // host. It is generated by the orchestrator at the same time as the
         // compose file, with the URL/token/instance values baked in.
+        //
+        // docker-socket-proxy is the only container that mounts the docker
+        // socket. It exposes only the read-only container + network APIs
+        // (CONTAINERS=1, NETWORKS=1; POST stays disabled). promtail's docker
+        // service discovery needs /networks to compute the network labels in
+        // addition to listing containers, so both are required - but it still
+        // never holds write/root-equivalent access to the host.
         let promtail_segment = format!(
             "\
+  docker-socket-proxy:
+    container_name: docker-socket-proxy
+    {image_docker_socket_proxy}
+    restart: unless-stopped
+    environment:
+      - CONTAINERS=1
+      - NETWORKS=1
+    volumes:
+      - '/var/run/docker.sock:/var/run/docker.sock:ro'
+    expose:
+      - 2375
   promtail:
     container_name: promtail
     {image_promtail}
     restart: unless-stopped
     depends_on:
       - asb
+      - docker-socket-proxy
     volumes:
       - '{promtail_config_file}:/etc/promtail/promtail.yml:ro'
       - 'asb-data:/asb-data:ro'
       - 'promtail-positions:/var/lib/promtail'
     command: [\"-config.file=/etc/promtail/promtail.yml\"]\
 ",
+            image_docker_socket_proxy = input.images.docker_socket_proxy.to_image_attribute(),
             image_promtail = input.images.promtail.to_image_attribute(),
             promtail_config_file = PROMTAIL_CONFIG_FILE,
         );
@@ -516,11 +539,19 @@ volumes:
 ///
 /// Values from [`PromtailConfig`] are baked directly into the file — the
 /// container does not need env-var expansion at runtime, and the bearer
-/// token never appears in `docker-compose.yml`. The scrape job tails every
-/// `*.log` file under `/asb-data/logs/` (where `asb` writes
-/// `tracing.*`, `tracing-libp2p.*`, `tracing-monero-wallet.*`,
-/// `tracing-tor.*`, etc.) and labels each stream with the component
-/// extracted from the file name.
+/// token never appears in `docker-compose.yml`.
+///
+/// Two scrape jobs are emitted, both labelled with the same `host` so a
+/// deployment can be selected as a whole:
+/// - `asb-tracing` tails every `*.log` file under `/asb-data/logs/` (where
+///   `asb` writes `tracing.*`, `tracing-libp2p.*`, `tracing-monero-wallet.*`,
+///   `tracing-tor.*`, etc.) and labels each stream with the component
+///   extracted from the file name.
+/// - `node` discovers the `bitcoind`/`monerod`/`electrs` containers through
+///   the docker-socket-proxy and tails their stdout, labelling each stream
+///   with `job: node` and the `container` name. These daemons log plain text
+///   (electrs has no log file at all), so they are shipped as raw lines
+///   rather than parsed as JSON.
 pub fn build_promtail_yml(cfg: &PromtailConfig) -> String {
     // The single quote in YAML single-quoted strings is escaped by doubling
     // it. We single-quote every interpolated value so URLs containing
@@ -573,6 +604,22 @@ scrape_configs:
           action_on_failure: skip
       - labels:
           level:
+  - job_name: node
+    docker_sd_configs:
+      - host: tcp://docker-socket-proxy:2375
+        refresh_interval: 5s
+    relabel_configs:
+      - source_labels: [__meta_docker_container_name]
+        regex: '/?(bitcoind|monerod|electrs)'
+        action: keep
+      - source_labels: [__meta_docker_container_name]
+        regex: '/?(.*)'
+        target_label: container
+        replacement: '$1'
+      - target_label: job
+        replacement: node
+      - target_label: host
+        replacement: {instance}
 ",
         url = yaml_single_quote(&cfg.loki_push_url),
         token = yaml_single_quote(&cfg.loki_push_token),

swap-orchestrator/src/images.rs

@@ -38,6 +38,9 @@ pub static CLOUDFLARED_IMAGE: &str = "cloudflare/cloudflared@sha256:6b599ca3e974
 /// promtail 3.4.1 (https://hub.docker.com/r/grafana/promtail)
 pub static PROMTAIL_IMAGE: &str = "grafana/promtail@sha256:8b2aa61745bc4a9343cc47bd391fb935a80e7da0793c7566d5985c75858ba3f8";
 
+/// docker-socket-proxy 0.3.0 (https://hub.docker.com/r/tecnativa/docker-socket-proxy)
+pub static DOCKER_SOCKET_PROXY_IMAGE: &str = "tecnativa/docker-socket-proxy@sha256:9e4b9e7517a6b660f2cc903a19b257b1852d5b3344794e3ea334ff00ae677ac2";
+
 /// These are built from source
 pub static ASB_IMAGE_FROM_SOURCE: DockerBuildInput = DockerBuildInput {
     // The context is the root of the Cargo workspace

swap-orchestrator/src/main.rs

@@ -193,6 +193,9 @@ fn main() {
             ),
             cloudflared: OrchestratorImage::Registry(images::CLOUDFLARED_IMAGE.to_string()),
             promtail: OrchestratorImage::Registry(images::PROMTAIL_IMAGE.to_string()),
+            docker_socket_proxy: OrchestratorImage::Registry(
+                images::DOCKER_SOCKET_PROXY_IMAGE.to_string(),
+            ),
         },
         directories: OrchestratorDirectories {
             asb_data_dir: PathBuf::from(ASB_DATA_DIR),
@@ -482,10 +485,15 @@ fn print_promtail_instructions(promtail: &PromtailConfig) {
     println!("  - Instance label (host): {}", promtail.instance);
     println!("  - Loki push URL:         {}", promtail.loki_push_url);
     println!("  - Config written to:     {}", PROMTAIL_CONFIG_FILE);
+    println!("  - Ships: asb tracing logs + bitcoind/monerod/electrs container logs");
     println!("  - Verify after `docker compose up -d`:");
     println!("      docker compose logs --tail 50 promtail");
     println!(
-        "  - Grafana query to select this host: {{host=\"{}\"}}",
+        "  - Grafana query (whole host):  {{host=\"{}\"}}",
+        promtail.instance
+    );
+    println!(
+        "  - Grafana query (node daemons): {{host=\"{}\", job=\"node\"}}",
         promtail.instance
     );
 }

swap-orchestrator/tests/spec.rs

@@ -44,6 +44,9 @@ fn make_input(
             ),
             cloudflared: OrchestratorImage::Registry(images::CLOUDFLARED_IMAGE.to_string()),
             promtail: OrchestratorImage::Registry(images::PROMTAIL_IMAGE.to_string()),
+            docker_socket_proxy: OrchestratorImage::Registry(
+                images::DOCKER_SOCKET_PROXY_IMAGE.to_string(),
+            ),
         },
         directories: OrchestratorDirectories {
             asb_data_dir: std::path::PathBuf::from(swap_orchestrator::compose::ASB_DATA_DIR),
@@ -80,8 +83,12 @@ fn test_orchestrator_spec_generation() {
     let _ = make_input(true, None, None).to_spec();
     let _ = make_input(false, Some(sample_cloudflared_config()), None).to_spec();
     let _ = make_input(true, Some(sample_cloudflared_config()), None).to_spec();
-    let _ = make_input(false, None, Some(sample_promtail_config())).to_spec();
+    let compose = make_input(false, None, Some(sample_promtail_config())).to_spec();
     let _ = make_input(true, None, Some(sample_promtail_config())).to_spec();
+
+    // promtail's docker SD needs the networks API, not just containers, or
+    // discovery 403s on /networks and no node logs ship.
+    assert!(compose.contains("NETWORKS=1"));
     let _ = make_input(
         true,
         Some(sample_cloudflared_config()),
@@ -123,3 +130,39 @@ fn test_promtail_yml_escapes_single_quotes() {
     let parsed: serde_yaml::Value = serde_yaml::from_str(&yml).expect("must be valid YAML");
     assert_eq!(parsed["clients"][0]["bearer_token"].as_str(), Some("abc'def"));
 }
+
+#[test]
+fn test_promtail_yml_ships_node_container_logs() {
+    let yml = build_promtail_yml(&sample_promtail_config());
+    let parsed: serde_yaml::Value =
+        serde_yaml::from_str(&yml).expect("promtail.yml must be valid YAML");
+
+    let node_job = parsed["scrape_configs"]
+        .as_sequence()
+        .expect("scrape_configs must be a list")
+        .iter()
+        .find(|job| job["job_name"].as_str() == Some("node"))
+        .expect("a `node` scrape job must be present");
+
+    // The node logs are read through the docker-socket-proxy, not a file path.
+    assert_eq!(
+        node_job["docker_sd_configs"][0]["host"].as_str(),
+        Some("tcp://docker-socket-proxy:2375")
+    );
+
+    // Only the three daemon containers are shipped.
+    let keep = node_job["relabel_configs"][0]["regex"]
+        .as_str()
+        .expect("keep regex must be present");
+    assert!(keep.contains("bitcoind") && keep.contains("monerod") && keep.contains("electrs"));
+
+    // Node logs carry the same `host` label as the asb logs so a whole
+    // deployment selects with one query.
+    let host_relabel = node_job["relabel_configs"]
+        .as_sequence()
+        .expect("relabel_configs must be a list")
+        .iter()
+        .find(|rc| rc["target_label"].as_str() == Some("host"))
+        .expect("a host relabel must be present");
+    assert_eq!(host_relabel["replacement"].as_str(), Some("asb-test-1"));
+}