diff --git a/swap-orchestrator/README.md b/swap-orchestrator/README.md index af0c3c071..fca7cf2ed 100644 --- a/swap-orchestrator/README.md +++ b/swap-orchestrator/README.md @@ -40,7 +40,7 @@ If you're not compiling the `orchestrator` from source you can grab the latest [ Run the command below to start the wizard. It’ll guide you through a bunch of questions to generate the `config.toml` file and the `docker-compose.yml` file based on your needs. You can always modify the `config.toml` later on to modify specific things about your `asb` like the minimum swap amount or the configured markup. -To also ship `asb` tracing logs to a Loki endpoint, set `PROMTAIL_LOKI_PUSH_URL`, `PROMTAIL_LOKI_PUSH_TOKEN`, and `PROMTAIL_INSTANCE` before running the orchestrator — this adds a `promtail` service to the generated `docker-compose.yml`. +To also ship the `asb` tracing logs and the `bitcoind`/`monerod`/`electrs` container logs to a Loki endpoint, set `PROMTAIL_LOKI_PUSH_URL`, `PROMTAIL_LOKI_PUSH_TOKEN`, and `PROMTAIL_INSTANCE` before running the orchestrator — this adds `promtail` and `docker-socket-proxy` services to the generated `docker-compose.yml`. All streams are labelled with `host=`; the daemon logs additionally carry `job=node` and `container=`. ```bash ./orchestrator diff --git a/swap-orchestrator/src/compose.rs b/swap-orchestrator/src/compose.rs index cddf21da9..1d39d6319 100644 --- a/swap-orchestrator/src/compose.rs +++ b/swap-orchestrator/src/compose.rs @@ -45,11 +45,12 @@ pub struct CloudflaredConfig { /// Promtail log-shipper configuration. /// -/// When set, the orchestrator adds a `promtail` service to the compose file -/// and writes a `promtail.yml` next to `docker-compose.yml`. The shipper -/// tails the JSON tracing logs from the `asb-data` volume (mounted -/// read-only) and pushes them to a Loki endpoint over HTTPS with a bearer -/// token. +/// When set, the orchestrator adds `promtail` and `docker-socket-proxy` +/// services to the compose file and writes a `promtail.yml` next to +/// `docker-compose.yml`. The shipper tails the JSON tracing logs from the +/// `asb-data` volume (mounted read-only) and the stdout of the +/// `bitcoind`/`monerod`/`electrs` containers (read via the socket proxy), +/// then pushes everything to a Loki endpoint over HTTPS with a bearer token. #[derive(Clone)] pub struct PromtailConfig { /// Loki push endpoint, e.g. @@ -59,7 +60,8 @@ pub struct PromtailConfig { /// `promtail.yml` only — never written to `docker-compose.yml`. pub loki_push_token: String, /// Short identifier for this host (e.g. `asb-de-1`). Exported as the - /// `host` Loki label so operators can filter by deployment in Grafana. + /// `host` Loki label on both the asb and node log streams so operators + /// can filter a whole deployment in Grafana. pub instance: String, } @@ -84,6 +86,7 @@ pub struct OrchestratorImages { pub rendezvous_node: T, pub cloudflared: T, pub promtail: T, + pub docker_socket_proxy: T, } pub struct OrchestratorPorts { @@ -321,20 +324,40 @@ fn build(input: OrchestratorInput) -> String { // The promtail config file lives next to docker-compose.yml on the // host. It is generated by the orchestrator at the same time as the // compose file, with the URL/token/instance values baked in. + // + // docker-socket-proxy is the only container that mounts the docker + // socket. It exposes only the read-only container + network APIs + // (CONTAINERS=1, NETWORKS=1; POST stays disabled). promtail's docker + // service discovery needs /networks to compute the network labels in + // addition to listing containers, so both are required - but it still + // never holds write/root-equivalent access to the host. let promtail_segment = format!( "\ + docker-socket-proxy: + container_name: docker-socket-proxy + {image_docker_socket_proxy} + restart: unless-stopped + environment: + - CONTAINERS=1 + - NETWORKS=1 + volumes: + - '/var/run/docker.sock:/var/run/docker.sock:ro' + expose: + - 2375 promtail: container_name: promtail {image_promtail} restart: unless-stopped depends_on: - asb + - docker-socket-proxy volumes: - '{promtail_config_file}:/etc/promtail/promtail.yml:ro' - 'asb-data:/asb-data:ro' - 'promtail-positions:/var/lib/promtail' command: [\"-config.file=/etc/promtail/promtail.yml\"]\ ", + image_docker_socket_proxy = input.images.docker_socket_proxy.to_image_attribute(), image_promtail = input.images.promtail.to_image_attribute(), promtail_config_file = PROMTAIL_CONFIG_FILE, ); @@ -516,11 +539,19 @@ volumes: /// /// Values from [`PromtailConfig`] are baked directly into the file — the /// container does not need env-var expansion at runtime, and the bearer -/// token never appears in `docker-compose.yml`. The scrape job tails every -/// `*.log` file under `/asb-data/logs/` (where `asb` writes -/// `tracing.*`, `tracing-libp2p.*`, `tracing-monero-wallet.*`, -/// `tracing-tor.*`, etc.) and labels each stream with the component -/// extracted from the file name. +/// token never appears in `docker-compose.yml`. +/// +/// Two scrape jobs are emitted, both labelled with the same `host` so a +/// deployment can be selected as a whole: +/// - `asb-tracing` tails every `*.log` file under `/asb-data/logs/` (where +/// `asb` writes `tracing.*`, `tracing-libp2p.*`, `tracing-monero-wallet.*`, +/// `tracing-tor.*`, etc.) and labels each stream with the component +/// extracted from the file name. +/// - `node` discovers the `bitcoind`/`monerod`/`electrs` containers through +/// the docker-socket-proxy and tails their stdout, labelling each stream +/// with `job: node` and the `container` name. These daemons log plain text +/// (electrs has no log file at all), so they are shipped as raw lines +/// rather than parsed as JSON. pub fn build_promtail_yml(cfg: &PromtailConfig) -> String { // The single quote in YAML single-quoted strings is escaped by doubling // it. We single-quote every interpolated value so URLs containing @@ -573,6 +604,22 @@ scrape_configs: action_on_failure: skip - labels: level: + - job_name: node + docker_sd_configs: + - host: tcp://docker-socket-proxy:2375 + refresh_interval: 5s + relabel_configs: + - source_labels: [__meta_docker_container_name] + regex: '/?(bitcoind|monerod|electrs)' + action: keep + - source_labels: [__meta_docker_container_name] + regex: '/?(.*)' + target_label: container + replacement: '$1' + - target_label: job + replacement: node + - target_label: host + replacement: {instance} ", url = yaml_single_quote(&cfg.loki_push_url), token = yaml_single_quote(&cfg.loki_push_token), diff --git a/swap-orchestrator/src/images.rs b/swap-orchestrator/src/images.rs index 9c70dc006..5e70987d1 100644 --- a/swap-orchestrator/src/images.rs +++ b/swap-orchestrator/src/images.rs @@ -38,6 +38,9 @@ pub static CLOUDFLARED_IMAGE: &str = "cloudflare/cloudflared@sha256:6b599ca3e974 /// promtail 3.4.1 (https://hub.docker.com/r/grafana/promtail) pub static PROMTAIL_IMAGE: &str = "grafana/promtail@sha256:8b2aa61745bc4a9343cc47bd391fb935a80e7da0793c7566d5985c75858ba3f8"; +/// docker-socket-proxy 0.3.0 (https://hub.docker.com/r/tecnativa/docker-socket-proxy) +pub static DOCKER_SOCKET_PROXY_IMAGE: &str = "tecnativa/docker-socket-proxy@sha256:9e4b9e7517a6b660f2cc903a19b257b1852d5b3344794e3ea334ff00ae677ac2"; + /// These are built from source pub static ASB_IMAGE_FROM_SOURCE: DockerBuildInput = DockerBuildInput { // The context is the root of the Cargo workspace diff --git a/swap-orchestrator/src/main.rs b/swap-orchestrator/src/main.rs index 7dac4513c..5c3197070 100644 --- a/swap-orchestrator/src/main.rs +++ b/swap-orchestrator/src/main.rs @@ -193,6 +193,9 @@ fn main() { ), cloudflared: OrchestratorImage::Registry(images::CLOUDFLARED_IMAGE.to_string()), promtail: OrchestratorImage::Registry(images::PROMTAIL_IMAGE.to_string()), + docker_socket_proxy: OrchestratorImage::Registry( + images::DOCKER_SOCKET_PROXY_IMAGE.to_string(), + ), }, directories: OrchestratorDirectories { asb_data_dir: PathBuf::from(ASB_DATA_DIR), @@ -482,10 +485,15 @@ fn print_promtail_instructions(promtail: &PromtailConfig) { println!(" - Instance label (host): {}", promtail.instance); println!(" - Loki push URL: {}", promtail.loki_push_url); println!(" - Config written to: {}", PROMTAIL_CONFIG_FILE); + println!(" - Ships: asb tracing logs + bitcoind/monerod/electrs container logs"); println!(" - Verify after `docker compose up -d`:"); println!(" docker compose logs --tail 50 promtail"); println!( - " - Grafana query to select this host: {{host=\"{}\"}}", + " - Grafana query (whole host): {{host=\"{}\"}}", + promtail.instance + ); + println!( + " - Grafana query (node daemons): {{host=\"{}\", job=\"node\"}}", promtail.instance ); } diff --git a/swap-orchestrator/tests/spec.rs b/swap-orchestrator/tests/spec.rs index 3c0b90097..3a468ea65 100644 --- a/swap-orchestrator/tests/spec.rs +++ b/swap-orchestrator/tests/spec.rs @@ -44,6 +44,9 @@ fn make_input( ), cloudflared: OrchestratorImage::Registry(images::CLOUDFLARED_IMAGE.to_string()), promtail: OrchestratorImage::Registry(images::PROMTAIL_IMAGE.to_string()), + docker_socket_proxy: OrchestratorImage::Registry( + images::DOCKER_SOCKET_PROXY_IMAGE.to_string(), + ), }, directories: OrchestratorDirectories { asb_data_dir: std::path::PathBuf::from(swap_orchestrator::compose::ASB_DATA_DIR), @@ -80,8 +83,12 @@ fn test_orchestrator_spec_generation() { let _ = make_input(true, None, None).to_spec(); let _ = make_input(false, Some(sample_cloudflared_config()), None).to_spec(); let _ = make_input(true, Some(sample_cloudflared_config()), None).to_spec(); - let _ = make_input(false, None, Some(sample_promtail_config())).to_spec(); + let compose = make_input(false, None, Some(sample_promtail_config())).to_spec(); let _ = make_input(true, None, Some(sample_promtail_config())).to_spec(); + + // promtail's docker SD needs the networks API, not just containers, or + // discovery 403s on /networks and no node logs ship. + assert!(compose.contains("NETWORKS=1")); let _ = make_input( true, Some(sample_cloudflared_config()), @@ -123,3 +130,39 @@ fn test_promtail_yml_escapes_single_quotes() { let parsed: serde_yaml::Value = serde_yaml::from_str(&yml).expect("must be valid YAML"); assert_eq!(parsed["clients"][0]["bearer_token"].as_str(), Some("abc'def")); } + +#[test] +fn test_promtail_yml_ships_node_container_logs() { + let yml = build_promtail_yml(&sample_promtail_config()); + let parsed: serde_yaml::Value = + serde_yaml::from_str(&yml).expect("promtail.yml must be valid YAML"); + + let node_job = parsed["scrape_configs"] + .as_sequence() + .expect("scrape_configs must be a list") + .iter() + .find(|job| job["job_name"].as_str() == Some("node")) + .expect("a `node` scrape job must be present"); + + // The node logs are read through the docker-socket-proxy, not a file path. + assert_eq!( + node_job["docker_sd_configs"][0]["host"].as_str(), + Some("tcp://docker-socket-proxy:2375") + ); + + // Only the three daemon containers are shipped. + let keep = node_job["relabel_configs"][0]["regex"] + .as_str() + .expect("keep regex must be present"); + assert!(keep.contains("bitcoind") && keep.contains("monerod") && keep.contains("electrs")); + + // Node logs carry the same `host` label as the asb logs so a whole + // deployment selects with one query. + let host_relabel = node_job["relabel_configs"] + .as_sequence() + .expect("relabel_configs must be a list") + .iter() + .find(|rc| rc["target_label"].as_str() == Some("host")) + .expect("a host relabel must be present"); + assert_eq!(host_relabel["replacement"].as_str(), Some("asb-test-1")); +}