diff --git a/images/l2-op-rbuilder-bproxy.conf b/images/l2-op-rbuilder-bproxy.conf index d798e665..2ac41af6 100644 --- a/images/l2-op-rbuilder-bproxy.conf +++ b/images/l2-op-rbuilder-bproxy.conf @@ -12,3 +12,4 @@ Include=modules/l2/_devtools_users/mkosi.conf Include=modules/l2/_devtools_no_console/mkosi.conf Include=modules/l2/_devtools_no_root_login/mkosi.conf Include=modules/l2/op-rbuilder-bproxy/mkosi.conf +Include=modules/l2/vector/mkosi.conf diff --git a/modules/l2/vector/mkosi.build b/modules/l2/vector/mkosi.build new file mode 100755 index 00000000..0be3a97a --- /dev/null +++ b/modules/l2/vector/mkosi.build @@ -0,0 +1,57 @@ +#!/bin/bash + +set -euxo pipefail + +VECTOR_REF=2b8b875681fbf4ef8da827650f75e936f16a36a4 +VECTOR_FEATURES="unix,aws-core,sources-file,sources-journald,sources-prometheus-scrape,sinks-prometheus,sinks-clickhouse,transforms-filter,transforms-throttle,transforms-remap,transforms-metrics" + +# toolchain installed in chroot by op-rbuilder-bproxy module (runs first via Include= order). +export CARGO_HOME="/cargo" +export RUSTUP_HOME="/rustup" +export PATH="$CARGO_HOME/bin:$PATH" + +cached_binary="$BUILDDIR/vector-${VECTOR_REF}/vector" + +if [[ -f "$cached_binary" ]]; then + echo "Using cached vector binary for ref $VECTOR_REF" + install -D -m 755 "$cached_binary" "$DESTDIR/usr/bin/vector" + echo "| \`vector\` | \`${VECTOR_REF}\` (features: ${VECTOR_FEATURES}) | reused from cache | \`$(du -sh "$cached_binary" | cut -f1)\` | |" >> "$BUILDDIR/manifest.md" + exit 0 +fi + +build_dir="$BUILDROOT/build/vector" +mkdir -p "$build_dir" +curl -sSfL "https://api.github.com/repos/vectordotdev/vector/tarball/${VECTOR_REF}" | \ + tar xzf - -C "$build_dir" --strip-components=1 + +# reproducibility flags mirror scripts/build_rust_package.sh. +RUSTFLAGS=( + "-C target-cpu=generic" + "-C link-arg=-Wl,--build-id=none" + "-C symbol-mangling-version=v0" + "-L /usr/lib/x86_64-linux-gnu" + "-C link-arg=-lz" # openssl-src builds c_zlib.o into libcrypto but skips `cargo:rustc-link-lib=z`. -lz resolves inflate/deflate. +) + +ts=$( date +%s ) +mkosi-chroot bash -c " + unset DESTDIR + export RUSTFLAGS='${RUSTFLAGS[*]}' \ + CARGO_PROFILE_RELEASE_LTO='thin' \ + CARGO_PROFILE_RELEASE_CODEGEN_UNITS='1' \ + CARGO_PROFILE_RELEASE_PANIC='abort' \ + CARGO_PROFILE_RELEASE_INCREMENTAL='false' \ + CARGO_PROFILE_RELEASE_OPT_LEVEL='3' \ + CARGO_TERM_COLOR='never' + cd /build/vector + cargo fetch --locked + cargo build --release --locked --no-default-features --features '${VECTOR_FEATURES}' +" +seconds=$(( $( date +%s ) - ts )) +duration=$( printf "%dm%ds" $(( seconds / 60 )) $(( seconds % 60 )) ) + +mkdir -p "$(dirname "$cached_binary")" +install -m 755 "$build_dir/target/release/vector" "$cached_binary" +install -D -m 755 "$cached_binary" "$DESTDIR/usr/bin/vector" + +echo "| \`vector\` | \`${VECTOR_REF}\` (features: ${VECTOR_FEATURES}) | built | \`$(du -sh "$cached_binary" | cut -f1)\` | \`${duration}\` |" >> "$BUILDDIR/manifest.md" diff --git a/modules/l2/vector/mkosi.conf b/modules/l2/vector/mkosi.conf new file mode 100644 index 00000000..4b10a311 --- /dev/null +++ b/modules/l2/vector/mkosi.conf @@ -0,0 +1,9 @@ +[Build] +WithNetwork=true + +[Content] +BuildScripts=modules/l2/vector/mkosi.build +ExtraTrees=modules/l2/vector/mkosi.extra +PostInstallationScripts=modules/l2/vector/mkosi.postinst.chroot +BuildPackages=protobuf-compiler + zlib1g-dev diff --git a/modules/l2/vector/mkosi.extra/etc/default/vector b/modules/l2/vector/mkosi.extra/etc/default/vector new file mode 100644 index 00000000..80e503f1 --- /dev/null +++ b/modules/l2/vector/mkosi.extra/etc/default/vector @@ -0,0 +1,11 @@ +# Runtime environment for vector.service. Populated at deploy time. +# +# Variables consumed by /etc/vector/vector.yaml: +# L2_BUILDER_ENV +# CH_ENDPOINT +# CH_USER +# CH_PASSWORD +# +# Ships empty: vector falls back to placeholder endpoint and creds, sinks +# will not ship until real values are present. Local pipeline (journald +# read, VRL transform) still validates and runs. diff --git a/modules/l2/vector/mkosi.extra/etc/systemd/system/vector.service b/modules/l2/vector/mkosi.extra/etc/systemd/system/vector.service new file mode 100644 index 00000000..2b656048 --- /dev/null +++ b/modules/l2/vector/mkosi.extra/etc/systemd/system/vector.service @@ -0,0 +1,21 @@ +[Unit] +Description=Vector observability data pipeline +Wants=network-online.target +After=network.target network-online.target vault-agent.service + +[Service] +Type=exec +User=vector +SupplementaryGroups=systemd-journal +StateDirectory=vector +EnvironmentFile=-/etc/default/vector +Environment=VECTOR_LOG_FORMAT=json +Environment=VECTOR_CONFIG_YAML=/etc/vector/vector.yaml +ExecStartPre=/usr/bin/vector validate +ExecStart=/usr/bin/vector +ExecReload=/usr/bin/vector validate +ExecReload=/bin/kill -HUP $MAINPID +Restart=on-failure + +[Install] +WantedBy=multi-user.target diff --git a/modules/l2/vector/mkosi.extra/etc/vault-agent/gomplate/vector.env.hcl b/modules/l2/vector/mkosi.extra/etc/vault-agent/gomplate/vector.env.hcl new file mode 100644 index 00000000..ddaf347c --- /dev/null +++ b/modules/l2/vector/mkosi.extra/etc/vault-agent/gomplate/vector.env.hcl @@ -0,0 +1,41 @@ +template { + left_delimiter = "((" + right_delimiter = "))" + + destination = "/etc/default/vector" + + user = "root" + group = "vector" + perms = "0640" + + exec { + timeout = "60s" + + command = ["/bin/sh", "-c", + <<-EOT + # vector.env + systemctl restart vector + EOT + ] + } + + contents = <<-EOT + ((- printf "# %s\n\n" "vector" -)) + + ((- $service := ( secret "[[ gcp.Meta "attributes/vault_kv_path" ]]/node/_common[[ if ( gcp.Meta "attributes/service" ) ]]_[[ gcp.Meta "attributes/service" | strings.ReplaceAll "-" "_" ]][[ end ]]" ).Data.data -)) + + L2_BUILDER_ENV=[[ gcp.Meta "attributes/environment" ]](( "\n" )) + + ((- if $service.clickhouse_endpoint -)) + CH_ENDPOINT=(( $service.clickhouse_endpoint ))(( "\n" )) + ((- end -)) + + ((- if $service.clickhouse_username -)) + CH_USER=(( $service.clickhouse_username ))(( "\n" )) + ((- end -)) + + ((- if $service.clickhouse_password -)) + CH_PASSWORD=(( $service.clickhouse_password ))(( "\n" )) + ((- end -)) + EOT +} diff --git a/modules/l2/vector/mkosi.extra/etc/vector/vector.yaml b/modules/l2/vector/mkosi.extra/etc/vector/vector.yaml new file mode 100644 index 00000000..4652ce2b --- /dev/null +++ b/modules/l2/vector/mkosi.extra/etc/vector/vector.yaml @@ -0,0 +1,156 @@ +data_dir: /var/lib/vector + +sources: + logs_journald: + type: journald + +transforms: + logs_journald_map_to_otel: + type: remap + inputs: [logs_journald] + source: |- + ts = .timestamp + msg = to_string(.message) ?? "" + hostname = to_string(.host) ?? "" + unit_raw = to_string(get!(value: ., path: ["_SYSTEMD_UNIT"])) ?? "" + syslog_id_early = to_string(get!(value: ., path: ["SYSLOG_IDENTIFIER"])) ?? "" + if unit_raw == "" { unit_raw = syslog_id_early } + if unit_raw == "" { unit_raw = "unknown" } + unit = replace(unit_raw, r'\.(service|scope|socket|timer|target|slice|mount|path|device)$', "") + priority = to_int(get!(value: ., path: ["PRIORITY"])) ?? 6 + pid = to_string(get!(value: ., path: ["_PID"])) ?? "" + syslog_id = to_string(get!(value: ., path: ["SYSLOG_IDENTIFIER"])) ?? "" + deployment_env = get_env_var("L2_BUILDER_ENV") ?? "" + if deployment_env == "" { deployment_env = "unknown" } + + # Syslog PRIORITY > OTel severity (RFC 5424). + syslog_severity = "info" + if priority == 0 || priority == 1 || priority == 2 { + syslog_severity = "fatal" + } else if priority == 3 { + syslog_severity = "error" + } else if priority == 4 { + syslog_severity = "warn" + } else if priority == 5 || priority == 6 { + syslog_severity = "info" + } else if priority == 7 { + syslog_severity = "debug" + } + + # Top-level OTEL severity — default from syslog PRIORITY, overridden below if + # the message is structured JSON with a `level` field. + severity_text = syslog_severity + severity_number_by_text = {"trace": 1, "debug": 5, "info": 9, "warn": 13, "error": 17, "fatal": 21} + severity_number = to_int(get!(value: severity_number_by_text, path: [syslog_severity])) ?? 0 + + ts_formatted = format_timestamp!(ts, format: "%Y-%m-%dT%H:%M:%S%.6f%:z") + + log_attrs = { + "@timestamp": ts_formatted, + "facility": "daemon", + "log.file.name": "journald", + "procid": pid, + "severity": syslog_severity, + "source": syslog_id, + "sysloghost": hostname + } + + # Try structured parse: JSON first, then logfmt (containerd/dockerd-style key=value). + # Only accept logfmt if a `level` key exists — avoids spurious matches on plain text + # that happens to contain "=" (e.g. sudo COMMAND= entries). + parsed = parse_json(msg) ?? null + if !is_object(parsed) { + maybe_logfmt = parse_logfmt(msg) ?? null + if is_object(maybe_logfmt) && exists(maybe_logfmt.level) { + parsed = maybe_logfmt + } + } + body_message = msg + if is_object(parsed) { + body_message = parsed + + msg_ts = to_string(parsed.timestamp) ?? "" + if msg_ts != "" { log_attrs."message.timestamp" = msg_ts } + msg_level = to_string(parsed.level) ?? "" + if msg_level != "" { log_attrs."message.level" = msg_level } + msg_target = to_string(parsed.target) ?? "" + if msg_target != "" { log_attrs."message.target" = msg_target } + fields = object(parsed.fields) ?? {} + fields_msg = to_string(fields.message) ?? "" + if fields_msg != "" { log_attrs."message.fields.message" = fields_msg } + + if msg_level != "" { + msg_level_lower = downcase(msg_level) + if msg_level_lower == "trace" { + severity_text = "trace" + severity_number = 1 + } else if msg_level_lower == "debug" { + severity_text = "debug" + severity_number = 5 + } else if msg_level_lower == "info" { + severity_text = "info" + severity_number = 9 + } else if msg_level_lower == "warn" || msg_level_lower == "warning" { + severity_text = "warn" + severity_number = 13 + } else if msg_level_lower == "error" { + severity_text = "error" + severity_number = 17 + } else if msg_level_lower == "fatal" { + severity_text = "fatal" + severity_number = 21 + } + } + } else { + body_message = msg + log_attrs.message = msg + } + + # Build composite Body wrapping message in syslog-like envelope + body = to_string(encode_json({ + "@timestamp": ts_formatted, + "sysloghost": hostname, + "procid": pid, + "facility": "daemon", + "severity": log_attrs.severity, + "source": syslog_id, + "message": body_message + })) + + . = { + "Timestamp": to_unix_timestamp!(ts, unit: "nanoseconds"), + "TraceId": "", + "SpanId": "", + "TraceFlags": 0, + "SeverityText": severity_text, + "SeverityNumber": severity_number, + "ServiceName": unit, + "Body": body, + "ResourceSchemaUrl": "https://opentelemetry.io/schemas/1.38.0", + "ResourceAttributes": { + "deployment.environment.name": "l2-builder-" + deployment_env, + "host.name": hostname, + "service.name": unit, + "source": syslog_id + }, + "ScopeSchemaUrl": "", + "ScopeName": "vector.service", + "ScopeVersion": "", + "ScopeAttributes": {}, + "LogAttributes": log_attrs, + "EventName": "" + } + +sinks: + clickhouse_logs: + type: clickhouse + inputs: [logs_journald_map_to_otel] + endpoint: "${CH_ENDPOINT:-http://clickstack.placeholder.invalid:8123}" + database: otel + table: otel_logs + auth: + strategy: basic + user: "${CH_USER:-placeholder}" + password: "${CH_PASSWORD:-placeholder}" + healthcheck: + enabled: false diff --git a/modules/l2/vector/mkosi.postinst.chroot b/modules/l2/vector/mkosi.postinst.chroot new file mode 100755 index 00000000..181f3b89 --- /dev/null +++ b/modules/l2/vector/mkosi.postinst.chroot @@ -0,0 +1,9 @@ +#!/bin/bash + +set -euxo pipefail + +systemctl add-wants minimal.target vector.service + +useradd -r -s /usr/sbin/nologin -d /var/lib/vector -G systemd-journal vector || true +install -d -o vector -g vector -m 0750 /var/lib/vector +install -d -o root -g root -m 0755 /etc/vector