Draft: adapt Vector to VRL ObjectMap

Cargo.toml

@@ -225,7 +225,7 @@ vector-common-macros = { path = "lib/vector-common-macros" }
 vector-lib = { path = "lib/vector-lib", default-features = false, features = ["vrl"] }
 vector-vrl-category = { path = "lib/vector-vrl/category" }
 vector-vrl-functions = { path = "lib/vector-vrl/functions", default-features = false }
-vrl = { git = "https://github.com/vectordotdev/vrl.git", branch = "main", default-features = false, features = ["arbitrary", "cli", "test", "test_framework", "stdlib-base"] }
+vrl = { git = "https://github.com/lukesteensen/vrl.git", branch = "experiment/objectmap-backends", default-features = false, features = ["arbitrary", "cli", "test", "test_framework", "stdlib-base"] }
 mock_instant = { version = "0.6" }
 serial_test = { version = "3.4" }
 strum = { version = "0.28", features = ["derive"] }

Dockerfile.bench

@@ -0,0 +1,36 @@
+#
+# VECTOR BUILDER (local bench builds with VRL override)
+#
+# Build with:
+#   docker buildx build --build-context vrl=../vrl -f Dockerfile.bench -t vector:<tag> .
+#
+FROM rust:1.92-bookworm AS builder
+RUN apt-get update && apt-get -y --no-install-recommends install \
+    cmake \
+    libclang-dev \
+    libsasl2-dev \
+    libssl-dev \
+    pkg-config \
+    protobuf-compiler \
+    && rm -rf /var/lib/apt/lists/*
+WORKDIR /vector
+COPY --from=vrl . /vrl
+COPY . .
+RUN --mount=type=cache,target=/usr/local/cargo/registry \
+    --mount=type=cache,target=/usr/local/cargo/git \
+    --mount=type=cache,target=/vector/target \
+    cargo build --bin vector --release && \
+    cp target/release/vector .
+
+#
+# TARGET
+#
+FROM debian:trixie-slim
+RUN apt-get update && apt-get dist-upgrade -y && apt-get -y --no-install-recommends install zlib1g ca-certificates libsasl2-2 && rm -rf /var/lib/apt/lists/*
+COPY --from=builder /vector/vector /usr/bin/vector
+RUN mkdir --parents --mode=0777 /var/lib/vector
+
+# Smoke test
+RUN ["/usr/bin/vector", "--version"]
+
+ENTRYPOINT ["/usr/bin/vector"]

SMP_NOTES.md

@@ -0,0 +1,140 @@
+# Local Regression Benchmarking Notes
+
+## What we're testing
+
+We're measuring the performance impact of several VRL (Vector Remap Language) optimizations on Vector's throughput and memory usage. The optimizations are:
+
+1. **Flat object map** — Replace BTreeMap with a flat array-backed map for storing event fields. Better cache locality, avoids tree traversal.
+2. **Copy elimination** — Remove unnecessary `String` allocations when constructing map keys in VRL.
+3. **KeyString as EcoString** — Change the map key type to `EcoString`, a reference-counted immutable string with cheap cloning.
+4. **KeyString as CompactString** — Change the map key type to `CompactString`, an inline small-string-optimized type that avoids heap allocation for short strings.
+
+The object map implementation is selected at runtime via the `VRL_OBJECT_MAP` env var (`btree`, `vec`, or the default `flat`). The KeyString and copy elimination changes are compile-time (different VRL builds).
+
+## Tooling
+
+### `bench.sh` — Build and run benchmarks
+
+```bash
+./bench.sh build <tag>                              # Build image from current source + ../vrl
+./bench.sh derive <base-tag> <new-tag> KEY=VAL ...  # Layer env vars on an existing image (instant)
+./bench.sh run <case> <baseline> <comparison> ...   # Run smp local comparison
+./bench.sh cases                                    # List regression cases
+```
+
+The script auto-detects Colima and sets `DOCKER_HOST` and `TMPDIR` accordingly.
+
+### `Dockerfile.bench` — Local build Dockerfile
+
+Two-stage build using `rust:1.92-bookworm` (for native ARM64 on Apple Silicon) with BuildKit named context `--build-context vrl=../vrl` to include the local VRL checkout. Uses cache mounts for fast incremental rebuilds.
+
+**Important**: The VRL repo at `../vrl` needs a `.dockerignore` excluding `target/` — without it, Docker sends ~25GB of build artifacts as context.
+
+### `bench-all.sh` — Batch runner
+
+Runs all regression cases across multiple variants. Skips completed runs (checks for `Δ mean` in output logs) and skips `file_to_blackhole`/`file_100_to_blackhole` (need FUSE, unavailable in Colima). Tolerates individual failures without aborting.
+
+### `smp local run`
+
+The actual comparison tool. Runs 3 replicates of each variant with 270 samples at 1Hz. Each case takes ~15 min (except `splunk_hec_route_s3` at ~32 min). Raw capture data (parquet) goes to `comparative-captures/<case>/`.
+
+## Docker image inventory
+
+All images use the same Vector source; they differ only in VRL version and the `VRL_OBJECT_MAP` env var.
+
+| Image | VRL version | Map type | Description |
+|---|---|---|---|
+| `vector:btree` | baseline (no opts) | btree | **The baseline for all comparisons** |
+| `vector:flat` | baseline | flat | Flat map only |
+| `vector:btree-ks` | + EcoString | btree | Old keystring on btree |
+| `vector:flat-ks` | + EcoString | flat | Old keystring on flat |
+| `vector:btree-co` | + copy elim | btree | Copy elimination only |
+| `vector:flat-co` | + copy elim | flat | Copy elimination only |
+| `vector:btree-co-ks` | + copy elim + EcoString | btree | Copy elim + EcoString |
+| `vector:flat-co-ks` | + copy elim + EcoString | flat | Copy elim + EcoString |
+| `vector:btree-co-ksc` | + copy elim + CompactString | btree | Copy elim + CompactString |
+| `vector:flat-co-ksc` | + copy elim + CompactString | flat | Copy elim + CompactString |
+
+To rebuild: the VRL repo (`../vrl`) has copy elimination committed on `main`. KeyString variants are managed via `git stash`:
+- **EcoString**: `git stash pop` the older stash
+- **CompactString**: dirty working tree state (or stash pop the newer stash)
+- **No keystring**: `git stash` everything
+
+## Results
+
+### Complete data: flat map across all cases
+
+Throughput vs btree baseline, from `bench-results/full/`:
+
+| Tier | Cases | Flat map Δ |
+|---|---|---|
+| High (heavy VRL) | datadog_agent_remap_blackhole | +35.5% |
+| | datadog_agent_remap_blackhole_acks | +30.1% |
+| | syslog_humio_logs | +25.8% |
+| | syslog_log2metric_splunk_hec_metrics | +19.8% |
+| | syslog_splunk_hec_logs | +18.4% |
+| | syslog_loki | +18.3% |
+| | datadog_agent_remap_datadog_logs_acks | +18.5% |
+| | syslog_log2metric_humio_metrics | +17.1% |
+| | syslog_log2metric_tag_cardinality_limit_blackhole | +16.9% |
+| | datadog_agent_remap_datadog_logs | +16.2% |
+| | syslog_regex_logs2metric_ddmetrics | +15.0% |
+| Medium | http_text_to_http_json | +13.8% |
+| | statsd_to_datadog_metrics | +1.6% |
+| Low (passthrough) | all http_to_http, socket, splunk_hec, fluent, otlp | ~0% (no regression) |
+
+Memory (RSS) is 3–8% lower with flat map on VRL-heavy workloads.
+
+### Optimization interaction study (datadog_agent_remap_blackhole)
+
+All measurements vs old btree baseline:
+
+| Configuration | Throughput Δ |
+|---|---|
+| flat map only | +29.2% |
+| btree + copy elim | -0.8% |
+| btree + copy elim + EcoString | +7.4% |
+| btree + copy elim + CompactString | +2.7% |
+| flat + copy elim + EcoString | +22.8% |
+| flat + copy elim + CompactString | +29.2% |
+
+Key findings:
+- **EcoString helps btree (+7.4%) but hurts flat (-9.6% vs flat alone)**. The reference-counting indirection undermines flat map cache locality.
+- **CompactString is neutral on flat**. Inline small-string storage preserves cache behavior. Full +29.2% recovered.
+- **Copy elimination is near-zero on its own** but is a prerequisite for the KeyString changes.
+
+### Partially complete: full matrix
+
+`bench-results/full/` has results for `flat` (all 25 working cases) and `flat-co-ks` (15 of 25 cases). The remaining variants (`flat-co-ksc`, `btree-co-ks`, `btree-co-ksc`) have not been run across all cases yet.
+
+## Resuming the batch run
+
+```bash
+# The script skips completed runs automatically
+caffeinate -d -i -s bash -c './bench-all.sh'
+```
+
+Colima must be running with 10 CPUs and sufficient disk:
+
+```bash
+colima start --cpu 10 --memory 16 --disk 60
+```
+
+## Infrastructure notes
+
+- **Colima on Apple Silicon**: The `timberio/vector-dev` image is amd64-only. We use `rust:1.92-bookworm` instead (has native arm64).
+- **Docker buildx**: Required for `--build-context`. Install: `brew install docker-buildx`, then `mkdir -p ~/.docker/cli-plugins && ln -sfn $(brew --prefix docker-buildx)/bin/docker-buildx ~/.docker/cli-plugins/docker-buildx`.
+- **smp + Colima**: smp needs `DOCKER_HOST` set to the Colima socket, and `TMPDIR` under `$HOME` (Colima doesn't mount `/var/folders`). The `bench.sh` script handles this automatically.
+- **FUSE cases**: `file_to_blackhole` and `file_100_to_blackhole` need FUSE, unavailable in Colima VMs. Skipped.
+- **Disk space**: The Colima VM disk fills up from Docker build cache. Run `docker builder prune --all -f` before large rebuilds.
+- **VRL .dockerignore**: Must contain `target` to avoid sending 25GB of build artifacts as Docker context.
+
+## File locations
+
+- `bench.sh` — Main benchmarking script
+- `Dockerfile.bench` — Build Dockerfile
+- `bench-all.sh` — Batch runner for full matrix
+- `bench-results/` — Ad-hoc comparison logs from early exploration
+- `bench-results/full/` — Systematic matrix results (`<case>--<variant>.log`)
+- `comparative-captures/` — Raw smp parquet data (overwritten by each run; only latest is available)
+- `../vrl/.dockerignore` — Must exist with `target` entry

bench-all.sh

@@ -0,0 +1,77 @@
+#!/usr/bin/env bash
+set -uo pipefail
+
+BENCH="/Users/luke.steensen/code/vector/bench.sh"
+RESULTS_DIR="/Users/luke.steensen/code/vector/bench-results/full"
+mkdir -p "$RESULTS_DIR"
+
+VARIANTS=(
+  flat
+  flat-co-ks
+  flat-co-ksc
+  btree-co-ks
+  btree-co-ksc
+)
+
+CASES=(
+  datadog_agent_remap_blackhole
+  datadog_agent_remap_blackhole_acks
+  datadog_agent_remap_datadog_logs
+  datadog_agent_remap_datadog_logs_acks
+  file_100_to_blackhole
+  file_to_blackhole
+  fluent_elasticsearch
+  http_elasticsearch
+  http_text_to_http_json
+  http_to_http_acks
+  http_to_http_disk_buffer
+  http_to_http_json
+  http_to_http_noack
+  http_to_s3
+  otlp_grpc_to_blackhole
+  otlp_http_to_blackhole
+  socket_to_socket_blackhole
+  splunk_hec_indexer_ack_blackhole
+  splunk_hec_route_s3
+  splunk_hec_to_splunk_hec_logs_acks
+  splunk_hec_to_splunk_hec_logs_noack
+  statsd_to_datadog_metrics
+  syslog_humio_logs
+  syslog_log2metric_humio_metrics
+  syslog_log2metric_splunk_hec_metrics
+  syslog_log2metric_tag_cardinality_limit_blackhole
+  syslog_loki
+  syslog_regex_logs2metric_ddmetrics
+  syslog_splunk_hec_logs
+)
+
+total=$(( ${#VARIANTS[@]} * ${#CASES[@]} ))
+n=0
+
+for variant in "${VARIANTS[@]}"; do
+  for case_name in "${CASES[@]}"; do
+    n=$((n + 1))
+    outfile="${RESULTS_DIR}/${case_name}--${variant}.log"
+
+    # Skip if already completed
+    if [[ -f "$outfile" ]] && grep -q "Δ mean" "$outfile" 2>/dev/null; then
+      echo "[$n/$total] SKIP (already done): ${case_name} -- btree vs ${variant}"
+      continue
+    fi
+
+    # Skip known-broken cases (need FUSE)
+    if [[ "$case_name" == "file_100_to_blackhole" || "$case_name" == "file_to_blackhole" ]]; then
+      echo "[$n/$total] SKIP (needs FUSE): ${case_name} -- btree vs ${variant}"
+      continue
+    fi
+
+    echo "[$n/$total] Running: ${case_name} -- btree vs ${variant}"
+    docker rm -f $(docker ps -aq) 2>/dev/null || true
+    if ! "${BENCH}" run "${case_name}" btree "${variant}" 2>&1 | tee "${outfile}"; then
+      echo "[$n/$total] FAILED: ${case_name} -- btree vs ${variant}"
+    fi
+    echo ""
+  done
+done
+
+echo "=== ALL DONE ==="

bench.sh

@@ -0,0 +1,87 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+# Auto-detect Colima Docker socket and set TMPDIR to a path the VM can mount
+COLIMA_SOCK="$HOME/.colima/default/docker.sock"
+if [[ -z "${DOCKER_HOST:-}" && -S "${COLIMA_SOCK}" ]]; then
+    export DOCKER_HOST="unix://${COLIMA_SOCK}"
+fi
+if [[ -n "${DOCKER_HOST:-}" && "${DOCKER_HOST}" == *colima* ]]; then
+    # macOS /var/folders tmp dir isn't mounted in Colima; use a home-dir path
+    mkdir -p "$HOME/.tmp-smp"
+    export TMPDIR="$HOME/.tmp-smp"
+fi
+
+usage() {
+    cat <<EOF
+Usage: $0 <command> [args...]
+
+Commands:
+  build  <tag>                              Build a Vector image from current source
+  derive <base-tag> <new-tag> KEY=VAL ...   Create a variant image with env vars
+  run    <case> <baseline> <comparison> ..  Run an smp comparison
+  cases                                     List available regression cases
+EOF
+    exit 1
+}
+
+cmd_build() {
+    local tag="${1:?Usage: $0 build <tag>}"
+    echo "Building vector:${tag} ..."
+    DOCKER_BUILDKIT=1 docker buildx build \
+        --build-context "vrl=${SCRIPT_DIR}/../vrl" \
+        -f "${SCRIPT_DIR}/Dockerfile.bench" \
+        -t "vector:${tag}" \
+        "${SCRIPT_DIR}"
+}
+
+cmd_derive() {
+    if [[ $# -lt 3 ]]; then
+        echo "Usage: $0 derive <base-tag> <new-tag> KEY=VAL [KEY=VAL ...]" >&2
+        exit 1
+    fi
+    local base="$1"; shift
+    local new_tag="$1"; shift
+
+    local dockerfile="FROM vector:${base}"
+    for kv in "$@"; do
+        dockerfile="${dockerfile}"$'\n'"ENV ${kv}"
+    done
+
+    echo "Deriving vector:${new_tag} from vector:${base} ..."
+    echo "${dockerfile}" | docker buildx build -t "vector:${new_tag}" -
+}
+
+cmd_run() {
+    if [[ $# -lt 3 ]]; then
+        echo "Usage: $0 run <case> <baseline-tag> <comparison-tag> [smp args...]" >&2
+        exit 1
+    fi
+    local case_name="$1"; shift
+    local baseline="$1"; shift
+    local comparison="$1"; shift
+
+    smp local run \
+        --experiment-dir "${SCRIPT_DIR}/regression" \
+        --case "${case_name}" \
+        --baseline-image "vector:${baseline}" \
+        --comparison-image "vector:${comparison}" \
+        "$@"
+}
+
+cmd_cases() {
+    ls "${SCRIPT_DIR}/regression/cases/"
+}
+
+[[ $# -lt 1 ]] && usage
+
+command="$1"; shift
+case "${command}" in
+    build)  cmd_build "$@" ;;
+    derive) cmd_derive "$@" ;;
+    run)    cmd_run "$@" ;;
+    cases)  cmd_cases ;;
+    *)      usage ;;
+esac