From b50aeeb9a02179e7d9ea58b87519d4c1bae0db17 Mon Sep 17 00:00:00 2001 From: Stephen Wakely Date: Thu, 11 Jun 2026 09:07:40 +0100 Subject: [PATCH 1/5] feat(lading-py): Python port of lading with DogStatsD emission via dogstatsd-py Adds a Python port of lading under lading_py/ that uses the dogstatsd-py library for all metric emission, enabling direct testing of that client under realistic load patterns. All supporting lading capabilities are preserved: Prometheus and expvar telemetry collection from a running Datadog Agent, JSONL/Parquet capture output, passive Prometheus exporter, HTTP blackhole, /proc observer, and the full lading YAML config schema. The Dockerfile is updated from the Rust multi-stage build to a two-stage Python build, dropping build time from several minutes to ~30 seconds. Co-Authored-By: Claude Sonnet 4.6 (1M context) --- Dockerfile | 97 +-- lading_py/README.md | 254 +++++++ lading_py/lading_py/__init__.py | 0 .../__pycache__/__init__.cpython-310.pyc | Bin 0 -> 148 bytes .../__pycache__/config.cpython-310.pyc | Bin 0 -> 8350 bytes .../__pycache__/main.cpython-310.pyc | Bin 0 -> 3792 bytes .../__pycache__/signal.cpython-310.pyc | Bin 0 -> 883 bytes lading_py/lading_py/blackhole/__init__.py | 0 .../__pycache__/__init__.cpython-310.pyc | Bin 0 -> 158 bytes .../__pycache__/http.cpython-310.pyc | Bin 0 -> 1756 bytes lading_py/lading_py/blackhole/http.py | 30 + lading_py/lading_py/capture/__init__.py | 0 .../__pycache__/__init__.cpython-310.pyc | Bin 0 -> 156 bytes .../__pycache__/accumulator.cpython-310.pyc | Bin 0 -> 2296 bytes .../__pycache__/jsonl_writer.cpython-310.pyc | Bin 0 -> 1031 bytes .../capture/__pycache__/line.cpython-310.pyc | Bin 0 -> 1247 bytes .../parquet_writer.cpython-310.pyc | Bin 0 -> 2354 bytes lading_py/lading_py/capture/accumulator.py | 80 +++ lading_py/lading_py/capture/jsonl_writer.py | 20 + lading_py/lading_py/capture/line.py | 36 + lading_py/lading_py/capture/parquet_writer.py | 48 ++ lading_py/lading_py/config.py | 217 ++++++ lading_py/lading_py/generator/__init__.py | 0 .../__pycache__/__init__.cpython-310.pyc | Bin 0 -> 158 bytes .../__pycache__/dogstatsd.cpython-310.pyc | Bin 0 -> 4517 bytes lading_py/lading_py/generator/dogstatsd.py | 129 ++++ lading_py/lading_py/main.py | 131 ++++ lading_py/lading_py/observer/__init__.py | 0 .../__pycache__/__init__.cpython-310.pyc | Bin 0 -> 157 bytes .../observer/__pycache__/proc.cpython-310.pyc | Bin 0 -> 2400 bytes lading_py/lading_py/observer/proc.py | 61 ++ lading_py/lading_py/payload/__init__.py | 0 .../__pycache__/__init__.cpython-310.pyc | Bin 0 -> 156 bytes .../__pycache__/dogstatsd.cpython-310.pyc | Bin 0 -> 7741 bytes lading_py/lading_py/payload/dogstatsd.py | 228 +++++++ lading_py/lading_py/signal.py | 17 + .../lading_py/target_metrics/__init__.py | 0 .../__pycache__/__init__.cpython-310.pyc | Bin 0 -> 163 bytes .../__pycache__/expvar.cpython-310.pyc | Bin 0 -> 2269 bytes .../__pycache__/prometheus.cpython-310.pyc | Bin 0 -> 3149 bytes lading_py/lading_py/target_metrics/expvar.py | 54 ++ .../lading_py/target_metrics/prometheus.py | 82 +++ lading_py/lading_py/telemetry/__init__.py | 0 .../__pycache__/__init__.cpython-310.pyc | Bin 0 -> 158 bytes .../prometheus_exporter.cpython-310.pyc | Bin 0 -> 2615 bytes .../__pycache__/registry.cpython-310.pyc | Bin 0 -> 2200 bytes .../telemetry/prometheus_exporter.py | 48 ++ lading_py/lading_py/telemetry/registry.py | 39 ++ lading_py/pyproject.toml | 23 + .../smoke_test.cpython-310-pytest-9.0.2.pyc | Bin 0 -> 4124 bytes .../test_capture.cpython-310-pytest-9.0.2.pyc | Bin 0 -> 23048 bytes .../test_config.cpython-310-pytest-9.0.2.pyc | Bin 0 -> 16070 bytes ...est_generator.cpython-310-pytest-9.0.2.pyc | Bin 0 -> 9434 bytes .../test_payload.cpython-310-pytest-9.0.2.pyc | Bin 0 -> 20786 bytes ...test_registry.cpython-310-pytest-9.0.2.pyc | Bin 0 -> 10718 bytes ...arget_metrics.cpython-310-pytest-9.0.2.pyc | Bin 0 -> 16185 bytes lading_py/tests/smoke_test.py | 96 +++ lading_py/tests/test_capture.py | 321 +++++++++ lading_py/tests/test_config.py | 140 ++++ lading_py/tests/test_generator.py | 176 +++++ lading_py/tests/test_payload.py | 270 ++++++++ lading_py/tests/test_registry.py | 133 ++++ lading_py/tests/test_target_metrics.py | 175 +++++ plans/pythonport.md | 617 ++++++++++++++++++ 64 files changed, 3436 insertions(+), 86 deletions(-) create mode 100644 lading_py/README.md create mode 100644 lading_py/lading_py/__init__.py create mode 100644 lading_py/lading_py/__pycache__/__init__.cpython-310.pyc create mode 100644 lading_py/lading_py/__pycache__/config.cpython-310.pyc create mode 100644 lading_py/lading_py/__pycache__/main.cpython-310.pyc create mode 100644 lading_py/lading_py/__pycache__/signal.cpython-310.pyc create mode 100644 lading_py/lading_py/blackhole/__init__.py create mode 100644 lading_py/lading_py/blackhole/__pycache__/__init__.cpython-310.pyc create mode 100644 lading_py/lading_py/blackhole/__pycache__/http.cpython-310.pyc create mode 100644 lading_py/lading_py/blackhole/http.py create mode 100644 lading_py/lading_py/capture/__init__.py create mode 100644 lading_py/lading_py/capture/__pycache__/__init__.cpython-310.pyc create mode 100644 lading_py/lading_py/capture/__pycache__/accumulator.cpython-310.pyc create mode 100644 lading_py/lading_py/capture/__pycache__/jsonl_writer.cpython-310.pyc create mode 100644 lading_py/lading_py/capture/__pycache__/line.cpython-310.pyc create mode 100644 lading_py/lading_py/capture/__pycache__/parquet_writer.cpython-310.pyc create mode 100644 lading_py/lading_py/capture/accumulator.py create mode 100644 lading_py/lading_py/capture/jsonl_writer.py create mode 100644 lading_py/lading_py/capture/line.py create mode 100644 lading_py/lading_py/capture/parquet_writer.py create mode 100644 lading_py/lading_py/config.py create mode 100644 lading_py/lading_py/generator/__init__.py create mode 100644 lading_py/lading_py/generator/__pycache__/__init__.cpython-310.pyc create mode 100644 lading_py/lading_py/generator/__pycache__/dogstatsd.cpython-310.pyc create mode 100644 lading_py/lading_py/generator/dogstatsd.py create mode 100644 lading_py/lading_py/main.py create mode 100644 lading_py/lading_py/observer/__init__.py create mode 100644 lading_py/lading_py/observer/__pycache__/__init__.cpython-310.pyc create mode 100644 lading_py/lading_py/observer/__pycache__/proc.cpython-310.pyc create mode 100644 lading_py/lading_py/observer/proc.py create mode 100644 lading_py/lading_py/payload/__init__.py create mode 100644 lading_py/lading_py/payload/__pycache__/__init__.cpython-310.pyc create mode 100644 lading_py/lading_py/payload/__pycache__/dogstatsd.cpython-310.pyc create mode 100644 lading_py/lading_py/payload/dogstatsd.py create mode 100644 lading_py/lading_py/signal.py create mode 100644 lading_py/lading_py/target_metrics/__init__.py create mode 100644 lading_py/lading_py/target_metrics/__pycache__/__init__.cpython-310.pyc create mode 100644 lading_py/lading_py/target_metrics/__pycache__/expvar.cpython-310.pyc create mode 100644 lading_py/lading_py/target_metrics/__pycache__/prometheus.cpython-310.pyc create mode 100644 lading_py/lading_py/target_metrics/expvar.py create mode 100644 lading_py/lading_py/target_metrics/prometheus.py create mode 100644 lading_py/lading_py/telemetry/__init__.py create mode 100644 lading_py/lading_py/telemetry/__pycache__/__init__.cpython-310.pyc create mode 100644 lading_py/lading_py/telemetry/__pycache__/prometheus_exporter.cpython-310.pyc create mode 100644 lading_py/lading_py/telemetry/__pycache__/registry.cpython-310.pyc create mode 100644 lading_py/lading_py/telemetry/prometheus_exporter.py create mode 100644 lading_py/lading_py/telemetry/registry.py create mode 100644 lading_py/pyproject.toml create mode 100644 lading_py/tests/__pycache__/smoke_test.cpython-310-pytest-9.0.2.pyc create mode 100644 lading_py/tests/__pycache__/test_capture.cpython-310-pytest-9.0.2.pyc create mode 100644 lading_py/tests/__pycache__/test_config.cpython-310-pytest-9.0.2.pyc create mode 100644 lading_py/tests/__pycache__/test_generator.cpython-310-pytest-9.0.2.pyc create mode 100644 lading_py/tests/__pycache__/test_payload.cpython-310-pytest-9.0.2.pyc create mode 100644 lading_py/tests/__pycache__/test_registry.cpython-310-pytest-9.0.2.pyc create mode 100644 lading_py/tests/__pycache__/test_target_metrics.cpython-310-pytest-9.0.2.pyc create mode 100644 lading_py/tests/smoke_test.py create mode 100644 lading_py/tests/test_capture.py create mode 100644 lading_py/tests/test_config.py create mode 100644 lading_py/tests/test_generator.py create mode 100644 lading_py/tests/test_payload.py create mode 100644 lading_py/tests/test_registry.py create mode 100644 lading_py/tests/test_target_metrics.py create mode 100644 plans/pythonport.md diff --git a/Dockerfile b/Dockerfile index d8badb9f1..17b66ff88 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,90 +1,15 @@ -# Update the rust version in-sync with the version in rust-toolchain.toml - -# Stage 0: Planner - Extract dependency metadata -FROM docker.io/rust:1.90.0-slim-bookworm AS planner -WORKDIR /app -RUN cargo install cargo-chef --version 0.1.73 -COPY . . -RUN cargo chef prepare --recipe-path recipe.json - -# Stage 1: Cacher - Build dependencies only -FROM docker.io/rust:1.90.0-slim-bookworm AS cacher -ARG SCCACHE_BUCKET -ARG SCCACHE_REGION -ARG AWS_ACCESS_KEY_ID -ARG AWS_SECRET_ACCESS_KEY -ARG AWS_SESSION_TOKEN -ENV CARGO_INCREMENTAL=0 +# Stage 0: Build — install dependencies into a venv +FROM docker.io/python:3.12-slim-bookworm AS builder WORKDIR /app -RUN apt-get update && apt-get install -y \ - pkg-config=1.8.1-1 \ - libssl-dev=3.0.18-1~deb12u2 \ - protobuf-compiler=3.21.12-3 \ - fuse3=3.14.0-4 \ - libfuse3-dev=3.14.0-4 \ - curl \ - && rm -rf /var/lib/apt/lists/* -# Download pre-built sccache binary -RUN case "$(uname -m)" in \ - x86_64) ARCH=x86_64-unknown-linux-musl ;; \ - aarch64) ARCH=aarch64-unknown-linux-musl ;; \ - *) echo "Unsupported architecture" && exit 1 ;; \ - esac && \ - curl -L https://github.com/mozilla/sccache/releases/download/v0.8.2/sccache-v0.8.2-${ARCH}.tar.gz | tar xz && \ - mv sccache-v0.8.2-${ARCH}/sccache /usr/local/cargo/bin/ && \ - rm -rf sccache-v0.8.2-${ARCH} -RUN cargo install cargo-chef --version 0.1.73 -COPY --from=planner /app/recipe.json recipe.json -# This layer is cached until Cargo.toml/Cargo.lock change -# Use BuildKit secrets to pass AWS credentials securely (not exposed in image metadata) -RUN --mount=type=secret,id=aws_access_key_id \ - --mount=type=secret,id=aws_secret_access_key \ - --mount=type=secret,id=aws_session_token \ - export AWS_ACCESS_KEY_ID=$(cat /run/secrets/aws_access_key_id) && \ - export AWS_SECRET_ACCESS_KEY=$(cat /run/secrets/aws_secret_access_key) && \ - export AWS_SESSION_TOKEN=$(cat /run/secrets/aws_session_token) && \ - if [ -n "${SCCACHE_BUCKET:-}" ]; then export RUSTC_WRAPPER=sccache; fi && \ - cargo chef cook --release --locked --features logrotate_fs --recipe-path recipe.json +RUN pip install --upgrade pip +COPY lading_py/ lading_py/ +RUN pip install --prefix=/install lading_py/ -# Stage 2: Builder - Build source code -FROM docker.io/rust:1.90.0-slim-bookworm AS builder -ARG SCCACHE_BUCKET -ARG SCCACHE_REGION -ENV CARGO_INCREMENTAL=0 -ENV SCCACHE_BUCKET=${SCCACHE_BUCKET} -ENV SCCACHE_REGION=${SCCACHE_REGION} -WORKDIR /app -RUN apt-get update && apt-get install -y \ - pkg-config=1.8.1-1 \ - libssl-dev=3.0.18-1~deb12u2 \ - protobuf-compiler=3.21.12-3 \ - fuse3=3.14.0-4 \ - libfuse3-dev=3.14.0-4 \ - && rm -rf /var/lib/apt/lists/* -# Copy cached dependencies and sccache from cacher -COPY --from=cacher /app/target target -COPY --from=cacher /usr/local/cargo /usr/local/cargo -# Copy source code (frequently changes) -COPY . . -# Build binary - reuses cached dependencies + sccache -# Use BuildKit secrets to pass AWS credentials securely (not exposed in image metadata) -RUN --mount=type=secret,id=aws_access_key_id \ - --mount=type=secret,id=aws_secret_access_key \ - --mount=type=secret,id=aws_session_token \ - export AWS_ACCESS_KEY_ID=$(cat /run/secrets/aws_access_key_id) && \ - export AWS_SECRET_ACCESS_KEY=$(cat /run/secrets/aws_secret_access_key) && \ - export AWS_SESSION_TOKEN=$(cat /run/secrets/aws_session_token) && \ - if [ -n "${SCCACHE_BUCKET:-}" ]; then export RUSTC_WRAPPER=sccache; fi && \ - cargo build --release --locked --bin lading --features logrotate_fs - -# Stage 3: Runtime -FROM docker.io/debian:bookworm-20241202-slim -RUN apt-get update && apt-get install -y \ - libfuse3-dev=3.14.0-4 \ - fuse3=3.14.0-4 \ - && rm -rf /var/lib/apt/lists/* -COPY --from=builder /app/target/release/lading /usr/bin/lading +# Stage 1: Runtime +FROM docker.io/python:3.12-slim-bookworm +COPY --from=builder /install /usr/local # Smoke test -RUN ["/usr/bin/lading", "--help"] -ENTRYPOINT ["/usr/bin/lading"] +RUN lading-py --help + +ENTRYPOINT ["lading-py"] diff --git a/lading_py/README.md b/lading_py/README.md new file mode 100644 index 000000000..ef3397136 --- /dev/null +++ b/lading_py/README.md @@ -0,0 +1,254 @@ +# lading-py + +A Python port of [lading](https://github.com/datadog/lading) focused on DogStatsD +load generation. Uses the [dogstatsd-py](https://github.com/DataDog/datadogpy) +library for all metric emission, making it suitable for testing the client library +itself under realistic load. + +All other lading capabilities are preserved: Prometheus and expvar telemetry +collection from a running Datadog Agent, JSONL/Parquet capture output, and a +passive Prometheus exporter for real-time scraping. + +## Requirements + +- Python 3.10+ +- A Unix domain socket to send DogStatsD traffic to (typically the Datadog Agent's + `/tmp/dsd.socket` or `DD_DOGSTATSD_SOCKET`) + +## Installation + +```bash +pip install -e /path/to/lading_py +``` + +Or from the directory: + +```bash +cd lading_py +pip install -e . +``` + +This installs the `lading-py` command. + +## Configuration + +lading-py uses the same YAML config format as the Rust lading binary. A minimal +config that sends DogStatsD metrics and writes a JSONL capture file: + +```yaml +generator: + - unix_datagram: + seed: [2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37, 41, 43, 47, 53, + 59, 61, 67, 71, 73, 79, 83, 89, 97, 101, 103, 107, 109, 113, 127, 131] + path: "/tmp/dsd.socket" + bytes_per_second: "1 MiB" + parallel_connections: 1 + variant: + dogstatsd: + contexts: + inclusive: + min: 50 + max: 50 + tags_per_msg: + inclusive: + min: 3 + max: 3 + kind_weights: + metric: 90 + event: 5 + service_check: 5 + metric_weights: + count: 1 + gauge: 1 + distribution: 3 + timer: 1 + set: 0 + histogram: 0 + metric_names: + - myapp.requests{{0-9}} + tag_names: + - env + - service + - version + tag_values: + - prod{{0-2}} + +telemetry: + path: "/tmp/lading-output.jsonl" + +warmup_duration_secs: 5 +experiment_duration_secs: 60 +``` + +### Config reference + +#### `generator[].unix_datagram` + +| Field | Type | Default | Description | +|-------|------|---------|-------------| +| `seed` | list[int] (32 bytes) | required | RNG seed for deterministic payload generation | +| `path` | string | required | Unix domain socket path | +| `bytes_per_second` | string | `"1 MiB"` | Rate limit. Accepts human-readable sizes: `"500 KiB"`, `"4 MiB"`, `"1 GiB"` | +| `parallel_connections` | int | `1` | Number of concurrent sender threads | +| `variant.dogstatsd` | object | | DogStatsD payload config (see below) | + +#### `variant.dogstatsd` + +| Field | Type | Default | Description | +|-------|------|---------|-------------| +| `contexts` | ConfRange | `{inclusive: {min: 50, max: 50}}` | Number of unique metric contexts (name + tag set) to pre-generate | +| `tags_per_msg` | ConfRange | `{inclusive: {min: 3, max: 3}}` | Tags per metric | +| `multivalue_count` | ConfRange | `{inclusive: {min: 2, max: 32}}` | Messages per batch when multi-value packing fires | +| `multivalue_pack_probability` | float | `0.08` | Probability of packing multiple metrics into one datagram | +| `kind_weights` | object | `{metric: 90, event: 0, service_check: 0}` | Relative weight of each DogStatsD message kind | +| `metric_weights` | object | `{distribution: 5, ...rest 0}` | Relative weight of each metric type | +| `metric_names` | list[string] | `["metric{{0-9}}"]` | Metric name templates. `{{0-9}}` expands to 10 variants | +| `tag_names` | list[string] | `["tag1","tag2","tag3"]` | Tag name templates | +| `tag_values` | list[string] | `["value{{0-9}}"]` | Tag value templates | +| `sampling_range` | ConfRange | `{inclusive: {min: 0.1, max: 1.0}}` | Range for sample rate values | +| `sampling_probability` | float | `0.5` | Probability that a metric includes a sample rate | +| `length_prefix_framed` | bool | `false` | **Unsupported** — lading-py will reject configs with this set to `true` | + +#### `telemetry` + +Short form (JSONL output): +```yaml +telemetry: + path: "/tmp/output.jsonl" +``` + +Long form with format control: +```yaml +telemetry: + log: + path: "/tmp/output" + format: + jsonl: + flush_seconds: 60 + # or: parquet: {flush_seconds: 60} + # or: multi: {flush_seconds: 60} # writes both .jsonl and .parquet +``` + +Prometheus exporter (passive scrape endpoint): +```yaml +telemetry: + prometheus: + addr: "0.0.0.0:9000" +``` + +#### `target_metrics` + +Collect telemetry from a running Datadog Agent: + +```yaml +target_metrics: + - prometheus: + uri: "http://127.0.0.1:5000/telemetry" + tags: + sub_agent: "core" + - expvar: + uri: "http://127.0.0.1:5012/debug/vars" + vars: + - "/forwarder/Transactions/Success" + - "/uptime" + tags: + sub_agent: "trace" + +sample_period_milliseconds: 1000 +``` + +#### `blackhole` + +Absorb HTTP traffic from the target (e.g. agent intake forwarder in test): + +```yaml +blackhole: + - http: + binding_addr: "127.0.0.1:9091" +``` + +#### Lifecycle + +```yaml +warmup_duration_secs: 10 # wait before starting emission +experiment_duration_secs: 60 # how long to run after warmup +``` + +## Running + +```bash +lading-py --config lading.yaml +``` + +The process runs for `warmup_duration_secs + experiment_duration_secs` seconds, +then exits. The capture file (if configured) is finalized on exit. + +## Output format + +### JSONL + +One JSON object per line, one line per metric per flush interval: + +```json +{"run_id": "550e8400-...", "time": 1717959420000, "fetch_index": 0, "metric_name": "bytes_written", "metric_kind": "counter", "value": 1048576.0, "labels": {"generator": "dogstatsd"}} +{"run_id": "550e8400-...", "time": 1717959420000, "fetch_index": 0, "metric_name": "cpu_usage", "metric_kind": "gauge", "value": 0.73, "labels": {"sub_agent": "core"}} +``` + +Fields: + +| Field | Type | Description | +|-------|------|-------------| +| `run_id` | UUID string | Unique identifier for this lading-py run | +| `time` | int | Milliseconds since Unix epoch | +| `fetch_index` | int | Flush counter (increments each flush interval) | +| `metric_name` | string | Metric name | +| `metric_kind` | string | `"counter"`, `"gauge"`, or `"histogram"` | +| `value` | float | Counter delta, gauge value, or histogram mean | +| `labels` | object | Key-value label pairs | +| `value_histogram` | string (base64) | Protobuf DDSketch bytes (omitted if empty) | + +### Parquet + +Same schema as JSONL, written as columnar Parquet. Suitable for analysis with +pandas, DuckDB, or similar: + +```python +import pyarrow.parquet as pq +table = pq.read_table("/tmp/output.parquet") +df = table.to_pandas() +``` + +## Docker + +```bash +docker build -t lading-py /path/to/lading +docker run --rm \ + -v /tmp/dsd.socket:/tmp/dsd.socket \ + -v /path/to/lading.yaml:/etc/lading/lading.yaml \ + -v /tmp/output:/tmp/output \ + lading-py --config /etc/lading/lading.yaml +``` + +## Differences from Rust lading + +| Feature | Rust lading | lading-py | +|---------|------------|-----------| +| Emission library | Raw Unix datagram socket | `dogstatsd-py` (`datadog` package) | +| Generators | TCP, UDP, HTTP, Unix stream, Fluent, OTLP, DogStatsD | DogStatsD only | +| `length_prefix_framed` | Supported | **Not supported** (rejected at config load) | +| RNG | ChaCha (SeededStdRng) | Mersenne Twister (`random.Random`) | +| Reproducibility | Bit-exact across runs with same seed | Statistically equivalent; not bit-exact | +| Histogram output | Full DDSketch protobuf | Mean value only; `value_histogram` always empty | + +## Development + +```bash +pip install -e ".[dev]" +pytest tests/ +``` + +Run just the unit tests (fast, no socket needed): + +```bash +pytest tests/ -k "not smoke" +``` diff --git a/lading_py/lading_py/__init__.py b/lading_py/lading_py/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/lading_py/lading_py/__pycache__/__init__.cpython-310.pyc b/lading_py/lading_py/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..53ac151417dc48f4cbfb1734e34d79b37b6b1d1a GIT binary patch literal 148 zcmd1j<>g`kf{VvBvq1D?5P=LBfgA@QE@lA|DGb33nv8xc8Hzx{2;!Hyenx(7s(x`v zYC%S7UU_16YEGqoaZ$2J~J<~BtBlRpz;=nO>TZlX-=vg$iQMI IAi=@_03!AxTmS$7 literal 0 HcmV?d00001 diff --git a/lading_py/lading_py/__pycache__/config.cpython-310.pyc b/lading_py/lading_py/__pycache__/config.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..43813f88f29ce672c152174d7a41ed54b9851944 GIT binary patch literal 8350 zcmbVROK=-UdY%~!p2UkFMN*<3kmZM9S(I$Ysa>tsdeT}a7VLZ4BY2X<#0eph-4DhVT z0naf$0X#1Xzzd9LffvOj@JYsVazds>3E!ztEHV9wS!1#Qs9vpF*_J3^2nSx zJ{XzAz9+;y_MPW_OTbTxQ@~F#J_Y==I0O6)ijL!pK5SM^oV*I2qmyF8g zT{@v^#niDIt<8$1(>L8f-fau%>q3)$oJX$j2{&vjZ3a4_WY|@$I!s{4-9igO zfY8`7LUY?v$t`0E=PWD{6Sj!c35wJYlTXah+KGvDXm1(Y=1%;n@zi=^ZCSzD(4>(i zBTQ^rJ4s7C9!c6@4 zw2hv5+MG7kSxnT?4tyXzR?^Q-qa%%)DU6zR-;gm0VPB1Yt*cN}TpnBJ-)`0YZs0wV z_uN(kqG)<80C%JMEUr_I({h{AadgIUaF@D1^|OxiVb}Ge83j9G=z`<8tyVj9L$BQm z9EbTI0%a^x=KzS6&xomvkutL;j~&T@dJYST#>I_c8rAa{Rxbcl)r-_6VGn^ap|1cO zr_O~v?9gj)uOe&KLT6u4!mP2VZW=N!NKj@CHcXg|#~{EItXg{2B5nBC-ZpEQ!3cDN zM<)6!(jo(Umj#x?`U#N*mS?Ta>GUleILo}M)85`8=(OD9Rgstus3~a&-QHVh!3lsZ zW8EMXT{qUDa`?5@O<~SLWuIA9mBfllOfBFi64@p6o|)PXq`x{MFKzj4k~W`B!lJY3 zWd{^4`0f7>A#)%^T^^!>&P)*%>Qw^!=T-))tu^mF4gVCt@XOb{I&r=Cp*|fQOQDDk z3<>cL!?SKZwZOYst;BQ^nz-6lP4zmKs5bzfS?V?X=$L9XM&F$X+-Ap@B&&d4P&9kR zefFctLsXFZGZk~_5EW$n)~#)me`DJ)Z4kguV-lA*soWN{K2#;(`$VZP5IQ1>O!Ptr z%Slj7(VW?v9;JvDR0`@P+JZ$_p&lRjIqJPkfaOi>y?no7GveFLGaD~<}U012M zPYw6VGcA%k>^fRNav#;OzIMf+T{ikQnWS2h z`*HLWFi%N6@v~k_{JiuUYhkck!!A0}l(5TnMarP#@{xq;&IM9E^6Jv5ugUtuY9%!) zlIvI;F~}mhLFi2a<5+zY6Len!a)}uu3tcFhibjuS@O{i4!62CrVi2YSrX*sJn2py` zJf7fjghE)8FL455Q&Y-gYW(0f1dyQYq|(~{~`FJtrW!}Z|P}Ywi@9YSdy#W zhO>(MFNMyLkX)|?*?1z*Imk<@mg_qRkzGAecNNSuB4+R(cULio(AuUz`e+wOHW8K| zzBq*c-8R1sKY$ams=cp|BjL-T>dLYgl)J5<+v&7bD8-esXg3g!hJm=)*#shkG6Jfy z+~~9eS&p{37;#c2P9UL$@{$buQwN#6QeJiaKrTS_z6RQK=6yQ*MWxzmtm~aua$VL# zXB@Q;ut7k1VZyRZ%e4Q;wtE*wZr;M@JYTJx9Wgn2(z(6(){Xn`EIYT}tu8PA!t#=O z7cXEZ!cI#c8?0Eqac9YSZ}FaUcj=CLkC-_=n7fND-hThaJNFl{(f#VO`Z?O=9svqV z*=DOH>X9*0%LMKNXxk?nd5rL1B*30!cChc#jl1u?^LF))bMHp=&Z7D%Eg>{hCdiTo6twOe%e?OQ^2nr^#CzNT-K5N<;2?2a3* z_2RFV?|L_Nk#ie1Pl1FwieB=K%a?iTEK(1z*=^E&Ug>(iaOy6+x)XT4eoE9!eBsqE zzE-}eOM6RK5S~8d6&&;GX(Ujt?<4cY-Ywi~N=kI{k*hox0odoq)pXM-{auG1??rDG zXIiR_sj=!aJ<1OSX0)0kw~W<$>9AZD#yxYu#84bcM@@G(I0--|W(Nkwqih=)xUf8g zoj)?aOLc%4(ujcx;roZ!KfqIpG6lX!umdX=_?R}a;f2@eU-WoJT$h+{=6(}WMOiWX3` zP3l8QO7xwMV$m2i_k z`wY74heOUDUKLrGm#Oy=0U~ZJZ=0B))=!O#l-v`3BqzZPX7@SSOYQ{{4!GW_uY_Ue zrVn?%*7jvz53?(9=HyphAygIBi;=W}4~%IYXS|-^eP2i8`$5uIG;%b3rw`C~Skg>q zVT8u2w%0&;B{53dH?bSPFiL605%>)(`8HagKnRdVjwbL;%pMjX>>r{=h%?6a&mOX& zvE#6&C<53~ssaY$dgeWa3QxEuyTP)nP!#IRI@VPl2R%VmC!@G1sH8`^+L!n?h*6IT zd`y5Xn4;LJ>x7B@Pu zox=<&lC-ul8YrT@`gzO|!4XUQY>s2@m$B|Q(FV*Fi8&fQk~y`3StQq`h`76)Ck2Zz z@|#tiL7dn>9GyTM1!@ng>91k~7GQLPk&>=0!G-~1EI?u-4f z0#H#2UYGssesKxw(uc+=Tck46=z8QcIJOBMhks^&O<7G=U?uGd4^4%3DpKX5QH)Tj zoksFJQoslm+i9eD;7KaE)9B=ZQ7XLCXo-zuNf(x-FX@JF_Kl-zfpH!4k#3S~_2m93 zI05`L($QSQM~>k;zPlp*V7xz$;n`bg!E*o{TPFx|mCIBG^3OXKipOw$qF?W#7sKb# zkNOP)?D(jFGIW?_pn9U+4LjY?p_pU`1O4z8)m1nY+w9GcFtsqW{%99XipG@A6C8Hb zTV@zTF@0WGTjnEp1v1kD1#tA|sSYb_Qnad0P{9ht<|G0bof zP2y#2F$PY3wHc1^seTKK{)+gcOgjOmo`KV(>ak5+&Wv&yz5M`#bPRDcJ_;&wFoTsN z7_12Ew(-F%{S&~09PRQdji|Mfj=fN&QDO*^M`0b833h7YtEX3r=TDZEs3IJ1}%+#9R7n=*U=7q z=tQrhaVj=yrI86)oH->pE8@yU*P%} zaP}aKgG1m{b<8k6sFMjBF;0SNWvH9wkm*Hulrt7&5pz+_ILUqFjAEiz;(iu=a7xJ5LhFyPM}J_C&0{8#2@kfE$VTY#LP3_%rcQSjw_wI8vTFx urim57Wq=gl=$TkCUyS2BTTGPFB~(F5W-(SgS)9U{jc)?qc=7e=`2PZ)%pBVQ literal 0 HcmV?d00001 diff --git a/lading_py/lading_py/__pycache__/main.cpython-310.pyc b/lading_py/lading_py/__pycache__/main.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c10bae06d4ff137a27421f618c73a23ab82daf4d GIT binary patch literal 3792 zcmZu!&5zs073Yu?MNyxt{a){`Wv^3LZmn&ao(wmLV+V;_XKUC7f(T6sS~HTU<<|~} zUhjeeQYc8>gY(gAP(XXkKapz>{R0Zju|4I|-ire1n;|9bZYzQ_Z{EE3X87j4-y4P1 zs-?j1uYYNe{?$;Fe~7{PpALg};EfwdQLxfgFv6;bx=8kFSCzfi)nu=Cb=eoX1=$;2 z1A5IX`exVkOWhI@`nqTNyj>d4S%V-Bx%!Y`peyANte78 zf3>?RY0F#l*SqVIE_)mPW_MH46>rPG(!C<-s<-X$bay0O^LG8K-K#*?NeQd(D!4&f zeH}01=4X2M8eS$QZeSHSD|i(+*YPG^!|OopksEm9P^F)fy@>+)>Qni)w~`U~#_}BO z27}wtNs9!Gp0uLS4Oqvr9=UzeJL!4kj@4=%JJe62*55z>Q;WQe2z7lBwqxc{M(`!J z9LB?h;qW++156z^u)c@Yl!La)ttVl~?uS9&9q{@y;t`*~!WpBDc=4$_2plg4(tJV& zZVZz^m+$p@iJy253#l0XDh>nhBkD3DhSkRoJxU0hGnx;=!Bgh2_`&-mAT-yie!y7t zuIKa)hoMJ6Z|N}&Vfi6R;-@|8z^4JT^4`nng+m{Qo|lb5sP~)aF`+NyxVcaQ$q;-% zxyFdc4N6!-gC2z2{HFmW0`We)@$aBXAruoOE~UyyQnWl!o&tBIj1jbGtfml{a9~?Z zP0<7m5X`Dr1*{i&5qbUI2eF_tfZ>Jb~PCn7r zc{xm2lrTGT*pM6jkopdzCTx!up=oQh4jtF!wAT08w+dYc&cENUP z(Q(0#YcL_UHi0+(LRFOCBc?DkRnNc&W5l$H@<)Z~6J@Nj!qhlZ#@Zj0zD)Jc%(2dj zBNN_IUwxo_VvY-`Hnq}1s-LMyIocU0V}q5!`xUTNOpEl_teT=!O^bp@?}&C(3&8hS zf31(r)R@*;Vbn;?>C!9Z4B-McUaMyaiF2&bzLam$K3fJGLbo|l$ECSeDc35_wAczr ztxgc^t@T@VY>mrld9)_tgeFIaY+Z~}6>L=IHY&LdGqt`^{;bX{fwZu-A#L4|wyJ4$ zv?**+G=9tnzF z7rb15Wv;&>^wZ@t71w4KKyDA$4|UMmg?P9j#%JguWZVY;g0Ga(8`B@10hQ0)JU=)6 zaW*q*rv`xPl8e5WYo#sU7g+%QeUB?Ut-ALcJrVIk$K?gn8&{xtrA!r} zAcfI^fSH&8hUP#iv+G+xzc@obIi?$+$Spez?2P@UuqMn_(CMxl0I=CI$L@Mz4682G!)@ z`_JBc@*y{XczE!PHyw;~Z`wmAzyR?X=#&7#vrP=hftNMxXbXZ*zqHe z*rLXU*!Eq|b7$vMgK{grQ~%84zn+gC#g}c@aEi_xb~_0l^IjCoxc;4|zHFgWR#od`OQxl#D!vtlTLP z#D@m)#EmSvv;5f>o5_-BkuA4?8H>A*`e`LAKWas7Xf>p(mZl>$e;Ua6&z4bBYiJ8K zH4C^EbUmk5fUZiKagz+}2KyRr{)l}Eutswme`Uxy|0Ut#CigJ4&-k3qd zp!AD-=W8?;MMM>)MwSqTt41r0GhMWez5eOZMfXH1JJl$^2l5nzfA&jcg`^Gzkb zc?^X}JRif$%xNjrKu%}Yw9G20K0%^TNgOI3obJspfOlH+J5$RG9o(7+FeG^y6L7sN z^YYZZeLJhr&rY`&Hz>DD@1;A_rE+!Q?!|fj| zI92Kclbg0gUE6M}R7f4P-{3{t#$nI4sRN8`18)ggEk23qzTox6ura?TbuuT(51`tm zxeUA{((&zu${e9OGceC^aAzUXb0WBclFm6SmvF{+|5rxd3iC^uy!P?@6yX-vnHd7l ziHr42OvEfdx86KPIyfA_9|9KR4Bs{u>&=m_vjDn0?8HhJVXbo+)vhiKU6MQpcYJ*z zHLEJZQZIwRn~T|n4;Kr~D_aKQI$$o7MM1{T{Ln=~gcDvwjHD#In0Fxr6OX(tDp~w< dXmrIiOb8zNs0|3A|M#qE&8qdttXKdN{{@PUKwkg= literal 0 HcmV?d00001 diff --git a/lading_py/lading_py/__pycache__/signal.cpython-310.pyc b/lading_py/lading_py/__pycache__/signal.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f8a028aa7edd24ca6aa6f891d1f60b9d9b21deea GIT binary patch literal 883 zcmZ`%y^hmB5T5matmKFS5?9li;!@EegoF@XnsjbfCcAg>;n-n!!;$DZ@*q&=LD0R! zwpZ~ARLrbn%LO;m%+Aitj=%ZF&18}S%H>0L`1xll_8w;9l?fCOF4J0ZTZsMHbn((EEB-Ha7hDP1eTF zLA6fvLqD#pY5S{gM%Ptn#L?$%5wJH~83CC#J*Gzf+rUUXoFB{ovh z>{R_UF#1Xdtb^h?05&Q$Cp7;9@~o%2Et(P*T6fA0FMA~CCGoaqk;TMiU2G)hHsyTN zh+RegnDei@qG~hh0^2BRXoJs9Q>o{4r(p+%*iiJn5LG8=5=dAQ%>oxiHi;NT2^SbV Vb)6iW$=epeJNj4MVyVf@{|DWv#2Nqq literal 0 HcmV?d00001 diff --git a/lading_py/lading_py/blackhole/__init__.py b/lading_py/lading_py/blackhole/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/lading_py/lading_py/blackhole/__pycache__/__init__.cpython-310.pyc b/lading_py/lading_py/blackhole/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..dd4aece826ffa2e0fbb05a55821a6dfe41ffa25a GIT binary patch literal 158 zcmd1j<>g`kf{VvBvq1D?5P=LBfgA@QE@lA|DGb33nv8xc8Hzx{2;!Hcenx(7s(x`v zYC%S7UU_16YEGqoaZ$2Q(hh?;aMang+RU&a50a_R&1_`3o%$y>mR;TSGo$Yk@+76?H zj#hl{K3+1mpL=I@3QWLjRb} z;UKVi1ylAxFf>36Gb_OZ4F4ptW@JFH(b0s??15db?Zlb61Gii|i8u2HK1Sal<}&XN zG4IwI)R=XJI{xzei|gyx?O_r|Zzfs7&)Q)W@m$Gvm?Uk%-_E&I@@1RFG71Hg?P11Z z4vHw7r(iW)C@#S+;_;81b??DN)bC(TZuk)XKDbaS|7v&nd6tgiF{s_Ec$|ibgr#@M z$FWpm5t&a9hYOomFy(h3ID&Z5z+%VD;)D@S8RhoKVK#H_?195v?oK`Cf%Hd|`K)$F zA;vndGmkYu(_}}WX|M)F-O|lRerbOnzb4UWtUa-HO54Lp5VHvKvL9w`!90g4KLSzU z0tKkBKtfIm5}Uon5WHmdg$biWG!4wfez3pU4U%xk0RyV3O$3$y|Vrg&{JtVm(q z0t{2S0dj7&79%^2vZoHC%>ESvNKSwGZ0~uuWE8+uF>3ko!^w_~`@?$-F5LeO0*K^D z6{@-HxS9$cGSL9F_Tw}Xe8y9yNy1an1f?($8Q#e}OC|3Rab&b7MoWR-ixT)sv`W)p z5~eKSrAitgt#?QG6rbS6|8d-!YFAbi?>_?2kijIO1n?JF0Dx$^1&vu7>j!IPt%xFP zx_ym0j3te+PW2uN8J-sd_$ku8$7( z_WJl?S8~#JCCO=3us4QUigdvY>Y zEW1fHy$NHb-6-KkQ2PDjMc(l5iO1SU8X(Vv?A$%rHh}OHO z*neQJgy0_@8GI_o2HP`aK?i`*MeY{v3X+FjUaeC50mOdnDP*&J~V-j zCZ3pWNjCSgYNW!n3t?~&ri+MAjWA_?APnV%;fQGk(liARJ;SH=sZ(k@b?pWL%OZH) z4do(@;tUpXW-7jVt+j6r-KgxirDOU)U2(JYK|g=6Ry=`PfnMC*b*VC%^4$XuONXn< W_^d+W3sZ>Flo3T9R7LeWA^!sXG0RE- literal 0 HcmV?d00001 diff --git a/lading_py/lading_py/blackhole/http.py b/lading_py/lading_py/blackhole/http.py new file mode 100644 index 000000000..95c088285 --- /dev/null +++ b/lading_py/lading_py/blackhole/http.py @@ -0,0 +1,30 @@ +"""HTTP blackhole: accepts all requests, discards bodies, counts bytes received.""" +import asyncio +from aiohttp import web +from lading_py.config import HttpBlackholeConfig +from lading_py.signal import Signals +from lading_py.telemetry.registry import Registry + + +class HttpBlackhole: + def __init__(self, cfg: HttpBlackholeConfig, registry: Registry, bh_id: str = "blackhole"): + self._cfg = cfg + self._registry = registry + self._labels = {"blackhole": bh_id} + + async def _handler(self, request: web.Request) -> web.Response: + body = await request.read() + self._registry.increment("blackhole.bytes_received", len(body), self._labels) + self._registry.increment("blackhole.requests_received", 1, self._labels) + return web.Response(status=200) + + async def run(self, signals: Signals) -> None: + host, port = self._cfg.binding_addr.rsplit(":", 1) + app = web.Application() + app.router.add_route("*", "/{path_info:.*}", self._handler) + runner = web.AppRunner(app) + await runner.setup() + site = web.TCPSite(runner, host, int(port)) + await site.start() + await signals.shutdown.wait() + await runner.cleanup() diff --git a/lading_py/lading_py/capture/__init__.py b/lading_py/lading_py/capture/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/lading_py/lading_py/capture/__pycache__/__init__.cpython-310.pyc b/lading_py/lading_py/capture/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f903a7ca6eba53474f2c9b718f614ec6ad4579cc GIT binary patch literal 156 zcmd1j<>g`kf{VvBvq1D?5P=LBfgA@QE@lA|DGb33nv8xc8Hzx{2;!H$enx(7s(x`v zYC%S7UU_16YEGqoaZ$2TZl QX-=vg$lzioAi=@_0A`ydhX4Qo literal 0 HcmV?d00001 diff --git a/lading_py/lading_py/capture/__pycache__/accumulator.cpython-310.pyc b/lading_py/lading_py/capture/__pycache__/accumulator.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..7d3dcb6510bb815234f9a9c8b6378e846603932f GIT binary patch literal 2296 zcmah~OOG2x5bmD0$B%V3n~)c11QJ?<g%%G4GBES?}MX#(Eh-e z?Z<Lr zkK%$yB2QuZ%d`;iv5b3A%pC=#&Q=OWW_ zG!rt?X>vUHIGV)O1YDu(DeO{CWSpyL7AxFamepiB3|~S5$l<^;4(@BhuY{Iq@+lRcHl7r^l0`ys#rm<~AIE(dY7azBo>&XIuxhGFZJp6EPm2kg&9}59 zoMY=l*kdSF*sHw*w%NSLF70l5Y$nAsmaNfbI&3U-GGzb+@x#D1juP2e_QAEhf?^lN zRTP-wmFR$G0tRbR%@~>~obu@A7t+(ctqct}rV{({Fu@aV9@G(8zrkoK^AoM<5n0l6 zqHbtw2{|}&>jSy^jfnEG*q~{9oHhZ!?2gkv@2Vlmu#8XUK>AF^CC%0 z>G{)mQ1Hh%@#2No~mQO$-kYY5f?|T^i8vpN@07vwcXgy3{i} zV2vx9+CGG0X`(5fi_<-FMi)@aBRJnVd15cDg}rblWZ~A<(m5v&$afz?e|ZbP_QA?s zdbP)`bL$3K1gn|ayGyiRduJ5vT8prDmw|49ZHNduYS%5@IqK?O9h}iR`@*-`b@i(5VNywH_X5;a~mm&9i9NfwkK!tY$?oL{HAYr@PxNv+h4C zu~(oC8+sUo$Sqhl#(?5kUh*nKy~EhkD$dqlY`9J(GQM##yjnj1Ro+JNE{X`n1!XlF zcBAZpKzdUFh=AOoVbJrN*+Cb$l<_iQOumDrhz4{0)rB-%UqeI8{HpLxx$j(9Xpsqo w+I+a7;#G*QuAl_+0eZ!K8w@p@z9~i%++7{|6U3&%h#e9*0c5sAZ`r%-#$jKh(~R)O zT!9WITyYiL2*eRO)X{UMLKRJfYN;4kTWUwORR^P3MVP1ScHY~nEKW_{XyZhkHq(qF z=Ef){A}2WPA-W|jnPd(6Go*CQZ~2M=(n2ska@cjjpG8&28G|KQLA)A$rjjbpnnaj39qiFTm^eXHypC(P;{hhc+)ld~+CDk~0{Rfc zA@I8wcb8ALHSs4ie76rqF%2&Oso|RnBaffJIz2URD`7mz%e2TZ^;2B8^jHKu)_!v! z;f6{YC2eY1na;G7t|R5FQuBi1u9WBVw7B01Y-0=&A)*))v^gO6jNHF6d_?38JyVj= zQWs;^hyAeM@~cT`c5%(|%DVlB$2GWrQUkA~Pe2g=!*=n}y3RKw(0zg!y09Dk1#XVw Ai~s-t literal 0 HcmV?d00001 diff --git a/lading_py/lading_py/capture/__pycache__/line.cpython-310.pyc b/lading_py/lading_py/capture/__pycache__/line.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..89c8401fff9f37b14619e197a27ee420fdacfd0c GIT binary patch literal 1247 zcma)5J8#rL5Z+y{Uw0RdM|g_{DmS?X(GVddgn|U5K{Rfxjdw0FtnIK~1BXHtTK)kA zlr;Pe{=>FZ)KpM0TsIK$jyaJ;$?&bZauS?o-a&UC+qI8!KfaP4XTxgyLTI5m(d8j2u4*lpc zI9rtut48~{Z2QK6_2I7ScfqH7Rp;7WtLp76@<~xNs)nNQqoSx=-5ateMe(9n=4d6p zw~DZ}jPPI;;WM*S{F4EG-Jb2WHOx8(`+Ly5QqRFGW}Pi(M(L{AE%p~DtWxc}-ongO z4Q%W$Eb$!C%@Tx!NiKQ9huObKz-L;LV9{8S@RUISQu9p)A!>1?1JpDxk93GSLO((s zL!#sNJWnCPJ;7XDX1J1UGM(ll7^6Kxdko23=!sr?C-aFuh4fCne1w1PBilDcrM+}j z4SqZaSMC+aIK1}bdfjdWsE)ZNKxJ+;rAshQ>M|E)P=Kr4X>CUa2 z(3Gu4CN+@G0xvr-b9)UVdanP}mOV`~v1)+txDO@~z2<{kfzMcNPvhd!6DO z*ZtC>AGBu>CMX?oHDZZKu!xu66+Yo-#jqnw2SWY$xa~2fp!3V5fSqq8`8NotGlp~j YO6il$Wz*iF_v$El!cJ4JKt${OH^_V%Pyhe` literal 0 HcmV?d00001 diff --git a/lading_py/lading_py/capture/__pycache__/parquet_writer.cpython-310.pyc b/lading_py/lading_py/capture/__pycache__/parquet_writer.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..88b9039cac19e83e10172eb8efaccc8115c373a5 GIT binary patch literal 2354 zcmb7F&2QsG6rY)l9mjFf(uJ%DU(4rWAyN*A1FEV;%L)!qA?2`ABxJdsN#m}axMPQs zDm`tld*H@_Lvrk&!oM(APF(qr5C_D2?@1o2SFA9X#Ig^HtK7@}ukJOZD#eh8B>tWCm0ndExQn86^< zi_wt^(+EGY`fxDr!JB^$LK<3nOi)>YkeR4l@E!*WENQ<=geB}g7ggcFXcfY*2^Ta+ z*buy@8{dWMWGst^DlVkbJPV6K!~k{qppNf-c=K%#Q#vI9a1Ro)(IuDkz-QVDaHXyW zrv?Z{crKH^@?ga!Q7^&YzYlf?qg3wZh0F$Wcp5&E$z(TI(QXoocz6_KlNBurvtq2| zZdS&(!0luc?F7JQTm-=mhJ;ku)TL*wwVia9leCpUT5`yX1iazF+U6>-7|YCT?k`ZL zH0QXff*eyp1-pQeFUeyHHt@P?&nj~w_!DM4*0Vgqz8(es*Rh;cd&XC=YHJ>=ATxUo z@t~YTVK2VuuFa~J>qY$+r*MS#gq^b~Ij#vZCQ23VtPYk}ICB^cYUr6Yrfg2}2w@2y zL=^VEd@$QcOc&lik>exC_4n2}ozi3S6FXtuGM_$GIPH>Y*7-!!#3Z>CxJtWw2^4TN zO0)gf5DIa3$l2ZXJoS@tKIrTvqbN-BeNdKbHpZk{0A#)c5n%e&?d8Dr*Hs(FUdM6# zHlEI6lF6H4TDWWz<{L}Qn_)p?+l=+*i(yrthxOKrVL8vky0Z}rslmF)kzP|5wR1Qe zP#JW_=5SC}>!YGMW>eK*uC}1Rr1?f#-$78UkT$s)v3PfEM0+2dCh?A`(XvcTUQI#_ zlZFC}O(7PPh@D3RnTCE-vnK+V0-}`oXtfs4V~lkP}r$cdosPO4r;@iq$VkJrVkEpsZc&hrmJ zaEd>VwP~Ar@N1(*cPRg_&H3M~VgKQ7Z_{mhcIz2uYmH7k=g6{YcvybW42Wu=oY6VK zhER6*u4xUb0aJWd?#GIfQ7+-knb@C-*y#fUNqiu5T9<$%ZUK{3QRlj zYuM58P7DI=20=O!;{MU*>aF^GP^3mP8x}Z9XYpH{miZVf%gZBYUQ}3d9 s4~40}yQpFM$||@f>?%a tuple[str, dict]: + name, label_pairs = k + return name, dict(label_pairs) + + +class Accumulator: + def __init__(self, run_id: str, registry: Registry, writers: list, flush_seconds: int = 60): + self._run_id = run_id + self._registry = registry + self._writers = writers + self._flush_seconds = flush_seconds + self._prev_counters: dict[tuple, int] = {} + self._fetch_index = 0 + + async def run(self, signals) -> None: + while not signals.shutdown.is_set(): + await asyncio.sleep(self._flush_seconds) + self._flush() + # Final flush on shutdown + self._flush() + + def _flush(self) -> None: + now_ms = int(time.time() * 1000) + counters, gauges, histograms = self._registry.snapshot() + lines: list[Line] = [] + + for k, total in counters.items(): + delta = total - self._prev_counters.get(k, 0) + self._prev_counters[k] = total + name, labels = _parse_key(k) + lines.append(Line( + run_id=self._run_id, + time=now_ms, + fetch_index=self._fetch_index, + metric_name=name, + metric_kind=MetricKind.Counter, + value=float(delta), + labels=labels, + )) + + for k, val in gauges.items(): + name, labels = _parse_key(k) + lines.append(Line( + run_id=self._run_id, + time=now_ms, + fetch_index=self._fetch_index, + metric_name=name, + metric_kind=MetricKind.Gauge, + value=float(val), + labels=labels, + )) + + for k, samples in histograms.items(): + if not samples: + continue + name, labels = _parse_key(k) + mean = sum(samples) / len(samples) + lines.append(Line( + run_id=self._run_id, + time=now_ms, + fetch_index=self._fetch_index, + metric_name=name, + metric_kind=MetricKind.Histogram, + value=mean, + labels=labels, + )) + + self._fetch_index += 1 + for writer in self._writers: + writer.flush(lines) diff --git a/lading_py/lading_py/capture/jsonl_writer.py b/lading_py/lading_py/capture/jsonl_writer.py new file mode 100644 index 000000000..2690c33a7 --- /dev/null +++ b/lading_py/lading_py/capture/jsonl_writer.py @@ -0,0 +1,20 @@ +import json +import os +from lading_py.capture.line import Line + + +class JsonlWriter: + def __init__(self, path: str): + self._path = path + # Truncate/create on start + open(self._path, "w").close() + + def flush(self, lines: list[Line]) -> None: + if not lines: + return + with open(self._path, "a") as f: + for line in lines: + f.write(json.dumps(line.to_dict()) + "\n") + + def finalize(self) -> None: + pass # file is already flushed diff --git a/lading_py/lading_py/capture/line.py b/lading_py/lading_py/capture/line.py new file mode 100644 index 000000000..a634f72d3 --- /dev/null +++ b/lading_py/lading_py/capture/line.py @@ -0,0 +1,36 @@ +import time +from dataclasses import dataclass, field +from enum import Enum + + +class MetricKind(str, Enum): + Counter = "counter" + Gauge = "gauge" + Histogram = "histogram" + + +@dataclass +class Line: + run_id: str + time: int # milliseconds since epoch + fetch_index: int + metric_name: str + metric_kind: str # MetricKind value + value: float + labels: dict[str, str] = field(default_factory=dict) + value_histogram: bytes = b"" + + def to_dict(self) -> dict: + import base64 + d = { + "run_id": self.run_id, + "time": self.time, + "fetch_index": self.fetch_index, + "metric_name": self.metric_name, + "metric_kind": self.metric_kind, + "value": self.value, + "labels": self.labels, + } + if self.value_histogram: + d["value_histogram"] = base64.b64encode(self.value_histogram).decode() + return d diff --git a/lading_py/lading_py/capture/parquet_writer.py b/lading_py/lading_py/capture/parquet_writer.py new file mode 100644 index 000000000..090c21f09 --- /dev/null +++ b/lading_py/lading_py/capture/parquet_writer.py @@ -0,0 +1,48 @@ +import pyarrow as pa +import pyarrow.parquet as pq +from lading_py.capture.line import Line + +SCHEMA = pa.schema([ + ("run_id", pa.string()), + ("time", pa.int64()), + ("fetch_index", pa.int64()), + ("metric_name", pa.string()), + ("metric_kind", pa.string()), + ("value", pa.float64()), + ("labels", pa.map_(pa.string(), pa.string())), + ("value_histogram", pa.binary()), +]) + + +class ParquetWriter: + def __init__(self, path: str): + self._path = path + self._writer: pq.ParquetWriter | None = None + + def flush(self, lines: list[Line]) -> None: + if not lines: + return + table = pa.table( + { + "run_id": [l.run_id for l in lines], + "time": pa.array([l.time for l in lines], type=pa.int64()), + "fetch_index": pa.array([l.fetch_index for l in lines], type=pa.int64()), + "metric_name": [l.metric_name for l in lines], + "metric_kind": [l.metric_kind for l in lines], + "value": pa.array([l.value for l in lines], type=pa.float64()), + "labels": pa.array( + [list(l.labels.items()) for l in lines], + type=pa.map_(pa.string(), pa.string()), + ), + "value_histogram": pa.array([l.value_histogram for l in lines], type=pa.binary()), + }, + schema=SCHEMA, + ) + if self._writer is None: + self._writer = pq.ParquetWriter(self._path, SCHEMA) + self._writer.write_table(table) + + def finalize(self) -> None: + if self._writer: + self._writer.close() + self._writer = None diff --git a/lading_py/lading_py/config.py b/lading_py/lading_py/config.py new file mode 100644 index 000000000..3d126c141 --- /dev/null +++ b/lading_py/lading_py/config.py @@ -0,0 +1,217 @@ +import re +from typing import Any +from pydantic import BaseModel, model_validator + + +def parse_bytes(s: str | int) -> int: + if isinstance(s, int): + return s + units = { + "b": 1, "kb": 1000, "mb": 1000**2, "gb": 1000**3, + "kib": 1024, "mib": 1024**2, "gib": 1024**3, + } + m = re.match(r"^(\d+(?:\.\d+)?)\s*([a-zA-Z]+)$", str(s).strip()) + if not m: + return int(s) + n, unit = float(m.group(1)), m.group(2).lower() + return int(n * units.get(unit, 1)) + + +class InclusiveRange(BaseModel): + min: float + max: float + + +class ExclusiveRange(BaseModel): + min: float + max: float + + +class ConfRange(BaseModel): + inclusive: InclusiveRange | None = None + exclusive: ExclusiveRange | None = None + + @property + def lo(self) -> float: + if self.inclusive: + return self.inclusive.min + return self.exclusive.min + 1 + + @property + def hi(self) -> float: + if self.inclusive: + return self.inclusive.max + return self.exclusive.max - 1 + + def sample(self, rng) -> float: + return rng.uniform(self.lo, self.hi) + + def sample_int(self, rng) -> int: + return rng.randint(int(self.lo), int(self.hi)) + + +class KindWeights(BaseModel): + metric: int = 90 + event: int = 0 + service_check: int = 0 + + +class MetricWeights(BaseModel): + count: int = 0 + gauge: int = 0 + timer: int = 0 + distribution: int = 5 + set: int = 0 + histogram: int = 0 + + +_DEFAULT_CONTEXTS = ConfRange(inclusive=InclusiveRange(min=50, max=50)) +_DEFAULT_TAGS_PER_MSG = ConfRange(inclusive=InclusiveRange(min=3, max=3)) +_DEFAULT_MULTIVALUE_COUNT = ConfRange(inclusive=InclusiveRange(min=2, max=32)) +_DEFAULT_SAMPLING_RANGE = ConfRange(inclusive=InclusiveRange(min=0.1, max=1.0)) + + +class DogStatsDConfig(BaseModel): + contexts: ConfRange = _DEFAULT_CONTEXTS + tags_per_msg: ConfRange = _DEFAULT_TAGS_PER_MSG + multivalue_count: ConfRange = _DEFAULT_MULTIVALUE_COUNT + multivalue_pack_probability: float = 0.08 + kind_weights: KindWeights = KindWeights() + metric_weights: MetricWeights = MetricWeights() + metric_names: list[str] = ["metric{{0-9}}"] + tag_names: list[str] = ["tag1", "tag2", "tag3"] + tag_values: list[str] = ["value{{0-9}}"] + sampling_range: ConfRange = _DEFAULT_SAMPLING_RANGE + sampling_probability: float = 0.5 + unique_tag_ratio: float = 0.11 + length_prefix_framed: bool = False + container_ids: list[str] = [] + external_data: list[str] = [] + cardinality: list[str] = [] + + @model_validator(mode="after") + def reject_length_prefix_framed(self): + if self.length_prefix_framed: + raise ValueError( + "length_prefix_framed=true is unsupported: dogstatsd-py does not " + "expose length-prefix framing. Set length_prefix_framed: false." + ) + return self + + +class UnixDatagramConfig(BaseModel): + seed: list[int] + path: str + bytes_per_second: Any = "1 MiB" + maximum_prebuild_cache_size_bytes: Any = "500 MiB" + maximum_block_size: Any = "8192 B" + parallel_connections: int = 1 + variant: dict[str, Any] = {} + + @property + def bytes_per_second_int(self) -> int: + return parse_bytes(self.bytes_per_second) + + @property + def dogstatsd(self) -> DogStatsDConfig: + raw = self.variant.get("dogstatsd", {}) + return DogStatsDConfig(**raw) + + +class GeneratorConfig(BaseModel): + id: str | None = None + unix_datagram: UnixDatagramConfig | None = None + + +class HttpBlackholeConfig(BaseModel): + binding_addr: str + + +class BlackholeConfig(BaseModel): + http: HttpBlackholeConfig | None = None + + +class PrometheusTargetConfig(BaseModel): + uri: str + tags: dict[str, str] = {} + metrics: list[str] | None = None + + +class ExpvarTargetConfig(BaseModel): + uri: str + vars: list[str] = [] + tags: dict[str, str] = {} + + +class TargetMetricsEntry(BaseModel): + prometheus: PrometheusTargetConfig | None = None + expvar: ExpvarTargetConfig | None = None + + +class TelemetryConfig(BaseModel): + # Short form: telemetry: {path: "nong"} + path: str | None = None + # Long form: telemetry: {log: {path: ..., format: ...}} + log: dict[str, Any] | None = None + prometheus: dict[str, Any] | None = None + prometheus_socket: dict[str, Any] | None = None + global_labels: dict[str, str] = {} + + @property + def output_path(self) -> str | None: + if self.path: + return self.path + if self.log: + return self.log.get("path") + return None + + @property + def format(self) -> str: + if self.log: + fmt = self.log.get("format", {}) + if isinstance(fmt, dict): + if "parquet" in fmt: + return "parquet" + if "multi" in fmt: + return "multi" + return "jsonl" + + @property + def flush_seconds(self) -> int: + if self.log: + fmt = self.log.get("format", {}) + if isinstance(fmt, dict): + for k in ("jsonl", "parquet", "multi"): + if k in fmt and isinstance(fmt[k], dict): + return fmt[k].get("flush_seconds", 60) + return 60 + + @property + def prometheus_addr(self) -> str | None: + if self.prometheus: + return self.prometheus.get("addr") + return None + + @property + def prometheus_socket_path(self) -> str | None: + if self.prometheus_socket: + return self.prometheus_socket.get("path") + return None + + +class ObserverConfig(BaseModel): + enable_smaps: bool = False + enable_smaps_rollup: bool = True + + +class RootConfig(BaseModel): + generator: list[GeneratorConfig] = [] + blackhole: list[BlackholeConfig] = [] + target_metrics: list[TargetMetricsEntry] = [] + telemetry: TelemetryConfig | None = None + observer: ObserverConfig | None = None + sample_period_milliseconds: int = 1000 + warmup_duration_secs: int = 0 + experiment_duration_secs: int = 60 + + model_config = {"extra": "allow"} diff --git a/lading_py/lading_py/generator/__init__.py b/lading_py/lading_py/generator/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/lading_py/lading_py/generator/__pycache__/__init__.cpython-310.pyc b/lading_py/lading_py/generator/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..02717c0a923b9194eb014b6885b11b7abe3314a5 GIT binary patch literal 158 zcmd1j<>g`kf{VvBvq1D?5P=LBfgA@QE@lA|DGb33nv8xc8Hzx{2;!Hcenx(7s(x`v zYC%S7UU_16YEGqoaZ$2?T|Q literal 0 HcmV?d00001 diff --git a/lading_py/lading_py/generator/__pycache__/dogstatsd.cpython-310.pyc b/lading_py/lading_py/generator/__pycache__/dogstatsd.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..9edf0d7f8f6ffed9ddff74ac298406b782d551df GIT binary patch literal 4517 zcmai1TW=f372er9m!w3~vSP;$!lp@K8V#Kb+NP*$q_*57ZKGbC)+w80vECg~EA54v z9Y&Tw2M!b}&a5a=u9K3OvuDnnnLT^% zQ?}cIf!}+7?{DmP4dWj)ng6+%yo^!(4InXwM#2&^V?(B&=FrqnYiQwVWp?fi9Y#EM z=H}ke)3}rQd1KhnxSIuebJ)yV!&crNw)2JILcTa$%sa!5UiY$7d3V^=xSyTQmxfEg z8{A6*-cFi{x#lLVr2TdAFcPDbm!q#RYmo;*9= z>0O*{PI_^c@ityNAh6O{dJMvSvlN% zy)4$!QQuY0wXBRcRiGc&BZ#Mh*SQpFd>tpN#*KIJ85-@}T&M|+t-tQz7p?P{z;^(8rrY9(Za^!7x;g_R4l zrNu~D6_@>1&D_kj*+aQA=He7>w~Hv};tU-}XR1b8#DeE=2xY+?#WJltM&K-ga{zry zvz^csq$Xb71Wx)+*Gl zEfho<7gC#0`D2lmB9%LRvtIOaUR4pi=#&c5DlL$=ikK@qgJ-DbTq4qH6CTteNJ+}! zlmwI`Q<;HZ@;kD2vTqX0(*zpC*RJXe5h9Dko8mmtT*{7#kGDO#|Q^t793 z2dP1i7Miu0wb3qtP<6A*HNSFpHc1x9S_P470=eh^AO-+o%&&z7O4!w8(6H!s4i3( zum!feH1}0Qow8}kte zNxZ3jfSBB3;sprV_r(wJ5Z@*6JpwNQC<_TtyhzjU(A23i&c}U6Tp`j`%~6!w%Hvry zu6UyC?I@KG>(z_mtw}2QPrVg7UX^t`GMccKKt zIFx9^Pz7O_m&pW)Jw&z(weCt;mw^U{oyt*Ky%3U9&~u zc>aG?c^s8qRrghoh)v2rq}M$<^%nr|v-{?TvCj^S*HBSjrP{=#xn_RIJ_OV$ul0LF zS{wF<#v0nC#QDIN&V9Dwl9pGDJ@$v=oz#lBtBR#DbtAJM;q=ZwVN_I*-)9nUypPuG zK07dv-izIJBwEUgs+}TEOJuP!tP>f$Uf1=kEXRE&ZsD9y&03Ma0bZ-F2MmdBTSVgm zXhp9z@1144&pbk3Vp0sQASa>&y*ki)PKZ5I6}3p!62J(lJ}YGXD=DMm;Rqr$J_e8>he2qFl75ehA5Xp@0v#Z4KXd~qZ^aNU1?UGuPJ=$eK zJDWgs6)^90*phk4+FgFMz$el_#YhuFhwWd1pwTfl5UyxjbZ@?I9T?R+pxOHlih+fY zwf5LuDg@rXUnk|=bFwjOr$7((ac`RvN9zDB)gcApJ)_(*il=Quws&qZ<1U(`-Lr>0 z4pIQ|w?1KsD;K8D9vY;SeZuIRTWH1-@W1C2PW{|w+psQ4D4LWsbmH*n#?-7(J-H0W z(6DOP@xG0|N%BIq=mN%NMII_e2j8p~1>c(B-V?4xX*LmPzqt^l=y!yL<=Qf~K5MaB zpU9-#Mrxn~r1&N6w+m{Y=w>Z$5?CQX9gFxmfeQrQ0KnDjPRz%;UAJ{lPPM!5&P9gq zw4Wkg3jzV$iI3e@^spd?02K{lThOIRw^1lgUAByb;)G|?$ax}oGFXDsS5&BRvhUy&ME(Qn@TzIv- z>ijWz9POa%N{)96DonZ@95=wayVWL}Ke5imf<9SYI6hJG3|Ayvcn3!nM(c;JBDI~V sQ0otUlzx None: + sr = m.sample_rate if m.sample_rate is not None else 1 + if m.metric_type == "gauge": + client.gauge(m.name, m.value, tags=m.tags, sample_rate=sr) + elif m.metric_type == "count": + client.increment(m.name, value=int(m.value), tags=m.tags, sample_rate=sr) + elif m.metric_type == "histogram": + client.histogram(m.name, m.value, tags=m.tags, sample_rate=sr) + elif m.metric_type == "distribution": + client.distribution(m.name, m.value, tags=m.tags, sample_rate=sr) + elif m.metric_type == "timing": + client.timing(m.name, m.value, tags=m.tags, sample_rate=sr) + elif m.metric_type == "set": + client.set(m.name, int(m.value), tags=m.tags, sample_rate=sr) + + +def _send_block(client: DogStatsd, block: Block) -> None: + if isinstance(block, list): + with client.open_buffer() as buf: + for m in block: + _send_metric(buf, m) + elif isinstance(block, MetricCall): + _send_metric(client, block) + elif isinstance(block, EventCall): + client.event( + block.title, block.text, + tags=block.tags, + alert_type=block.alert_type, + priority=block.priority, + ) + elif isinstance(block, ServiceCheckCall): + client.service_check( + block.name, block.status, + tags=block.tags, + message=block.message, + ) + + +# --------------------------------------------------------------------------- +# Token bucket (synchronous, for use in worker threads) +# --------------------------------------------------------------------------- + +class TokenBucket: + def __init__(self, rate: int): + self._rate = rate + self._tokens = float(rate) + self._last = time.monotonic() + self._lock = threading.Lock() + + def acquire(self, n: int) -> None: + while True: + with self._lock: + now = time.monotonic() + elapsed = now - self._last + self._tokens = min(self._rate, self._tokens + elapsed * self._rate) + self._last = now + if self._tokens >= n: + self._tokens -= n + return + wait = (n - self._tokens) / self._rate + time.sleep(wait) + + +# --------------------------------------------------------------------------- +# Generator +# --------------------------------------------------------------------------- + +class DogStatsDGenerator: + def __init__( + self, + cfg: UnixDatagramConfig, + registry: Registry, + ): + self._cfg = cfg + self._registry = registry + dsd_cfg = cfg.dogstatsd + # Pre-build block cache; cap count at 20k regardless of prebuild size config + self._cache = BlockCache(dsd_cfg, cfg.seed, max_count=20_000) + self._rate_limiter = TokenBucket(cfg.bytes_per_second_int) + self._gen_id = {"generator": "dogstatsd"} + + async def run(self, signals: Signals) -> None: + await signals.experiment_started.wait() + + async def _wrap(i: int): + await asyncio.to_thread(self._send_loop, signals) + + await asyncio.gather(*[_wrap(i) for i in range(self._cfg.parallel_connections)]) + + def _send_loop(self, signals: Signals) -> None: + client = DogStatsd(socket_path=self._cfg.path) + while not signals.shutdown_is_set(): + block = self._cache.next() + est = _estimate_block_bytes(block) + self._rate_limiter.acquire(est) + try: + _send_block(client, block) + self._registry.increment("bytes_written", est, self._gen_id) + self._registry.increment("packets_sent", 1, self._gen_id) + except Exception as exc: + self._registry.increment( + "request_failure", 1, + {**self._gen_id, "error": type(exc).__name__}, + ) diff --git a/lading_py/lading_py/main.py b/lading_py/lading_py/main.py new file mode 100644 index 000000000..86b832aa7 --- /dev/null +++ b/lading_py/lading_py/main.py @@ -0,0 +1,131 @@ +""" +lading-py entry point. + +Lifecycle: + warmup → experiment_started → experiment → shutdown → drain +""" +import argparse +import asyncio +import signal +import sys +import uuid + +import yaml + +from lading_py.config import RootConfig, TelemetryConfig +from lading_py.signal import Signals +from lading_py.telemetry.registry import Registry +from lading_py.capture.accumulator import Accumulator +from lading_py.capture.jsonl_writer import JsonlWriter +from lading_py.capture.parquet_writer import ParquetWriter +from lading_py.generator.dogstatsd import DogStatsDGenerator +from lading_py.blackhole.http import HttpBlackhole +from lading_py.target_metrics.prometheus import PrometheusScraper +from lading_py.target_metrics.expvar import ExpvarPoller +from lading_py.observer.proc import ProcObserver +from lading_py.telemetry.prometheus_exporter import PrometheusExporter + + +def _build_writers(tel: TelemetryConfig | None) -> list: + if tel is None or tel.output_path is None: + return [] + path = tel.output_path + fmt = tel.format + if fmt == "parquet": + return [ParquetWriter(path)] + elif fmt == "multi": + return [JsonlWriter(path + ".jsonl"), ParquetWriter(path + ".parquet")] + else: + return [JsonlWriter(path)] + + +async def inner_main(config: RootConfig) -> None: + run_id = str(uuid.uuid4()) + signals = Signals() + registry = Registry() + + loop = asyncio.get_running_loop() + + def _on_signal(): + signals.set_shutdown() + + for sig in (signal.SIGTERM, signal.SIGINT): + loop.add_signal_handler(sig, _on_signal) + + tasks: list[asyncio.Task] = [] + + # Telemetry output + writers = _build_writers(config.telemetry) + if writers: + acc = Accumulator( + run_id=run_id, + registry=registry, + writers=writers, + flush_seconds=(config.telemetry.flush_seconds if config.telemetry else 60), + ) + tasks.append(asyncio.create_task(acc.run(signals), name="accumulator")) + + if config.telemetry and config.telemetry.prometheus_addr: + exp = PrometheusExporter(registry, config.telemetry.prometheus_addr) + tasks.append(asyncio.create_task(exp.run(signals), name="prometheus_exporter")) + + # Generators + for i, gen_cfg in enumerate(config.generator): + if gen_cfg.unix_datagram is None: + continue + gen = DogStatsDGenerator(gen_cfg.unix_datagram, registry) + tasks.append(asyncio.create_task(gen.run(signals), name=f"generator_{i}")) + + # Blackholes + for i, bh_cfg in enumerate(config.blackhole): + if bh_cfg.http is None: + continue + bh = HttpBlackhole(bh_cfg.http, registry, bh_id=str(i)) + tasks.append(asyncio.create_task(bh.run(signals), name=f"blackhole_{i}")) + + # Target metrics + period_secs = config.sample_period_milliseconds / 1000.0 + for tm in config.target_metrics: + if tm.prometheus: + scraper = PrometheusScraper(tm.prometheus, registry, period_secs) + tasks.append(asyncio.create_task(scraper.run(signals), name="prom_scraper")) + if tm.expvar: + poller = ExpvarPoller(tm.expvar, registry, period_secs) + tasks.append(asyncio.create_task(poller.run(signals), name="expvar_poller")) + + # Observer (target PID must be provided for proc observer to be useful) + target_pid: int | None = None + if config.observer and target_pid is not None: + obs = ProcObserver(config.observer, registry, period_secs) + tasks.append(asyncio.create_task(obs.run(signals, target_pid), name="observer")) + + # Lifecycle + if config.warmup_duration_secs > 0: + await asyncio.sleep(config.warmup_duration_secs) + + signals.experiment_started.set() + + await asyncio.sleep(config.experiment_duration_secs) + + signals.set_shutdown() + + await asyncio.gather(*tasks, return_exceptions=True) + + for writer in writers: + writer.finalize() + + +def main() -> None: + parser = argparse.ArgumentParser(description="lading-py: DogStatsD load generator") + parser.add_argument("--config", required=True, help="Path to lading YAML config") + args = parser.parse_args() + + with open(args.config) as f: + raw = yaml.safe_load(f) + + config = RootConfig.model_validate(raw) + asyncio.run(inner_main(config)) + + +if __name__ == "__main__": + main() diff --git a/lading_py/lading_py/observer/__init__.py b/lading_py/lading_py/observer/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/lading_py/lading_py/observer/__pycache__/__init__.cpython-310.pyc b/lading_py/lading_py/observer/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..3440c24705b5d12d6e15026e4f6e64fc4dee6c40 GIT binary patch literal 157 zcmd1j<>g`kf{VvBvq1D?5P=LBfgA@QE@lA|DGb33nv8xc8Hzx{2;!H6enx(7s(x`v zYC%S7UU_16YEGqoaZ$2YsYpHH%)0&Ae2(NRI;HGe}t+6EfpY8S~Zm_tiWpP89SS-cegX+ zrZ#J-D80kIs$?TB-1!C^_yU~f$|+v}iAogS?8d=K)3N65%zHC$XWs9<-#V#Q%Lv-f zzk6H1Itcx#C$r6l$;Z&ek02;&A&M!9a0};8(jxiOXc_P{BC}_;EUednDEYBS|YyZK!j(os|Nf>iiCiK2eGDuP}bG2YmNfgGqtX?47o63T?gsC#Z zSSqW_lYXk48|&9OPq=4u3wA0q4dkW{kLX8I;f^w+FlGw(enI>cEO>gkxta7>Q%II> zvUn%hX3?N2c)J+|G>p4`I+z(H`#|M~ZlnWs!cPM(m_G|}1>A~rAS&3z1edXkYcM*4 z$wTr_)LM1L_*>4C2c*a z#j+L}>O#B(BUPD&Z3Q$1USt-p#_@Qafz@s3huQ%peE@2PfQpc&A!bm*OlmNzV^fn_ zKp8f*nFEC4QkRxMTc&fMEzuHCM@3a`0%uIwKpE{$SGf??y#7_aAPzreJWQxBSX;EA zLJpc9>$=J7UEex`#PK!E$j6xhl28P%!E2tOOkX=M`3K&eeZQWmOu?cKe+4}7SG8Ex z;z$Ht-w)$Z`uSlTO0L(hwGEKdrt?8-e}gXm0P#J} z@D>{53Hkyk=5rttiboJnNiwpFLp(CUZj8+d5}O&?vbJn~Njlm#b|(nPY;l8(tdR}2 z5>UL6S&z{ZBIh!D7iZ=K=<42TzOTY>X9hKw!T&blw`KK(K7bCN^KljS+V7*0A#3A# z@V79*)ALTx$Cq{3^UynMMVf1B8dhgXqp_!pU})#MMqUx#qN=k88top7r7vW_C8NsR z2|}q1pg!e_&Az0`4iIA~AR#1w4V%*$_d0EZifj_ZNym0Uk9b?pX)A_3k)7Exu~52B-4 zWAJ|5gl$4QCl42Z0xLN|E&)9*5iMO($MwI43y0DOws~a*ITr^#=KHGb`@Mwr;f_Ak zvVX51M1_Ux0^qt5_}f~%p@q(~f-ZD<&YUUFX-x?HWe^_pJd<6tj908eS67bj)1>YD z{1Vs|NrWPtsfB~jL!+I4wevU6s*8lrU#`ylzgn`0^#G|sV_JIoah20fdT9v%3v$BK w dict[str, int]: + path = f"/proc/{pid}/smaps_rollup" + result = {} + try: + with open(path) as f: + for line in f: + m = _KB_RE.match(line.strip()) + if m: + result[m.group(1)] = int(m.group(2)) * 1024 # bytes + except OSError: + pass + return result + + +def _parse_smaps(pid: int) -> dict[str, int]: + """Aggregate all mappings from /proc/{pid}/smaps.""" + path = f"/proc/{pid}/smaps" + totals: dict[str, int] = {} + try: + with open(path) as f: + for line in f: + m = _KB_RE.match(line.strip()) + if m: + totals[m.group(1)] = totals.get(m.group(1), 0) + int(m.group(2)) * 1024 + except OSError: + pass + return totals + + +class ProcObserver: + def __init__(self, cfg: ObserverConfig, registry: Registry, sample_period_secs: float): + self._cfg = cfg + self._registry = registry + self._period = sample_period_secs + + async def run(self, signals: Signals, pid: int) -> None: + await signals.experiment_started.wait() + tick = 0 + labels = {"pid": str(pid)} + while not signals.shutdown.is_set(): + if self._cfg.enable_smaps_rollup: + for field, val in _parse_smaps_rollup(pid).items(): + self._registry.set_gauge(f"smaps_rollup.{field}", float(val), labels) + + if self._cfg.enable_smaps and tick % 10 == 0: + for field, val in _parse_smaps(pid).items(): + self._registry.set_gauge(f"smaps.{field}", float(val), labels) + + tick += 1 + await asyncio.sleep(self._period) diff --git a/lading_py/lading_py/payload/__init__.py b/lading_py/lading_py/payload/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/lading_py/lading_py/payload/__pycache__/__init__.cpython-310.pyc b/lading_py/lading_py/payload/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c00f65c23c79fbb7a3c01fd7ae8b6f7d07aca1c0 GIT binary patch literal 156 zcmd1j<>g`kf{VvBvq1D?5P=LBfgA@QE@lA|DGb33nv8xc8Hzx{2;!H$enx(7s(x`v zYC%S7UU_16YEGqoaZ$2TZl QX-=vg$lzioAi=@_0A++FeENCB=UgD*}Nax$yH)IaaCeFMLJF;ddW*F4|&OJs#2B8V^ULj$j=j>^OCw$S^2sb z07;25Slc_>GdtVU-P8T`Fy6DrH}HGwAIppTjvB^))5G?khli`UlK(XgLl_l9n8Iq9 z6_f8)#lk(;$TjVX-E=CBN&R-iZRRU^ZaWRHS*R3DLz?Tpa5v1#h{(&L@Z=s@JY>{t zQ4s#ec4btIWSnh1MX?7xW77V0LyU^Cj}0-lYOR`;apsJRz2NL+&R*tBh<)JfW6p%k z%W=6^PGHt?`pzMvvX4hiiv1Y1pGQrK1JYXDFAmBBvM3MMtV0IAU#z)eN*uzsr^I1d z=#L?ZCoz*Nj)`p#!x#+dP%C#J;(^nYn6)n4fEqIeCw z<3rvAc$dWM;GNiYo_(0-%i<06pA>J(eVF+m-gryA4bCZO`zzud=;)nQt8!Xcb4K~A zD;xfMaeXcg)8xHUJ6vwWp(xd5OR6x9;?}h9-&V2cRAo}S-iWJ9rFec(R@0<(wi-4X zB_WflirQ(cl5(jUw~|g%iqiaYDP53*X<{|)U&Gi$swiwkD?D*2O7L<`DuwZBToQ3T zAtA*1_HrpqF}MYO)RfcyCb{iw+0q3OreU=aCW&@xk!*;vshxW*h>6D7a2Yq^RxPTp z?7RQr{SVLH7w5|7ncnZ+?`b=2wj0`2GVQ2V6`nu*70_~U-N2Rn5F|C$&91Rw_ROxC z_+6uJ&KarIHP>^(+%Q+oe%qq9W%aCHu4{-K`t1$l6Zl)t?izSXae!k~*-StA(+b~Vz!IcM>x-P6`ciw`CuTOti`ZlvJo_RIvdu`CTbmoKUO;XoMna)r0o(wpQOkFATk9y1lISVwwZEk|_}1vQ5`KW{%^}HYdy} zvuIjRZHq_g!a(+9hwXECacKJz9$+ch9(E}7$l^w7b`hW07IqhY@Wdp4bmij|OXsP} zIdu##>RjAOwVCqTGV&6!NBX*JL`ixIjD*6kZF(%FI)R5Ot{p!T>ME|}I*2q7n2m~c z&yYE33-hue9o#M4={bkHD|3eowC&4=a4LD}2zJ(N1EIbq|0rIEfUs~Sv``3FtZF02V~jPUJ%e5L7@F0Xd)t@D}Wyb4?1BZ>roRB zRF4Hg*lNYt%h;C*#yB;^lvLZabXCTXZMq>3P^2Jr667MTqzK|0p5;L~MRU^Ro);_U zH!yq`<&aWHIZj_GP>`@IP?4}%DGop9htH%Wo0U|Uc_>>c>&Qu?v_T4ioz+=MDOoSJ_MR@77?62V3w)7(W0!yc! z7cc!C^zO3sfnN;#AzS*^QhDiEs;kH6qlfd%Qn7S886;T|%`1!td&O*Hu&O-#bp zjjr*5f8Xk1%C04hC0ijz{lI!a`P@_#=h*_~oVKPfYI8}OkG7eF!;ycPc4IYewy%7P z=B1S!HC9dr+0vv4stYkZWBM|3e%MH^pfPx1hjjEJNw<$?nFL|!+@86H+;_;xq}uv= z*;1D=Nc|ef6H|Sa+FL&IE*>kmSe!{iQH;bJJc{1nwIS|05yE()^O=lE*;cd_c$~JE zt22XOVPbAB*&OtxXJ~cU4^z}+ARb;OPT%62Y!?LEH zx=&Y5&v&8*94On6K|78cySWhzdx|EZ$XDLODNq;td5H2!qTZ+OTSO?nXsNc>UYtZ90u`C`YQr?6zOh`ny%&ka?-XwVw-u{q*wDE~{0Mu50|RX-MLxYr4z13B>~MGTB?s3HT*-ZqF6zN0OBKn* z*Wscfw*i-*Bfbs(RG;sl0vz!3;GY6IHh`2pQP{AAzlzw@=ZuIVID3dw9-YlrNm6YJ zkZgGjRmUNlqCHywvp=QZ-@jYOgJG=w?NwDJHa`s;SO4(pCIuoabQ(8+O<-ikvw*X} zu=!3aszD!cI}R4K@~ya(*?2PO`%d2t+2Lu=C!-*Q)v}8MhFS#cIn|K_th$qEKmmyT zU31Y~g1b_=u$Ws%eAs|WY59Vpy)~<97+ZEwltALRsV7+ohQ2DE59gx>AcmrSrbmZe z86tNxc?WA^7w2#~jH!AeXok!4a za)2X<`b=OWaY&<_`e@Tf5aNS;KlLGV{)m&_mKg!UFJl;|7i~V=>d5TtUlvg{%X&Y? zFitZEDC?o#4f-}mw%Z=AhA)(zEL>51{$hjKf6+@bF&bZn?mUFIFMOz3JZT8F8BY`V~p96^YdWP#h&pg z8sr(2zBYhKxD7yqwj~^@pcgISB3k4rh4pen9#UbS2d^O?5(R9Nb7~Rh%`{7ox?bMza;S zAE+elJbd(cc_jb}A!xC3^jC~2^<&L{ybCXQ4Wrd1B4>%bPK2O6@G%Y3_F^2h)OBJn z5&1eXi@~+`XYSl(o6qTy;P#!H-?(%0?#-Dw#`xtt8yOjw3XY&5TiPQuORa4m9DeIr zyN|?$HpmR<#_)nPzLjK`nMa90}LU@ zE03s$G;2a+h6uU8KGMXIPsEutXot{!dn>8w@ok-qk<@R|iwx~p+)wC{>;lhVNw);2 zj{)r?3}zUD*YNu^p7Gk~PQm{U9bBLIIfOZD6#L>ZE;?lJEcHh;m%}UEp<@CfvC~Nh=}{(66$1D1kec~mwOnq=n|x-6Fy-dL_XuOd5nF7$9fn` zg+qZy;0&-o0=P+C8_Xt{FX$XFzjE$Q+^H{=9z{x?PofoBk_kW}l6wgtr$kx4)Bwy( z0VAe&EQ#EP{TX+5o9{q895+%80|CEjpvpi|IcKPsF#_em{$Mb5<^sX|aw-!*4$ibx z@;w`k3YSUukK5|XKhsA@f!0UJvuAgmX`4$8UVEXyit^P!upGAiK#b$hi=qA)yniA7 zpMY%A;oK%kyGgOTJfi(5iCQ@6XjLU2T>yc1nta%#evA=WKsigTi$u1jbenE>&v3B;A-xG1%b64IjnDy|(rN~KqEC2xWZ4m-q%uy8nqV=M_EB|z~H z1c2<1or;Tpv*c?QL2Df9d%}l96t&N%^EW~SI3+O=Z*grWQVLyYhL2H?p$4gvF$O;e z4T>P_4tLO?6K}v&m|vK5V!fK>bUP*3ilC437DX(=lJz}v7}SvK5!BqQ%mtP+g1kW9 z4m^gp?kr~n?E=IiwLNFqK?GCZ$M;#Ch*Y#4IZ`R!&pV^jzoV1UtBc5ua`VjA!l#2E zYDH-fZ1bt%u|;-xP{?XYzoz15=E>Oi1J*cbc1F& z4T4}n7mip6axJ2wILHOoB8+wbJpuk{0lz?XK`9Q2_AhW#zaT>1jeYZ%)ZWF3aWV|# z@t^2rEf>uL@Z6QLZDLaYEX+i1hQj(4n&=#nSBa2UWoLm@wX44Z(GI_@{+bwDTV~6y zKSjsC;^HrQ#!+*kpomvK1N{dguHqQkQZlpA$nd{Fuo{IkMVdibz1&8$Q{>@#eE83U zX`CA1zdTD zM#fFc%dSb!%8#2R=csEPB list[str]: + """Expand 'name{{0-2}}' → ['name0', 'name1', 'name2'].""" + m = _TEMPLATE_RE.search(tmpl) + if not m: + return [tmpl] + lo, hi = int(m.group(1)), int(m.group(2)) + prefix = tmpl[: m.start()] + suffix = tmpl[m.end() :] + results = [] + for i in range(lo, hi + 1): + for s in expand_template(prefix + str(i) + suffix): + results.append(s) + return results + + +def expand_list(templates: list[str]) -> list[str]: + out = [] + for t in templates: + out.extend(expand_template(t)) + return out + + +# --------------------------------------------------------------------------- +# Call descriptors +# --------------------------------------------------------------------------- + +@dataclass +class MetricCall: + name: str + value: float + metric_type: str # "gauge"|"count"|"histogram"|"distribution"|"timing"|"set" + tags: list[str] + sample_rate: float | None = None + + +@dataclass +class EventCall: + title: str + text: str + tags: list[str] + alert_type: str | None = None + priority: str | None = None + + +@dataclass +class ServiceCheckCall: + name: str + status: int # 0=OK 1=WARNING 2=CRITICAL 3=UNKNOWN + tags: list[str] + message: str | None = None + + +# A single metric/event/service_check OR a batch of metrics (multi-value) +Block = Union[MetricCall, EventCall, ServiceCheckCall, list[MetricCall]] + + +# --------------------------------------------------------------------------- +# Context pool +# --------------------------------------------------------------------------- + +@dataclass +class Context: + name: str + base_tags: list[str] + + +def _weighted_choice(rng: random.Random, weights: dict[str, int]) -> str: + keys = [k for k, w in weights.items() if w > 0] + ws = [weights[k] for k in keys] + return rng.choices(keys, weights=ws, k=1)[0] + + +def build_context_pool(cfg: DogStatsDConfig, rng: random.Random) -> list[Context]: + names = expand_list(cfg.metric_names) + tag_names = expand_list(cfg.tag_names) + tag_values = expand_list(cfg.tag_values) + + n = int(cfg.contexts.hi) + contexts = [] + for _ in range(n): + name = rng.choice(names) + n_tags = cfg.tags_per_msg.sample_int(rng) + tags = [ + f"{rng.choice(tag_names)}:{rng.choice(tag_values)}" + for _ in range(n_tags) + ] + contexts.append(Context(name=name, base_tags=tags)) + return contexts + + +# --------------------------------------------------------------------------- +# Block generation +# --------------------------------------------------------------------------- + +_METRIC_TYPE_MAP = { + "count": "count", + "gauge": "gauge", + "timer": "timing", + "distribution": "distribution", + "set": "set", + "histogram": "histogram", +} + +_ALERT_TYPES = ["error", "warning", "info", "success"] +_PRIORITIES = ["normal", "low"] +_SC_STATUSES = [0, 1, 2, 3] + + +def _sample_metric_value(rng: random.Random, metric_type: str) -> float: + if metric_type == "count": + return float(rng.randint(1, 100)) + if metric_type == "set": + return float(rng.randint(0, 10000)) + if metric_type == "timing": + return round(rng.uniform(0.1, 5000.0), 3) + return round(rng.uniform(0.0, 1000.0), 4) + + +def _maybe_sample_rate(rng: random.Random, cfg: DogStatsDConfig) -> float | None: + if rng.random() < cfg.sampling_probability: + return round(cfg.sampling_range.sample(rng), 4) + return None + + +def _gen_metric_call( + rng: random.Random, cfg: DogStatsDConfig, contexts: list[Context] +) -> MetricCall: + ctx = rng.choice(contexts) + kind_weights = {k: v for k, v in cfg.metric_weights.model_dump().items()} + raw_type = _weighted_choice(rng, kind_weights) + metric_type = _METRIC_TYPE_MAP[raw_type] + value = _sample_metric_value(rng, metric_type) + sample_rate = _maybe_sample_rate(rng, cfg) + return MetricCall( + name=ctx.name, + value=value, + metric_type=metric_type, + tags=list(ctx.base_tags), + sample_rate=sample_rate, + ) + + +def _gen_event_call(rng: random.Random) -> EventCall: + title_len = rng.randint(8, 32) + text_len = rng.randint(16, 128) + title = "".join(rng.choices("abcdefghijklmnopqrstuvwxyz_", k=title_len)) + text = "".join(rng.choices("abcdefghijklmnopqrstuvwxyz_ ", k=text_len)) + alert_type = rng.choice(_ALERT_TYPES) if rng.random() < 0.5 else None + priority = rng.choice(_PRIORITIES) if rng.random() < 0.5 else None + return EventCall(title=title, text=text, tags=[], alert_type=alert_type, priority=priority) + + +def _gen_service_check_call(rng: random.Random) -> ServiceCheckCall: + name_len = rng.randint(8, 32) + name = "".join(rng.choices("abcdefghijklmnopqrstuvwxyz_.", k=name_len)) + status = rng.choice(_SC_STATUSES) + return ServiceCheckCall(name=name, status=status, tags=[]) + + +def generate_block( + rng: random.Random, cfg: DogStatsDConfig, contexts: list[Context] +) -> Block: + kind_weights = cfg.kind_weights.model_dump() + kind = _weighted_choice(rng, kind_weights) + + if kind == "metric": + if rng.random() < cfg.multivalue_pack_probability: + count = cfg.multivalue_count.sample_int(rng) + return [_gen_metric_call(rng, cfg, contexts) for _ in range(count)] + return _gen_metric_call(rng, cfg, contexts) + elif kind == "event": + return _gen_event_call(rng) + else: + return _gen_service_check_call(rng) + + +# --------------------------------------------------------------------------- +# Block cache +# --------------------------------------------------------------------------- + +def _estimate_block_bytes(block: Block) -> int: + """Rough wire-size estimate for rate limiting.""" + if isinstance(block, list): + return sum(_estimate_block_bytes(m) for m in block) + if isinstance(block, MetricCall): + return len(block.name) + sum(len(t) for t in block.tags) + 30 + if isinstance(block, EventCall): + return len(block.title) + len(block.text) + 20 + if isinstance(block, ServiceCheckCall): + return len(block.name) + 20 + return 50 + + +class BlockCache: + def __init__(self, cfg: DogStatsDConfig, seed: list[int], max_count: int = 10_000): + seed_int = int.from_bytes(bytes(seed[:32]), "little") + rng = random.Random(seed_int) + contexts = build_context_pool(cfg, rng) + self._blocks: list[Block] = [ + generate_block(rng, cfg, contexts) for _ in range(max_count) + ] + self._idx = 0 + + def next(self) -> Block: + block = self._blocks[self._idx] + self._idx = (self._idx + 1) % len(self._blocks) + return block diff --git a/lading_py/lading_py/signal.py b/lading_py/lading_py/signal.py new file mode 100644 index 000000000..7adcaadaf --- /dev/null +++ b/lading_py/lading_py/signal.py @@ -0,0 +1,17 @@ +import asyncio +import threading + + +class Signals: + def __init__(self): + self.experiment_started = asyncio.Event() + self.shutdown = asyncio.Event() + # Threading version for sync code running in threads + self._shutdown_thread = threading.Event() + + def set_shutdown(self): + self.shutdown.set() + self._shutdown_thread.set() + + def shutdown_is_set(self) -> bool: + return self._shutdown_thread.is_set() diff --git a/lading_py/lading_py/target_metrics/__init__.py b/lading_py/lading_py/target_metrics/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/lading_py/lading_py/target_metrics/__pycache__/__init__.cpython-310.pyc b/lading_py/lading_py/target_metrics/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..44f25a9d2a176395d8cad90f99328f3467ce0ae4 GIT binary patch literal 163 zcmd1j<>g`kf{VvBvq1D?5P=LBfgA@QE@lA|DGb33nv8xc8Hzx{2;!HUenx(7s(x`v zYC%S7UU_16YEGqoaZ$2PO2Tq{9+~`!NLFloxCV_ literal 0 HcmV?d00001 diff --git a/lading_py/lading_py/target_metrics/__pycache__/expvar.cpython-310.pyc b/lading_py/lading_py/target_metrics/__pycache__/expvar.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..3183efee29e4a4df77b24e2bd8179b3ade74bb4d GIT binary patch literal 2269 zcmZuy&2Jk;6rY)$U9Ufqrf!-xC5S}=1z+qUm0PJQrKJT`DQYSe+(xVI&e&OZy}Owi zCv~)z3aPmD0O9~tNVYg3apGU-f&XESa6nu+R)X?oZMO+9t9d&gZ{EDG-jFw);-oDaLkO>bq?h z#ttvD60dY9b6NQ@ZC9@$R$hXu!ULd>{NhOBOKR{))*}%GfwUBl@ z5M8b|(xel0VYRdsb(0{LFm!Lt&S=U46=<3TY9}P-{H)pVH@?__EY5y(!C58pbw#{Tt^rwGPWk@!fgKCPk<)eqhou5Xmw~ zBE7P~-V>-hPsmIDWk8#CL6m8#g09r%s}Dn-DWgUyki#U5QYb7j=b1PM2ODMani2lG z5mJ&fg(+01tSBh{#iRkF?mrZwcndB4wjDL4z6gFHGnh;`rP+* z#rOLu8$k8&b=jhd=igw}$v gO~Y+8`9C$advR*ky3FK_2>HA4Q4ozp)TI-kHN{{n=OHer-$F>TYb)!JIw>TMlbJvRDg+oULC z#8%&K+a+(tPTy_2lyGfaVb+AUt0iXxS7Q#ZGnb#?bzbjkmq^!O6;^#^w9m3yIkUhQ zagI0mnXYk(loEAz1|-h?ny|BM;T2&Ehg$nQ(;gFVak%+76hX#iGiZJx(mv0(`9L;v z{vu!PrlKF@O`fnUjgq_>B&;cTmnYu!+^{~Zw!bmaTb*S5h^Ycl8AHc$40T^=8-NfEBa0gIBJpB_P6)PganFGsWY>g>72tHHF@2(qjNB-UL&&vVQ(P+y1{qOk z(3+RXZ8BQ}874m3b8>TR75apZ%)*p6Va^=e403K{fIODuq+nUD%VN?HFy<;q^ylq{?Ot3D``#&E+KM^XtuB@W}P% zH$gmr&eV@yYiNDgeDd^*jm;wr27{jUe%F3*+W%RygRewYrD z9NIk1gIGC!D0q-_2GJP|dfYpsDl&_sJdP4Bl?mw+WiSMBm&^Sq--aQaR(i}6SpD$9 z<_5-$3D^>D3-t0d3e0Yt2Dz&I7H!@TA{9yp+)xex&{UBXWEph4s%W6Gw%~FQ=ddRp zRw@H6P$qXlT zvxuJ|`8g53j;pKPE(VB}YTTUy6K>5uOjh+8cJ(lEu z!YBU-K=FG()E$8233&kc`Z*+Hu8s8x`FPg<^LGFubjKK*6C(dQd|7BaRzY^`9Vgdz zT!i%302i;dWBEo2G?lr&k?|$yHQ;}!&v!_wAI-GOd;5@_QwV^as|Y>-D5$?Cp5*l@ zcSsX(^~fqr0E9aoV`PtYKpV878UL+S5UmHJVovV0ZI-Z1mdSa&@CkbIAn<~ndO*w0m_fh($Mz>KADszxY$5u z;rMOS3d@&dx$EiSDM*wXtM^Z&a~^~L@NGKBtkP7BzJ3Fr&7-C=>^@zH=KMQh(R2 None: + await signals.experiment_started.wait() + async with aiohttp.ClientSession() as session: + while not signals.shutdown.is_set(): + try: + async with session.get(self._cfg.uri, timeout=aiohttp.ClientTimeout(total=5)) as resp: + data = await resp.json(content_type=None) + for var_path in self._cfg.vars: + value = _resolve_path(data, var_path) + if value is None: + continue + # Flatten non-numeric nested dicts by path extension + if isinstance(value, dict): + for k, v in value.items(): + if isinstance(v, (int, float)): + self._registry.set_gauge( + f"{var_path}/{k}".lstrip("/"), + float(v), + self._cfg.tags, + ) + elif isinstance(value, (int, float)): + self._registry.set_gauge( + var_path.lstrip("/"), + float(value), + self._cfg.tags, + ) + except Exception: + pass + await asyncio.sleep(self._period) diff --git a/lading_py/lading_py/target_metrics/prometheus.py b/lading_py/lading_py/target_metrics/prometheus.py new file mode 100644 index 000000000..c2a3449ac --- /dev/null +++ b/lading_py/lading_py/target_metrics/prometheus.py @@ -0,0 +1,82 @@ +"""Scrapes a Prometheus text-format endpoint and records metrics in the registry.""" +import asyncio +import re +import aiohttp +from lading_py.config import PrometheusTargetConfig +from lading_py.signal import Signals +from lading_py.telemetry.registry import Registry + +_LINE_RE = re.compile( + r'^([a-zA-Z_:][a-zA-Z0-9_:]*)(\{[^}]*\})?\s+([-+]?(?:\d+(?:\.\d*)?|\.\d+)(?:[eE][-+]?\d+)?|[+-]?Inf|NaN)' +) +_LABEL_RE = re.compile(r'(\w+)="((?:[^"\\]|\\.)*)"') + + +def _parse_labels(labels_str: str) -> dict[str, str]: + return {m.group(1): m.group(2) for m in _LABEL_RE.finditer(labels_str)} + + +def _parse_text(text: str) -> list[tuple[str, str, float, dict]]: + """Returns list of (name, kind, value, labels).""" + results = [] + kinds: dict[str, str] = {} + for line in text.splitlines(): + line = line.strip() + if not line: + continue + if line.startswith("# TYPE"): + parts = line.split(None, 4) + if len(parts) >= 4: + kinds[parts[2]] = parts[3] + continue + if line.startswith("#"): + continue + m = _LINE_RE.match(line) + if not m: + continue + name = m.group(1) + labels = _parse_labels(m.group(2) or "") + try: + value = float(m.group(3)) + except ValueError: + continue + # Prometheus histogram/summary data lines have suffixes; look up base name + kind = kinds.get(name, "") + if not kind: + for suffix in ("_bucket", "_sum", "_count", "_total", "_created"): + if name.endswith(suffix): + kind = kinds.get(name[: -len(suffix)], "") + if kind: + break + results.append((name, kind or "gauge", value, labels)) + return results + + +class PrometheusScraper: + def __init__(self, cfg: PrometheusTargetConfig, registry: Registry, sample_period_secs: float): + self._cfg = cfg + self._registry = registry + self._period = sample_period_secs + + async def run(self, signals: Signals) -> None: + await signals.experiment_started.wait() + async with aiohttp.ClientSession() as session: + while not signals.shutdown.is_set(): + try: + async with session.get(self._cfg.uri, timeout=aiohttp.ClientTimeout(total=5)) as resp: + text = await resp.text() + metrics = _parse_text(text) + allowed = set(self._cfg.metrics) if self._cfg.metrics else None + for name, kind, value, labels in metrics: + if allowed and name not in allowed: + continue + merged = {**labels, **self._cfg.tags} + if kind == "counter": + self._registry.increment(name, int(value), merged) + elif kind == "histogram" or kind == "summary": + self._registry.record_histogram(name, value, merged) + else: + self._registry.set_gauge(name, value, merged) + except Exception: + pass + await asyncio.sleep(self._period) diff --git a/lading_py/lading_py/telemetry/__init__.py b/lading_py/lading_py/telemetry/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/lading_py/lading_py/telemetry/__pycache__/__init__.cpython-310.pyc b/lading_py/lading_py/telemetry/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a1e31d7a0162543943b49710ea80477f89fdbb7e GIT binary patch literal 158 zcmd1j<>g`kf{VvBvq1D?5P=LBfgA@QE@lA|DGb33nv8xc8Hzx{2;!Hcenx(7s(x`v zYC%S7UU_16YEGqoaZ$2n3mj literal 0 HcmV?d00001 diff --git a/lading_py/lading_py/telemetry/__pycache__/prometheus_exporter.cpython-310.pyc b/lading_py/lading_py/telemetry/__pycache__/prometheus_exporter.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..3944b71e6ba392cf05b099b2c14c34f3a1015486 GIT binary patch literal 2615 zcmb7GPj4GV6rb7u{+GCE6-rx#p&YP;94;V1qo`FH6;NeD6A9I7S#5VF_PXoc&CEEd zEnA{WuSG)Q1K2mX@Fh6#9p=i3GZ)kgytf-CX&Mf!wQt|d?7V&Rd%yRd(^{=e;QICV zX8$LbkUwy6_A+5`8*cd>2u3=DQD(%nL*Z%0#=z{D6z8ni8rU6s;B=gU+i?e8#~YM7 zrF_kf%Y#a%lIu=f9n?BYl-wiCW!?+Iyd$Ht%#0nH33B$SOkSZX}r-AF1iYD9@j8`;U*ei%nQQROg=V;-th$VQN`hUDUa%f`K} z-NqW&6%p7uh=N8Cr9GvxR{2jj9%(nJHV^rermZh+E(|Qbr0sjbu*y}Q-k^>-g`Z~6D{>~8Jsf<<>H>Lx)fLG>2KLj2s>%YwmexaBqo zP5@uhF&O10Gq|;9F_T#@td7l=bJVPk!%E!kd&~w|g7q@1zs88x(YH6K=TyVq(m43e1g1#?+iz6LUg* zs*H(sL_sqrC@p0l5qLTgN;j7tBjAYHenmh3lY*gUNjpMjaip~E_fmk*giwNx!Pa%a z^fliEdZm|qjP$9)B|j{JA+_swF+37tXGtFKi3)5GRorFe<}Eb*3lY~~>ecc+8PBqqQ?UJ7Hxu(w^@}Nu+$g2DarAh&pv>8E!9s zj<3IoervG@VBl&fl6Hu&EH2E-65KiPKZ7zMeKMnS^2nUhDL@Z_Fy`bgdA#)qLt#zr zsWY)=Cg$hdfG03#ZcJTfG5ZCX*mubhoqEhUGb&A7uxcGpaZA~tyAuz_4s%~xI6raM z$+%W%ZrgRg0+7=o?D#_YlU1P#++~UR%?arv59s_S%UsfJmhs zxaAcPT{zQY2YiR~Oz4lsKGc9bI=N_?+Gu^O=~MA8yn8w?uES85*JDWkFdbwY7|*{9 z5Ld|fql<8sP&HcXaT-DnZh*4b_&>Ze-t6V$HI+ zj9WaJ1exrmK=?@U0nk_QEDl00R&g8RBoHI*#K99DOKnQ7bZP5Z$TJnC3D|inw{yaO zr=Wk~k(C4kt}FRoKMNuuwS5r8Lk_tS>~UQ#R-W>aTZtww|PHq^n^H6*y788fCoch)=+Ra8Mv-0>6Z%R%U4eg&*1)6rLO} z@hL7j=}=`ub^f$<|F|>wJ!ly*&!Z?k*kOF{!I78f_~u2&Jhw}`vZ(j3ofr!#-ua0Y zD9@?N^BK5r^f6!Y+}7Lx1jx*u7;_3aZqA%J`4aN#3q*#@Tr9VJ%+k^vi?1~+pRP}6 zzcQ`d*TdDvmwTZaJ@cQ=fAkPNe?T|{6V-!+3Q zgKrfkEs2JZV1>vN+=RssVM?NH#J`?LU>4dO;*$OVbeVz=n3ShD8~(!1s08 z_XjB(#;8|)e}5Rn$16+s^DeXK%8+GBh#TlhPN|$o?t_BB;xn8_JOpw=e1+oF>s-ro z9!od&9f^uZR%p4jT3fXR4xtmfzR%JSTyX=;t|$ei_!m;d<5G71&rII?ikg5FDwvL} z*7cKi)hbAryQXiPPM`Fy)}jOnY%8LT0zJCu2-;gO0NcQpls7bscvh9(fQ+xw74u&p C;-=LA literal 0 HcmV?d00001 diff --git a/lading_py/lading_py/telemetry/__pycache__/registry.cpython-310.pyc b/lading_py/lading_py/telemetry/__pycache__/registry.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..2cc7791d85b3eb2c8ecb94c634ac81321148d871 GIT binary patch literal 2200 zcmah~-D(?06rS0gU9DEKYb$PAC2pg(ghr6c?Ny-|<1~gs3BHI6C8A~7?2PQSSG&s0 zC_zGof_z!XMPII-Or@woI z{Dqz6;zMU2szSp_n{djVjJ7Gf-OTB_ZI=?x1mm9YxG(&U&r3YGVC~WmgqL}EL3lWJ z+5vZtNV78h?xzzeV*ZtiJEGAQTBb=uisMviIcPPW7X4fcsUA0u!Knc zkB~{XPC5?%fNwxHtDHgp?wH{DG+7A+hh_!ah{A^W5i~Tpq*oB=$kDE5V+t)V#l8>y zlI;ha91)2DXqHV`TUtQq40~22XU2;%cuCyUxM{9UIZCGFFr{e14#P5Fhp9vhk^noH z5?RM)X<{n4MG!@4p6V#Vp2DDpG^9RVr^C$|qSkEAJA`v+lWx4%an=_$_X!eUR*+D* zjYa$n8k&sg$n62jv2)Hwgd#*%& z-MC_y>p9_GLI)B+-hM_=$$(^IdQNT2h1OIc;5)nYuE+tX+_`Ie{uNLR0J}2^<^d4! z%VoGgV`t=cT)6IR_~g0OQ`LAhyM3#qx!GzQ$T-c_LarMs?t+x6)(sFyT|{;SRbl#3 z6MilK;+%uiBMSKfAb&WoK#(mbnpC1-gEBj zd^iuT%Hy6oDRgTWG$qc|UTd-9uvxM<%rM-<-ZN>*iTRafJkd)s5z8jCx6cH#Kp20j zLg7MwZo3xDj&8t4Y8_1Apzv(bD&77k^p?bDHWU6fAH%%Ei88LCa^W8?icA z6p(;!``aEqb_8U)6h*v9qDUeXQ%Q;}6NyfXTv?@-kKyy=czY0TW0$Tv%>=s>ID06! O3eRP5WR>1^YwSOljo85e literal 0 HcmV?d00001 diff --git a/lading_py/lading_py/telemetry/prometheus_exporter.py b/lading_py/lading_py/telemetry/prometheus_exporter.py new file mode 100644 index 000000000..77f21d6c1 --- /dev/null +++ b/lading_py/lading_py/telemetry/prometheus_exporter.py @@ -0,0 +1,48 @@ +""" +Passive Prometheus exporter. Syncs from Registry into prometheus_client +collectors and serves GET /metrics via aiohttp. +""" +import asyncio +from aiohttp import web +from prometheus_client import CollectorRegistry, Gauge, Counter, generate_latest, CONTENT_TYPE_LATEST +from lading_py.signal import Signals +from lading_py.telemetry.registry import Registry as LadingRegistry + + +class PrometheusExporter: + def __init__(self, lading_registry: LadingRegistry, addr: str): + host, port = addr.rsplit(":", 1) + self._host = host + self._port = int(port) + self._lading_registry = lading_registry + self._prom_registry = CollectorRegistry() + self._counters: dict[str, Counter] = {} + self._gauges: dict[str, Gauge] = {} + + def _sync(self) -> None: + counters, gauges, _ = self._lading_registry.snapshot() + for (name, label_pairs), value in gauges.items(): + safe = name.replace(".", "_").replace("/", "_") + label_keys = [k for k, _ in label_pairs] + label_vals = [v for _, v in label_pairs] + if safe not in self._gauges: + self._gauges[safe] = Gauge(safe, safe, label_keys, registry=self._prom_registry) + try: + self._gauges[safe].labels(*label_vals).set(value) + except Exception: + pass + + async def _metrics_handler(self, request: web.Request) -> web.Response: + self._sync() + output = generate_latest(self._prom_registry) + return web.Response(body=output, content_type=CONTENT_TYPE_LATEST) + + async def run(self, signals: Signals) -> None: + app = web.Application() + app.router.add_get("/metrics", self._metrics_handler) + runner = web.AppRunner(app) + await runner.setup() + site = web.TCPSite(runner, self._host, self._port) + await site.start() + await signals.shutdown.wait() + await runner.cleanup() diff --git a/lading_py/lading_py/telemetry/registry.py b/lading_py/lading_py/telemetry/registry.py new file mode 100644 index 000000000..86256f6e6 --- /dev/null +++ b/lading_py/lading_py/telemetry/registry.py @@ -0,0 +1,39 @@ +"""Thread-safe metric registry. Counters, gauges, histograms.""" +import threading +from collections import defaultdict + + +def _key(name: str, labels: dict) -> tuple: + return (name, tuple(sorted(labels.items()))) + + +class Registry: + def __init__(self): + self._lock = threading.Lock() + self._counters: dict[tuple, int] = defaultdict(int) + self._gauges: dict[tuple, float] = {} + self._histograms: dict[tuple, list[float]] = defaultdict(list) + + def increment(self, name: str, value: int = 1, labels: dict | None = None): + k = _key(name, labels or {}) + with self._lock: + self._counters[k] += value + + def set_gauge(self, name: str, value: float, labels: dict | None = None): + k = _key(name, labels or {}) + with self._lock: + self._gauges[k] = value + + def record_histogram(self, name: str, value: float, labels: dict | None = None): + k = _key(name, labels or {}) + with self._lock: + self._histograms[k].append(value) + + def snapshot(self) -> tuple[dict, dict, dict]: + """Returns (counters, gauges, histograms). Drains histogram samples.""" + with self._lock: + counters = dict(self._counters) + gauges = dict(self._gauges) + histograms = {k: list(v) for k, v in self._histograms.items()} + self._histograms.clear() + return counters, gauges, histograms diff --git a/lading_py/pyproject.toml b/lading_py/pyproject.toml new file mode 100644 index 000000000..34f9c6750 --- /dev/null +++ b/lading_py/pyproject.toml @@ -0,0 +1,23 @@ +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +name = "lading-py" +version = "0.1.0" +requires-python = ">=3.10" +dependencies = [ + "datadog>=0.49", + "pydantic>=2", + "PyYAML>=6", + "aiohttp>=3.9", + "prometheus-client>=0.20", + "pyarrow>=15", + "structlog>=24", +] + +[project.scripts] +lading-py = "lading_py.main:main" + +[project.optional-dependencies] +dev = ["pytest>=8", "pytest-asyncio>=0.23"] diff --git a/lading_py/tests/__pycache__/smoke_test.cpython-310-pytest-9.0.2.pyc b/lading_py/tests/__pycache__/smoke_test.cpython-310-pytest-9.0.2.pyc new file mode 100644 index 0000000000000000000000000000000000000000..4ab99c31d170d5a81160ac44e535f9eee0364bc5 GIT binary patch literal 4124 zcmb6cO_SV4RjrR1jpk!#zr1U&6U#}c%23I!Ka&7qu}MqT%SAC=1phkOvnyg7Jwn$5?y1UdZ(K59knC)d& z-(8^<>O9cyllCgD(%J*9eS+$@v{rre(7Dx*_n0eKD!%NdNf5b%#P!{GqTsGeec|`G z-*?lvv&V#+GQQ9F3oai-Zs^k>>b;l@-EPd?i!j%TBbvV8_$iPH_s$TubPpJ3E@vGU z>@({65p~55bK`+X2Ey$IA#)FWpyGiLEZTIAKv3F^(NkNZpKKK;cq#JdA%B|%8?u7E zm-u2w+Oxf-lZrU;*glIy$G~l-4=TZi?=|?+511uu!);1=9r5Y%}xbf+)eu>W~Ukfxqwicuwr_xlHE3vylDpFY*L}3u^ z$8V32 z*A+J#iW@_inL$tA*FRj$O<|2onVy*wooMfZ)=hZt8(>XFM!MjH=e4wZ0M=Dl%42-% zsT;CXc#5nKz#YArJ07`k`H^+ry&io12qh#dH14Is7b%sMAnJsJ6#VT7qf<+lMn8z4 z@$Vj?f{#$Q9A$ab#s;C5AxnKGc+ioS@*r79O$8n1b#_>1??@R1p42Kys}m0*A+4T2 z=rLKNV4^&@GY~-xr=Y!AXjoiUc3>vP@G)9a-dmh-@7}o=FXg#3ShO$Aoj4VvYQkfR z_zQX7sz@6GiC}lZ@M{o$QZHfL>!&^0><>Z_?EB#W*it-f%wZBAV*-!w_;-RZ5JOqp z1BdVqSkT*nts2GYkH)p>IP&`}l@&P1D?kT9bx7K>q?s~Cc^Pgln<}CyZi-kZR}!E5 zVaP&YCyH1HMUcw!zRv?c5^5D}QR%UWabLtB62U@-2ZKuppZA#X3N})?a^Um+Ao1t` zNMRpvl};+x*byvRSXZ;ZhPOk zdizaj#^CzWRP-ht^b=sshA=ygqhM$SWOTYcP(im>3`tjce$kvnIn)Z@F3vv-nDh!X zHC;CxaANi9HG;3M+}k82aBA}(`my;>^SHEH(3C!{ZPdWsonu>X{@d31E!gA zqrm+|u)s`%a7%K%r{x9&ZZgrpxwKtfD0j=uu7TO`p#cHe8e14*MQL2728M172%+7t zgjLAiSZh>vwD}%3gWx^#H5Z` zAD^6P`&xPr=1+-LNKrolXak{$K=`UYqWAzQGPoQ67%30}imHOE6QcUsgwWcdc|yw@ zFg{J|Fm9}AbRla51h5O@DZoFSElfZLKP4EN@+y4wye7`o1~ zmKD4Nlq-i8@VGqXae0o%Ch)kJFM;lWuMZb;o37Fm546mF9aF)|6Z)&1*MRFY^BNS- zWGkQ72~hnQ&i=es)@B?7#hb{pcxFOABK$gXiZF1!M%U-4zK>MdilX`k9r~ z+By0EX=i*^nCgU|i!a%_vsnE|=EGN>r8OP!^ z6-oGW;BPI$msR%}*=~{1))$^nh8J6@b8iZ`Uv&{W*Giwq{(LKKIk~&7kT11Th(UwC zsvugXG=L79^VnR(=B3d_6i*AoS>5LPP*$qya@2ITr?n&Q{R(ae-DqvTT8v`hb~y6| zqa*qR{mU~p0hqFRX@&w|in=gE0r2=Tn^@of9Y%wTZ^{jC`#B2ok@FsVyQcOrQxQYJKyZ zB;kL@Tjy1b>7BTrY`>3N;{NN}=xNNUMRC0OY8ZF?Fx>`VMyiS|X+hn{($UJi2!qsR z{X`5~&p|%g0a;=lFRg1>Im^;*g;B*9{}!Bt-1%Hc6O%d&LdFu=oR;RvaI;7Q{9Pn% zJVy6@s5wk*`CBl>zm4;i)bBDcjC~5}IHr&{A(W)r;zk< z`4+A;cVhtGL(sXB;Gc!?d@dI~xOV7qmT;7h&!HZHY46S;fJy;!aj6iDT=&wQco0%W z>h(gX5<->lWyM3La0l28{$%<-?5zG^u;gf5RUqW!Yn(#@U!IXbUJm_8W$MC0uteaP zu9V<;1)$cdN^1POh(!xho&hBV{HTDZ@clw+df4YTVHU3LAIMT6P+7%$6pUP$A6%Xv zTq*_y+b_=#zBEsEetvLqe(=)#npb!anM9MFOZU!WJG%xm={aa@V(KvBKuFCk7G4nTY?oDjqZ%b2F zF3a8Et{8AuC<{5A6M+(jDtqi#!DI&^+s4qT{$5P&8Pf(CJbq1M-7ptGCeQu{aOkF> literal 0 HcmV?d00001 diff --git a/lading_py/tests/__pycache__/test_capture.cpython-310-pytest-9.0.2.pyc b/lading_py/tests/__pycache__/test_capture.cpython-310-pytest-9.0.2.pyc new file mode 100644 index 0000000000000000000000000000000000000000..579979890fadfa30b9cc10e6750f93f25dd3fbce GIT binary patch literal 23048 zcmd6P3zQt!S!Pvrb$9h^X7rY9*%?2A-F}QTk}TPYC6D98c5E|taAL=UL(`h78MV~Y zJ-SsRSxrw?iA)07yjZf!%N%Ba6ByWK+1+!1WnmA?KG z0>Cxq4A*q6ra5QHF|%Y_hG{Hi=W?dLlb_4W9cRvwV_~i!$KqU3j-|Ph9Lr0UxiNVr z(;Q!#n44JIHn&aA?dJBS9dkRRoNex0+BLUJ%DLw5r9E?dmiEr=mGgXa-_lKUH!a;f zce80cVYrT4c-3$Vm#n${I4im(oRx5P3(m@J1!onU-HNj@cN}NqIC~4uCfsc}+lI4A zoNae^;A{uZZgZ{Yjq1+MJKyX1fj_y>R+IDf|Ds z%@^fKoK~V|qYGA8dbHn9SgbAAmG9Ngco)!|^EKXL!w=Mj(B_qSc9Z<;E#Dzr^^7?a zONgcOG7FY#xtUk(Ioq|}?5oCH*3G$j9CLaiv5ZB$ekpYL`ZDO8^6>Vpc{9-g+b`hv zG=BaOfI!f|&A4`8p0-?RwPCu>C-SRS*ILVT&2`f>0^2P#jIQYo z`7noeY7IBEgT|5!q^bThsdXyh**t!av7!6$MqOjgST|P9uDNF7h_!Le zea33WwN~wJrfZ+EReHq_ndohtcam%6P6UOu;<|CpKuxYqDZtu%+Ehi9ZI)Xm>TqQp ztmwAaGFZg|*-Ala8wy!!@O!8KfYmhw!ii6=VS)!z130D!a7^uGKjQ@Z2qp-&0aVNCb{xWd(5{i|g~gd=GToq7SK&6WrDn6H zyk#}tUV{Ab!a~ijFL=BXjy)L)^LlO%CQlA0 zPY&r5J&*^IC&!Z2jwJ6KO`bg1=iur#_P;|WBbuGsLc?piK4;9|4&a!Mc{=+4jM+2; zv$Ma)l7~E);?3mCJ>L9roDA}2*Idnjo~?7lciW#4EVsHgc(Pni(YYOD*K(wDzlwVK zuC;EeDU`r>g~lQ6gEiM+pD(VzB3z+i)5^wf}+u0RQ08*c{SH+ z2a}B!kac&Aton+eJFrtdgcqq=f|Cl0rIA8MAhNoT6^ep&>Hx~>Ai*>Nl@;{>!BK)^ z1cwQzVN~;i5b8l*U@z)z1P=ow7x0Lh;pLtj)~k4TYB5^v{!3APs-1^~u6aw#!G-(r zu%83q(ss?x^zh}pMhmGmCdpIL7Ww_IcaHDc%Dz4%Cs_7Q*lN#xC131)JI~3iumd@ut~E& zmLmQ)4aOT?6c#LK_fUKdg{a;L)_L;i5wA7hcD>^2-RutRv07Z6a=qvpPQu*Dy6+u5 z0(2u^6&x0vRZ}=6hkYot!R)669`1}M*!?uRmCFxQE9yx;O->jvs?Pd2F3pVW< z*i4``HwRlO@8(?xN0^>&5yygCa?3at-HJPgW62$NCvYsg+kpGq!*NbU!q^MVmZD}p zZvn)AER#G)7*{AS8a`klrl8cdnl6nEhx)J>kp&nGurgbew>zEu;}7#{DM zd+@Mpb1nCig$6PXDpV^W?*bFf<#9wGi1^e7FhOLDk^|5!I&>k~6HL z^*6L(EZ~WaW6`v~n0vi*U|^CGCXGM!uvQmNpb)pXVTM^%Z!LNomZpz6v_3e3Qr|A} zPvHI)vUZqH#oS+aZoWo`sg`Vat4H!q5Qpap(P-hJ?dbPkBEI|x7QYPEyH5vhh z*KDx+B7G^{qOki?P{!R#x0GP_WbXxxFz3Pqaw`yAWp`|~(k*u@5Kj#HO(}a1YTM(%zO~bF zBGz(Hy%n|9R}oYR`0!<$CGvS@BPWc6-@tDaCtz^>ILR zJjAJLdMCb%d%W$OyXhUF5rcGfDU}yNDxuASL*HG7)lqj*&YC~3gK z1Aaw5)s4^L+)44R$Vu3>g>Pjy@vVacd@D=t=H@{CZ~=h}KxOqmB6LZ91zyAJ!nX=N zuJi)#YrgfhD3Frc^l8RJ^;D)G2 zTjS~*_+Xb{mEa!{^yo!hDrWfJH-4KLbk(*7oTv{h%V+a}dNygl+-h^02^(&~^`@vpsY@ ziu=U?q@>cAVc;}#ZUIW2t zhs8EnjOay;=$aK1ZD4SniYy>)Pr($5Y$rIh^d%IcONZi1C`6YaB&NtfVLmN-f#2xF zHWh53gw&Wuz0hj=f;3e)=+QSN$02n$N(sY?Hr2tjHHFT^nmUvc1A@gLLiC=tWgvd~fP1g426z%Qf=8& z5*JxOD?QmsHbHMq(X=jFU%LJE&fP=jb4W9(DdnqNM{=g6rq=wUYlD&#`aZ*J-Hb?1 zJ2|PM@u#K~8ov$62`fRnd?Ni>OHN8mIBVz%63N-u=Z8y9Fozyv(UKEPCz6~fffW^m zv491|48|>r`0oj{`)}wSQ;157pU5h-|A#0Hqy0eQyfixlNHiUfg&)tlln3=^)_`z}TGh+N zCTfshbHESaF@zDj*+e~(RP|tn6<~tp8Tn(YDMR&4P+Tj)Q%FMj#7LbOTxcIJ?e2S7(8CxO~{u4!=gNhL!uf9KO;7U8Q4n+GvQ%&BenvK zi9wmMQ&^tFqH7AFXP0IoHMyQ%v>y&OjYfghxpOEQ4Lm*zZ{!Q?i~5y%KLA`it$?8w zSfMfjq7{Cax4)U-TM7Ot!9OGT9)dn^mJ-by^%4MIMF!^>yG>`zDLDlv=M>70+?~hW z)E{l-8T|aG0NyNb8yM0=xQ%m>%Artm=;PX+HubfrGR&_=L+W|_B)dUu`rU(mwb-NP zW_H4@LxvBJTCV3&%L8#%TGNd`%7CXyp2L(jf7NHuZkX{}=a8v@F>4>2-S@Fcz)aOp zOVtVbQ!lgb;u;y{CV!d*e_9b9HSwoOk(?dD#~Ld(tvnj3*<^#q`3)~mT-d~Jfy)4V;s4e>Io z7nNb|#Angcb_#6pKPQgLB!u!urGM~1^>OJFB@xOTX*^;^U5XNmhZ7D$S-aszk4Z;a zD#)jys|t}?CLt*b=MEh1y?8W~{4ZPE_g(?l@S^~VR`%7$h%;S)jjU++>L2sM-30d$ zFm#ggI*l+WS}RN7dgsx4>~&&9B^%UFHoc8ZqG@TBu{vqGx3nr3sr{B=-L$n;1=te}Tq_=?$Z4 z5m`ONdBj3sJG=ynv`<4B?B}TEWnWr zibO230BMgBupfFvuf}_Dy+TS*FGvlowC1HuYnV^b-Qz^=9!4eOT$OR$o#-YV#}1TM zIHbH*cbwIssR%C7 zYue0H{`@I#sSXpFK`dZ(n3-=k1s|%A4eY2-DEIL690A3Bb=xqMRzJ-9|AOF00K%N^ zAk0ep2^Hl5Ryj&A1hYq=uAu3^;WL827g-W?7Cq%dJRLyYn+5Om7;1R4m8F(nTUMU$ zwSwl`p(?2IEhiX3nYg63iQ+{jIers z(QxhWB}?x|T;^y;4MtpWO?;7ZBs%h(iY<>+Z1aMaMW$2s6x$N+mQ#vt?4XIFC^~Ny z<3cQDEx&Gv)mRP?O@Uol=~n#jWN0xMTN@W5`ViWi2oOC{A7Q^Nfr!Rr42-RCER3y; ziwGG*u%7*oB}7?ywIT#Dp$S4=;_?dCXCj^2FrvbpGg0BljPP){j-Dh%fHi9}Kp`T7 zA;AO$5vGtmvwVTdCZtBfH-rEI$lxyad@q48bs~S-4Ft+lAIH^A7AEv6!{?9V{t!c- zdHQBu6LVb9aGQpo-l=PVW}_o#IzZhTb)Feo-y*_ zlz1ze|HOPH@`epz59y~GLoMYg<+vn3TZwjq(OC-BA zsnX>l`kf8}{2)(d&3=fbzGgJB1u>%jC7%&JXqu%of()odn_VV-d=VrNMMosa3>$lt zU;rI%6#^O&1n^uACxIDSctG(vID!&KHk3k=0$O$O6D0Ws6h%lGGc@<6u^K$ zE@_06Au$l)ewis;bezc2PeJ)823-2XD3w^k(q~u}#hyo+B1pT9o_K;!M5YMuYE#6P zCx|3jF-08s;;0fqx};4Jt=b45&uKp1S%}yTv+~|7|Nb>Ke)R*G%Gg2QSHMM4>6r!U zpQBO&uSV(I)xuC_)MqZwvYqD$hUg>Ww)hF$l(Zl1UU$sTnw`5wAkLtReHPiUFR<|! z3EoezMzBuM_mli6Pk)TyX9)fS!M76h(OgQm@!IzSd?S8>(2ZrikuY~ ztVjgvX5eRLV(AH{C?KH4#8M;%b+JhVnI+jk7$Oo%@4)@6qz~SWQZ7m#%t{%|69zN4 z0c7b4>?M}waTn3vB*tQs0aSG)ZzH9(*LN57`hDZN^o^hynKAmV<37T0ps5AvPg6)q z$~V?Gz1C%J#mQ+eOx1)o@Liau(41EP5y#j-l>~39`%xkyl$y|{f{g?X(b+{=c^97$ zjfHKf|3t74V56Wo?^|W38&Ov4e~F*h@>VeZabBAt=pz0bS|;3-o9(2eywI9W3%@)F|{&qlWt$HC#aljT!|h-#BU% zfg0-DdXp`sJ9x?oRDDQ?EIU-dgnYsMT93q!!=0d8Lxo zXOVR`z>L^}`tRw^`kdxxaf4_meK<_S%qm{n#9?x{IKzqxpwD4Ekh-ce=Y!f6U$O?!j@~-Rthd zal*X`RCsf^ldqoG-S)U7^*Act)RxytIllJYW-!e7k7bVkG5o~5Bwqv1l5oc-Y_x9A zcHo3+-9O!!K-`7<>Li__<|E0s<@!iH=JVJ#A!_e0qOAT0!NB_2HSF^HW!#XQjR~`} zZ`hcpSSv+YR59WKUZIg57%(p}#5JT0BH_j|k=}%yZ=?%$P2^)DK@gD`GUQ1A)GWSh z>A<1RIE@o<5Ga%s8+bwALw3u!t+2@lp*a@B3wj(SNvDD8!BAn!SVd|d_xHfpf;$qA z=_wT1D6m(Y4YRWC?y`%)r(Vs%1slKT@w-ClByRB{U+%O)XZRUQ;iXcEZrYu~Rnt!G zHcRQLmh702ZL)*?M(2s4GaZnX&FkHGHP>qfb$H1XuF3Ky?5|#i=%&GQ7&I8K+zB8C zV?__1CS$vT*T5R}j7Mh0X4L;e)kXa9_tMNPM8Iy8^mdwI@A8wl`|J2!;Y3*%Z#O!R ze|c}yvnKx?8iak=x}bgBLK{LXKPE}xi9M2p5y-4JkYtTqvZvcWPTf9$2Be;* zKF<!tf&&Do);ITJ)(mPf;Z)OQ^EYN9;aLg7>i@ zWU2`g55r#){U72dw0M)zd2*`@quJ{lAx4f_h*2`@P9rzEM}p$NiKy@0pg!~3g!g$4 zToWFW?6%BCMhrbgd76euc^hLwd5TmnG^c)prQZYyv-3@_uC_{V>bLkHUta^!?cOqk z&m{-QekAp!PeFRih0xKFH?mwQbV?D@zei_pn2@$2&-%pklE|~zFrMiV52FMz@i?1^ zCzfa0KpsXs$b9<_&hTddQZJgM8}<9R3+j-Xf57UaXyea$lLXRv-&Us@@uzmJv4G?+ z$y(_#?4i3FHVrYBS&GOF74^q)m{otoVGJANPvQn+JL)n(=e=*{7)Q{9`eSxC43+O2 zg32Q&gX8tL10?Y}k`9s(G^j3W+a@h(Mz)@s)+CCn|A{W~Ap%5ZmRC5%hVe5`&!noS z5as`%f>0||gFP&ZYV&n04O%|{R*1L zi2jsa{Xc+N^{Xrn8xk?(Z+Y|A0Q6RZ;Zu99A&tO=jaV!3y`Uz~4{*q>vPR?--pVnG zdj3I{ej6Yx=`n3Z5!o%sx9J`Jl5PDiKgD8SJ4 z@m&Qf5NET{^<`_2==zyR*N4+Z`BV9k*BouE0f}8P$l&((=U+enspne}^ zJZN1X3I(r8d{pyBC(`wk&O^jQ#c2qeH>3Cl>fZuHa`wbbn8&A0uun4WQ5sGL8woN9 z3r|A#l&!);F^|MRSCl4(heywUZ^vD(Hn%jbWTVW1&^#59$j+?@2=!N(g^_eH-lK!3w}N-B zs$+YSTdObN6EvfhY#j^|kJT61`_B>dJXY}1Q}n67A)vL`*Id<~@bn!UXGICvTn0l^!v&DZ8?n}@J zxiEQnY+yLjuOnupu5nUoFzB=>HFVI4W@wr2_sG1I9m>5dnF{WX4a<@dOs~v_=A!y` zt{f&N77NCV`d7Sk2LLhvZjEzd;3SU&*pv54$O%#IYG`BcFt*489u$8?A|yGku~0PW z0W93b9&QK~qum^7Jwd(BcD_XLc7lrpf|N5XJxVZulsc6q@x>^7H!5cnSpl14?U2n};Rb6vNgY)y}S6!DG%q$7THN_jZa zQyw1Lq&xt1M0o&`MUFzGZQ+U_QqNInBhtfI3JfdC4~;0_j}rFksJ5GWw^(PO)6J(s zCw6}1G2|BY#-O{64Jm9%#7EOioy9#$hkYwo#YT1OB7BC&M{@=3eb;$O>YSI{Pd7q- zt3N}tab#uywFHd>tB!N118AhT{0#Q^77n$H>N$A4w3LOy3JfE6K z;#m5uM3Va(9iOI3CSeMGtsvg(%jC~kX?vM9B;H$RjVL1` zFJ&TIzpb;ZDQ%tO-N+2WXAFC7eV8@)!8-L(c^)MRim+W8N<1=SuOOedx!(58d|e)0t*_*R2)@ zuKBSX+2xC`ZE>{V=E7 z3dff(Akxx4A8&RE^U=l!NskYA_r9_^6@OZlnA6{572dkJhJ4~B+Eyyx90?~kSBm#n zXx^f-9OEsUtJVA8?$wk}wJqG=uNZhu#BK2jUrdJnmapa&bT7(>s$~D1jW0S@F{<6<92X0{a-dGtTHSXepF#BKUN)ka_o_@^7d2z0O|y8 AdjJ3c literal 0 HcmV?d00001 diff --git a/lading_py/tests/__pycache__/test_config.cpython-310-pytest-9.0.2.pyc b/lading_py/tests/__pycache__/test_config.cpython-310-pytest-9.0.2.pyc new file mode 100644 index 0000000000000000000000000000000000000000..54eebdb499fbc6279d9751b707266dde69af24f8 GIT binary patch literal 16070 zcmc&*Yj7mjRi1uKPtQZ6$7;0?KQdl>?VYu)m9%fio87E8YeQ@i{77t3u3@BZjYc~U z`F3m98qa`X%_9#dn@0gL8>4jsL{%XPV4w`CR0#ZmGX4crs)4Fd`~X!!71UDv*eYVa zb8b&h_h?3|wUt7uIdi-3?R!t(`<=%<_e_0YAQi*kf4rSvd}}xs`xm+h|7o~5gOBqy z5GytlvlL6MC^IU5YgJv1DY0sN#!zHUVkW^g=8TDNt&*&!W>Q?ISJKtYOr|<8Gf>UW zWUGTSgVmv#q3ZC=uo8PTX2mVzm6&B*R%b?Vm9R`)nYh}4tE82}RSH+5xJp|YTxD>z z6ITOP7FStZjalkbvHW0T&(pT!I=T6}$j#Mj^W{=*sVJOstu(%lmxxX06>sq4#Y)*K zy5)N9BSO?g-tbaS*6Z#g+};~JS1&!~7G39@e+spn$ZLYHHlrP%yI!&G1 zh1nI?cD%u-?TTHsU9ln`oTCB5e=07{;Nv_FV#j6_%qh?1E3p~P(kva{x}1xdxSWL< zL%u&|mfsvR%kMmsv<9sqv`<;Xc<&J}OYi+L;^r(dGpAsP_FobO6Q4)$alQj$DJ{i~ zHI>UsOSRM%#%}3Nt*P&bwc<^+8NZ;mjHcmy%vDxZOK%$UipBTm)oZ$CxZ0|IEp{>H zy8nu2$vvt<$8gDL2$pr}9eU)cylYD@XE9 z>LtIBJC&o-MBX_--&1)fpK9e!oyzg$@w}5ec#vD1M2$#LUvW>v14O2X93gU?$jKXn zMaQv)%MVPVFHc`unVecn@`GOL%o0t#TPO-|xG-0&R0_gg5_9z`P-1%-R2OEK%N4g= zb3CI`M+?W>U2qoa%N2{87D|=+Y_U?P6|1(FEI7q^n|koFALen}fc171fbVM2E!ZzE zRf;u+az5$lj$N7e(lm}_bTWK#IDBy|d~rhZ=07tLt~x5KwGhcH`Kg zg?iOKqr6y01O2g=Q>29TX3`E4c{vmk}NPI_6;oUJL1pL`F9m*5BD+u)y^1p0mW8)A^UA0h&y5ECOr zc7QP2C!<1e)E9!IVhk+*-vb0mAHH#hXAk9U8CP|{mCa5w&Tut+xEej+s>5gyJd>EX6O$*W za%ao29A>~r<;0dzNe!WfrRvptVedvZq;&yW#-fB^QYM>8$8ptF4VozF zYa-LvMEYu+?d)OH^8$(!C?x_-gfiP%`i7mgOt<~SuNW1h@vYGAyf@HVO!gZo$GB7gJjl{(GQPN!hifO{% z4^Ol95)YuvIDx&sa7#FuJl3}CRPs$bN(b1OpCElERof)&eW2UO=I3GcrIFUkH8*c| z;f$>NF(M~e$4v+V0~e1qA~qjm@a+@%l+*_wp_kc3WH*sLMD`N7jmYgp?jUj}NIugA z*RY{I9=Qd6v5#)loyQ}(^JF*X2O0D8qBuuhaIr$nJMgArKT?Kb zjXQh6ztcFPOE*_pN=w_IvQ$=Ce4lZy-$Z5UeN|T6S6POYxCNDEPM1$YgEIuh!+2-8 zlBcNjAtGe!q-H-&r7q2$<*9?)Rjw6)C0=Jz0*yO*&05edGHGG=B``qoBRRN2>$D0| zP}tzVNBgsrxtuFva{_PEt~q5Fn-)T$U~6fQ8Jq4v*rJW27LcqXP9gB zS-?!N|G>=N){9xc)pS1XnFaQk3k5G#C{*j#a>YhDQz*Q!T&$2UT_}iy^y_H$90dyJ$DwuRgT@^Yn{kZ|Lx zhQEi?q!qy)PQwmNS_<-lVWD6d=o=OT-mJ#%0eQ1+_aKSdR5R=z(0&cB0XRv^on=V+ zUq$OSC+QxX)KJg%OgKF1ws7LxA~rhfG>2=dr&Y@}km8GLikB?6H}q|jbdXFEY=+!I zqR`nwj#uh(^GHp|^Hs!})-feyK#oSd6>@9{Gq7}2Rg6i9;E|1o0r;sX5nJMR1{+fJ3Ds#DJFA5g??(&teUKTG5!$eMaXefX59bFnc(sKQ1p_pppvAdDgJq0qhu zXU9`6u4x3dNx3~ckEbJ`gO4H}#m#)yGeogw)vKO?XcfLgu!^Gw@vM|ntS&)Y!jOef z#%@cOD-(DzYrYq=j3GQt)JlS|<|zg70)2UtK$R8q=_k^y~P(SOvqw+ASDJl+?W;sW~2`->+kxG25$W%_&TbO{&u`_I!N zx`?8NyJCq5O@I(YTq1H21WlB7s>Oyn|=&hQ=c_%o>3kB?)5uz8GUOhI@18bvXK zS{S??#93FoHL^IQvgt!w!}&U!z00gK7AbTYHFamPQPHf8?qz#ZbT{fbbR!P%hDRD0 zoy%nE{LPUS_f1{ct(2Rf-li@EPRxj-`zz?H%}Fvab$YNpTR^?!q{^di7bj$NQ<9Tq z7-B>T-eykHZ2mJnIQhMRlRwx#PFMpDOqX67{pO?p+<5bJ+q6EiiD|7aBQAQeSXs6U zOU1bhg(ZPV=xn)Cc2^?C(bMMUOM-wUY$9Y{#W@hz*oQaipJk1Mk$pNo4{ty%J2&a3 zV%JLU0=Tp1%P$tNC#u?*@m$&Am@s?-7;p06z1ucwKv*}i8bHw z{}Rndj>rO$yNHyDoFUQ?UpDjutUWAyzKGd(Xa~eHY{0^^jlZjhcuGs(ekLNUWbsG` zu&~lr%8e~TtjJ4!Rs9@-B+|UUswlBf;5P6lL@CK;grfdF%pAhr{GUejkCm>3I2q`{$qNA|7q^uY zl0%9dbvXIE5GUClocwXX$)9Z>CoG49)1C&biZ2toRzcP&aJr@nvf$#ksH}KerCwqI z6~9h*8HrJoCtjn9t36q(`}ZC38z`kS(@2d5D??h5?8 zK_jmb`38}16X_@~F0t$TB~(0)kCOu7=#OEN^Go;pI=}7s4<|)*a_`P#A)hN&D-n6w z#r^ub%6H)5k|w*Xd~N_aY)FBi6N{D_-cQy{>?TLUyGc5E;Lx$w)E46{1cIA7&fw#a zG71hGuziVU!nx=sR!yW-Bp_wT8$#Y2<-5g$VDVYhr(Fbr#G@!749vk|I;B(>Dd8WhR`-nQ$hbw(0RL#&@$x&B`A++F>a;y z;EcYRS{!VpEy~zK0DDAB~9C z9saB=>xlu%F>DSnUs<28Y>2Nkc*dR5a=PZF4+rekPS!>kV*~~I$Lq2Ci-JF5DL2%RNZW!E+;lN;5YZylZ z!^jFFenznNlGDWyr@N^p$|>IEkv6Ya6!F+rxaBs(;Cc^dJSMHZ)@^<+*bblT9lf}w zwZ>-{9D@7Bk=AH)ba6*3jX560Tio6?$A1yx{ElFb?`*F+IA3MXN3UT&EPk80LK!Q6 zCzsnR|NiE=&Ed`s9IDP}h{F%`&^4)3MleV}wq{HtG>+ma@^SsX&Y4Q6uYH_rmza9;IJXco387z;CL?SeI-@o(=j z{y&Are_;E^XTR>j>Bhm$6Nvph;_(%@2J3K`gY>bVA~z2~cX?KiLg%cWQ^F3nt5EQ( zaKJiVvTL>|x^?(|el|GuL%CfVLc5+JGnDgjd9?3fKss+dEG!zDf{W3lFeICa$Eb=} z%PThw_M3 zx%hOtD@KV>GT!~h z(kXryaj{5SyO2~-hj(~-eF-A2lPd9&^aBfIw-ha*p?5Ef+%daSz$qmCJP5zQ2o#0* zLmHiSB3sDw^adhg#!(}0v={unq*X6D^jjm#Guv1d^E6CJGQ?*x1s@$=CJYYo1!|HE zR@BQz*Fk)f>J2#+`CWW0<|*^Y*(hETZ_*eoA`3)5N@O3ApC@uJkv}1lBXSp!GLbVx z=&WCSn8@8k?jdp?k++C6iToUq&Tf%5ir!c|oPwN0|A!&83ZzsW>LZ0?x7*)Msozs6 z6#KgJUBy)2Q0V^)%9oVCR$o?L@egx0ZWv{J7D1P2-;sT!i7~Kyl&6#DpsOh!##waQ zW$Y&{?DWbpL}752QC3Qb*Ln*H^Qhax30v=>Y4PWnRPlX~^&CiJh^-VYwr{4y zU*b0DD#x}hDGdooXSs-^qu)eTJ7{xe6TJl#+oD{xYi_|>hV6!OBr_5{XppGx3c@}_ zgDZ`*n{@7H2+;_23#q4-3eJ=_o?{@PN=J>$**mN;#P`s-x4R#6TUHo*`*VO?bw1C2 zn7fRSZS{U;SPE-Bgkf{0~w< zv!8fHyNbvb$yFd*^4xJ%DdYEm?X1+2A4P=ZP+H14&IMICil_$hObysHSaS9Uxu6F_ zxu6=CDW2KRO4a>%CSou_Jo6@5sa>3W5`F2GD+-yH~(759wCGvl$>`$*cP>C1iBrQN)fHfoY>+Vnx2g(^W5Qy9&9X1(6^c-%YV zBU}q*QopCl@F$JR0>0tx%3#$_9GUb<@w5v%Xcf{9HN5eOf*5PnD2im#4WcQ=jo?%d`GmdCuQa-l3$P zO1Zjgyq0o}OKN$iYkJvb)y=xrYg+jR^yS<<`ts8@ zSX>T7FxpTpq*ErRxta(wKvqP`p+DmmB zn3ombi$tx0k=7Y5Jx-5jPoMW15f^99dg6So;yu3NRo1AbLN6WusHiO98{Q4#rOFDV zz`Oris+@Myu7+RD&A2*#Gp>PsFym?3hoPla6g=Dem?#+dK8|nrGmwZy_lB~eM9QXm zIklm7lub;1Sy585rK9dnZQxZKS|{Dn)>0dpP9}UV(l#?LO3R9?Q~R=dS#ym@-!v|# z&Zolt=r=oQFV6ZwBZz`Vtr8nf<)v0lc(Jw6 zT#vA^wj<(Mn}A>ouPG{l-*nKON4LGys@0=fBaC&Rz^RAvZaZ8FT6LF)+SPini)c^_jk?liAuhSoT?mRHa(*K!FZ*070d znZ3>xklxKLUik<1Q{=J3SFVC|mF~Bsbn1!Y(7L&v)}1)sQ#%Hy*n#2g8-Zc35_xVB z-TUSbg%1|1K~!v~=MOH&O4y#UPK;`!*oDXbgfD#-#5Aaw1!+%3L9hn19Te+9qgt9` zeHDACe>V}FUny}jkz0t|O5`>ow-YIXKyABRdEM(-U+gm6?ajYz$(btihGEV znu&Xf+(+boA|)cEM3Q_4rT7lUZ?qzUD5`s*R0J|rP~wW`xt_Zd!@?O54LZ=%gzqY^ zEA9P>y0G0`lhOYxyx{uLe_eX;0%LEo9uWE^_D*ewJ+|~c%u7o@j`^;o|1;bUMx1yE z+*CjOHGTeeQtU-d>?gufA$*A&i4bJE2K-Y2Vjz8SnC3~8so#Q0Z`M7#Z}6N-rG-`q zm+&mg#nLEtd4ywE1wb?bxzyKX6-ktqo_weyQcu>DaJs@gS}A2$7?sYv$Dx z7)1_*G&ZEe-b|T_Y2+<#jdkHgEzub9GswrlmP?ml9|}?GQffm|X-1Vdlul|@Syk;c z+y(d_{kqm6p`{l~N}QKd+vlBn%X_kv5j1a{3z|3*mRid=?8KwglC^F4Kb`<*;McoY zi)_23#(Je*LwlUY*l|hZZ661eEk_V2LOM=#MOS9w3I(rrlpLFa-6q5E*$Mft6Ydpx z&E&IH6*qG!v!OX4Glgr?X9ucHvRSas?(}^UAuUCAG39E{o?rG3-r#>jJ>$4=a%j~>PZqkB{Z?-O=N zLm&Lzq5X-qm0A>5F$q4mU~eixs}WrxKCh7L3;#RApcJ1)O*~6Y;~NrZYmJKVd>~K| zT*Mh7V+KJo&a;zPyNu6(@!b;`=dlwR4-*;JCNQpoak%<@#`VQmS*hOs&A&eXi`%jb zadxE^MnP3Les?E+cTzl)#H#o*u{M#(0!z0)8RO7uAh40$4ctHzBNa z$;F3Dv!-qs9V0v)Wj1w$u?AdygWAcxMjOhP75>%x4!y~ajLFV>HZswh?PQ0X_YA^T z1A(n{W@4qv$Yq1eriUd;-t+Ok_q-o5>v8ElA1Z}1R^3lw>7kKW^@xmBk8D|q);!BG zsUxc4kQ-dY1`XUCJb4#2ED#}3#EKp3YZvID;rMaiaOEKTzr#V~;o-{BaVvR`6zt&i zu^U0dvt4i5fh!dvJj6-kmGSnENS1Vllhf2t+J_S5{|OU)N^m4v*FHEh3=Zx81P-xG za$wU#=W*9=|K_B`2~|an$SM(^NP|c~)KG z4Cx-dLwZtk*LqUoIC|sRZU@pyT{kPyv)Cmi2ZIlfO=b;w_$uZIEn`A1ZeY$p6u$|RaS+^JAuI{fFXS)IQ2qjD*Lt~IC`wCwpx1;# zw+XpL>4#ez^K5L{y|XrY#`|a~6pzuYn%IdSvfu|qy<0juV#^^PnXHljm> z94B#t2w8{N8oYAIz$fH6lI?w*S)#-@FHj~kS~TlW3h~HrWyHomEYq7Zd$NDukW#bV zxtY<+)7z~O*^i-^kL(nO$huNc3ySt{OCxK)skBQ8I8L798Z1-9L1rMl8Y1S!LQOF(ra;v$m>MD1yWMl&wo1O(T_1F8xTx-?$b>=9Gh%2?_8px8MQIu+r-

5_zhHbTJl@c?-!zN}~X?@UfN2EAQg(gY68%_&g(q zflhq`Up5S6HpdJDA8!+b&NPxTe3AA>W+?G;X1#c>hJ3CQh?u z<>BlP&_=V@B=apga$lOF;?MfmAUKfL>o0v?VUO%GJ7$$ZPvC|0b!EXH!gE?C%{Tz5X_a>YUlADnSC$mLf>M&by zY;dP|NxRng%d{Nzx>`TGl38q96}pz4r@X6BFk9PP3Bu^uauAeMDR0B~!qBPq^K0a1 zNfA%shLLV|ZJvWg4UpVL{R<$m8dk~7S?QM+io}Uya^(1b6+`cPehsK{C z*`5(E@W+^UczYoAbkgSZLn9eCr$lyaxY^*VUXxDX{#mzu+-xl( ziAS3+zJX=7cZ5*BXazoQu@)AW1l=!+_t3*WTvCiw>W0XDfr$5Mc$3J$ycMZ>2a!8L z@JwGm;wRK9h=fEUBIGa0yP)$_xCMwrDF$VewNSNgt)gSQ*!=T$z)DtTbU+?$$MDQC`oSj!rY*HJpFtipPeip zp|tN$kWb>n2~qMZwASwt`2mr)h`dW=lL$#n{FKN5MqHu;RJ;sA+c1cd4Krs>;j5b# zqGakX?T!uGc7qCCru6>%fbTxzn{b6sjLypl6#3G5;a$Lig#SJ0H^oM`O=`t%Oc_tL z8nuX@^buI|EsjW|Luk}{YybZP<_G^2(Em>wc_~Xr3tuzDHxEWPo%MR)xbuUtEJ!{T rNhA1vO8O|2A7J*9_9ZqSl{aH2$R_jVXsJ7uo%~lY3U?IDxkLW}k$>wm literal 0 HcmV?d00001 diff --git a/lading_py/tests/__pycache__/test_payload.cpython-310-pytest-9.0.2.pyc b/lading_py/tests/__pycache__/test_payload.cpython-310-pytest-9.0.2.pyc new file mode 100644 index 0000000000000000000000000000000000000000..57164cb493ea2d7cb1fa25a38ce279205b06970d GIT binary patch literal 20786 zcmds9dypK*S>Ku6ot=H&-sxfaeYRtqHS$>}TYh25a%?5B6Q4tD;&8#)?A`30R=T}C z^~}lAdY4in#~~&P4zGk@;1tZ`0)$6F5ugAUP(^@1C|*SkPy`4B@k|k3rNlPB-`6v< zGpk+YY{Ea>)qc~{)6?BE{e8dh(cRN2mkTNUZTsEYkxze3D)l!!iT|Z>au7fF_duM~ ze9AE#bH$i9<(O_~%#@L8XXh}4s^PUdt25{D@Ts|4)}M%q z-Pc)NYAySD9^cwnU2gi72UZtX*4@_eramveskQ1n)@&^w_FTXGP}5Vb#o)A-_a~Ys z)*7o$-D|ejRvKQ@F9sPaE!Xp>7S>xUPJOYn>NQVz^|elC#jgY%YStH4I*UjB!s~dr zud#Tz=@)eM_cc~l{QRNg%~em5)km7@cx$nF-{B@p;^K}vp19RUgSwXb!bz{`F7o-} zeodS+Ir|6V$w4Jh3!7=Yl z;AL0+DZcDO^3;z854~uh`tUD@1b_G8=Y9;tG5Us=>KUhvKH6S3`stq8OCL4+nO??y zgJ*1*j@ipBfz6n|WS-49885w&Ih#73a$ke%Sub_gP&K41rdADHN4jU6HRNap$Mjjl z$v&6and*bloLn#4GLW{s97@?e>nwWk9=w4}s&C+D;+N^CofMwRp6eFdGO{zPjdpX> zIB)LXziD*$T>sj&legB~!jnN_djB+2dur}$dE8rbYlZ&wew}-J%~gEz&5WN$8Irvu zx02j`zS3~rrt+kMJ8Ld8@7&DG9e&~98VAs;Hv++oVywY%5tIPGZlQ;9(U7__rZfDj``c-))tDQ)?wxhZ) zlBqvtbg#RR4`+tUk7dj+K95S`jJce0%#S5m^NnB|6HH0g%tWl2jac(~T-U65BiS%u z%`DR+Sk~Y4B#IS&U{e=R#Mg|RZ3)yDX0NU zWDW4|d|y^O$qHh}$X4EEDdK!p3kT-R<;pl191uDuYXY8l^b?7?r=Hy3bR zn3Z=~GrCg&=7&Ik3((tAO?Q38+cebOSHY|cax}i)8b{vSh6Vdl5r&v8Qg5TTEr|Iv>Txq5CF7LQ z-91h`M%|hiVh(RK#6OA?8jQ!>#mF-Pn~EYd zu&BPlGdEz{U7)ry8(FcauE%uOAEQ6B^xc3 zMZEKL1bX>Eymd=)@y^kzL+yrBm6j$GE&b|Oza;^askv@R@RBfcXTN`r118838jyes zZ9pP1WI)a)49MGv87s|Ik^aq0geE$Wx=o1edCgU=sa)RRQhIIU6UGs!hVHfFWNvWNT&>{SbupOhb-z%rw>!={ zlwaL1)$7OB8!NQ4>UA}PdeyBYw~^dI@_Ld7NZvs*u&N|Q(@IrAXpp%~*yx#^4*oNx zv|ObQ@z&L9!fhjFP~6|dTRCK2%7e%OTmi3VV2_@G;Nu)lQh4rUm!S)t+%o)Wr1EF7 zeL%xTx(8>;NO>8j(1QE*yzxN;y018=9z?C1rk`%NRzVskHVr@D3Z33_>_E4yy`jM9 z^wZ7N<9_CF$Mw3!HPvxKjhXfBz_)e%5+K#BuQgS@?JoNzU4gt^*U#fxkSs7Moa)MJ zgEyXWT8o}aqg_S4*2dF0!i{i|Ha5ws>dhd2h7k_+5YOL2^7SCKvb=sUJK`jrwr?S< zf}`SFlDoDHFnIwOKaIim(`t2Dt+R>4Jm!r|kfOB%tA*x6UoqHBeT*s5D!Qk^!VC}8L2AOPqd!o6+)&@c5$y&^;fPvd?9cfpbbF4Ag5 zKKm~1lxQt_VGloyyh5*t9zG{mkwOm<&lG#I>`XY-URea867-}J>q)7fJ?0&o5K)+% z6H!o2j6moXK1_CppQqU!i2=JybZ~^-2cv9;YOGFWS?I@thp^=|8lFN6n^`RhVzLpX z)V(Bx)AJLtFybr%kpSJ3vwO}Pcd2j2<+$}1$Qr%XG};SIK&kH_>tm(0ioo-*WaXn? zzA}uS&_>{pjGba@0@TFlxUqM;dM&RJuoJ<^>^-Rw+UGd`wDc`NTwva6h8^PPtbU?k@%059!f5zhCG<)j-?)GMNg{L6|8kkcTou4 zT_B-kP)QYFG(Tf}z%XBgvbu4ceCXE}uWYWCQ{TjHJx=mYk}kCN!`Re z7vunbt~R643(K}kdC6Zye&UZt$U*$vw}K4J^(JPFC})^P6gk68mCXL&Xi?MPnA4Ms zn8}i9Ma*Q$gyMWbPbJP5ohcag+x$sB^c(baVN+)1R2^+4q#yF0VsAwY6T0Z0(KTZq z&ZNarV_N@b&RDd{&!ka*F5q6)u$lHPZBF^wCa3xQGF%%@@zpU6xOlW?`6UMvh^n=) z&Y4R;yWCjE++@mrH`PmRq|T1_oftw+pHBBPuyLUO;Ze~#cqhjq z@`P^i@_`+*8>!Ha!Q5w5;trM&J7zyi1a=HCJ1X&j)B#&Y%}E_d0pYSz8d&D&sT~U_ znuj%ms{x9lS4S#{dF7%=6#MrBC^f-JARs%uE*>;Ak~TF|(iku;-X>?6?L zDTq0r=R9o?6{9`H9>nb_M#Zu)`surs=g!@TSA2e7%!atg^d_v8-D0lmc}~C3axwqt zHC7kFiZXkxN~kV~ROtZIPGo4~{W+#4L12^IauLrU;3MzkT5~X$3TrCi4%~}+5^*a| zwi3c>RrMom=r~E_87RuA`Zx)DsIDTpnna>m(@b4MaxKYqAbv0bTxUNdP`_Cm#%8j( z`Z1ClNW@DJr-6W~Efl`q7}klXTp~z}SI4Q9;AuI<+d)~E-6QxGAT=LBh^AY+0EM)t zaN+$Ihz=iGfa%|m7y(QKD*Yb;6T%XhYIhLUMOZ>+UL-7G!L@*ZIJwgX4DfAmB5Z`1 zwXj(p0=!<~w5jgsWx$Ho5tt1|>e%)0aBXUyuF6i0SESnhBj@C1A6|S zhIYYiA}UE9aurpZI!Hq9t?d{B)?t?2Kp{UXldwbV>!%xzqkfV#3Hxqg8fi4|{w$j? z>b|j{t#M)AvF7#F!na(D5ED@ss|~PLrLnNmq_ny0&;aoZ*#hEjCH#}0%~Pk$?(7(T z9$#;Qugj?4#!vYAQ8UKZSTJ%1Fn)$pA)G|eJe-L_YuSinp&Fnba)EkqBv2N*h&Oo! zCi740D#}ii|mx5R-eK3qE|($-^P}hLaU`% z&%c)?q~{wf5%yftR3yxUB9SAEc1*L-K`+WQOXwKrK1YrLlnE7zjb8!1%^aB;*Z*e? zcN#T^Pf#841pkQZ`U$?kK8$#R7g>V+e%4SX%Y5~pA$0E{|0(U`6FD{w^*l2+Nkoqb z?voqUkIVJ*#sOf!gRd?I;I+IJXgm7<2Sme0?*Dt)!qX)013?Q9yvi03nI3AOwk?K0 zWA&vx;uiEJj8Hs-jt&rzFq?`d5vaSKoj*Y$LAFs!lftaSwcpCrYf0Wt@&S_F5fSwn zOl8wd)+?jJ1#}MIP!RGj8P>~%v{s@o7@sn_UpoeN#M~aLCjmGK8+0CA5hzmH`2SA; z2hlW{gAJl-KLo+5M$xp;9vuMS)1eT;zrTnG0zMdCj?dB-0>~R@kHVu*ikZ!O)mulvEer?1yi`BIdp|+4ixo_6ML47@TM!OQ zH>ekU{1n?5V(KeIwj3N*f5q zLPLEWQV1A{hJw2im3RY7h=zJIO9bc+H$5uxcBupHtH-4dq@d->QkuH866n?m>oD?OfvQihB2#xWDXeL+?Jvc9=r%c1fGi(Q@Qlxl}~a3fhYtQnZDjwrIS|hHd4~ zgFW!~g?r9raZv9fSp-2qKs4bGBkfmXkpqcEoXfFXQegcI7e9-{4owgaO${d46Q&)O z=7RD0yA0&r@ctjS!GM^&_Kef!+pt9am@MGhH0BVvi@ZC`5t}o6e4{-qWcj-(QYY5b zfgfR|tld1dE7iSgj1_PZ#zR#+sp5=(1jp zb>CJ5QaKNicFz`UF-HbC!|J3AjN80>jDm-i zybiw=ah^6= zlU^zWrNsPzVJjOGA}DjXUKNIY1gS};0xdpCoh^CLozUjmgyh6*I7K{#tJ@Os9t(Pm zF`sN7Q^4pACV+dTqej2%Sp5p7gDXhey|Q{5J)^!aJCnUKg~*v=$zEkCjp<_c1DY7N zMA%DecHK*A_GB-$cAe~Dg4o$6ntj6A?(FDIh-R;Px|i_5Bh6k!FU#yHex-ip*fpYe zcMc1i`gM%l@<+dif4^{#`V7u4RKCBDT)z@Yp#+J}?L{VnQC7>HUhR0WfN#H;1xyLn zQbRLXZV^$XDJ+~nWDlF?&Aaz2pKX{re#;ewck56WRt@S|ZzO_QZ^V&jVmw>u>zlSx zGOQI&Fid=z^uC0z;bD@mCmA@zThgN=a6|nu>)9$^Ope5h0ZNFb6zzEfs}+>Ektvz5 z7Tqd9BJQo|)(`ON5RhokHulo$tyR4XgQyyX*5dk=fwgC#W6e{UFOT(cN7Xo2D-BQ6 z(c;p<^Pgtw7f61UdA8H|57*LZ-yG}dWMy4&28YJM zhJ*W4&l-;(;_Ukd_LL!H4QgVY9RgR`9;}7FjUxkaoPEz@bqwdcIZvI#wjT!Hpnrql zn-vA$9^{<2td4n@bKb$~n0VOm9V|g`Q}1F4rT}?4NmEA;>RMw-scW5Aqq+tG#&@$s z4*;n+*Ir}_b=i_;;4c51C4oCw8sld410X`s&O->&!)8i^)EY?V!ZZT;)qc5`1FZlq zD|izVy$Sbaw8A$rA<&L|f%ZXk;4{)UKIb1vf8x(sM&AZc@i{5Ur#L4i<4-|YEeotx zCHz&T?^b~4^u1ReR_!~SA*&v1WHq9*-7Kk)`;q$gwhjM4|C zH2R9thotoXs^7;h4+4(oroq_fwu5l}EtcWxHHqPj*&MNa#0nzVShBbXV4Pe%r12OK z8sIT-T?zt#Q4rBN&THL`=CHG6FD6E0-GgB5J;>Axj79)1Y`fQAM6GZs++fyj4A|>J z;WQ!L20cS4&Ao-E1bMXd=tWwJ%|U#{1`}mN15;co(F_dnVeKHomFln9`WCgqz~fY> zdD~BdAZ&C!(KV|%_pNS0*HQ@HN=ABO00)%-924uwqgqv^1yw0?()Tkm@f%67PXpj2`?p{JuybUS_ny>KosZ z;);b2na8j-U4{=c34e#{o2L3L9E2ONq~hK~)#|B0!820^FJMY*Eh{{^R?W!ArfByr#1QDU*?#&W@gO1iO8Xm`BrtD2}54i0IPCanQ5$lqzJ%dm0E# zQU=uTpb!N~uQfT9Nm=GqFBe|!?tfM9Dhv>*PoqlAv#y@h;hJDoB#}Zzz~S2p(+A3h zkf*z?HT4Bv{mMZaERY&tL{8RCxL7@{6MAD`Y(W7Z7k;-ehzy8G{^eToM-DyhGnenn zGIko5;kVPUOxxV`QYLM^Xr@j4GIGoWN1ey~QgFTcqF(ZK0WOSeFu@BYG)z<=uDPES z=Tomsz7P?(m}I%F^5z++5y2dG&n4=u;e?XBrTXlDH&P=EJ;hv4g$GYTNy=h zxcYG-5yc4tx#(?GdRtBO_V-af>@DoK3wpaDec*nLACf+V`!z1??axU`_VyR06m9f9lT6vNL-d>c7~TQHoR4Y7#ku&F)XA#v0cPV*`ludED+2y(u|d1j+yH(3L-9 z*e`XbMbR_{O71iqg_DS@&^A$0gW`Z zBYxpdAZ#V0hCDY3GT;})As(U`2mA9CE{xZoG8wZh{=eDCdr6)qc^^o)3^&PkelDoz zl`UM?n~U_kwY0@?-0~dtc7bP^5yAQxcMV&Nn$EwW2A9SXx$8HAFIEgA?9e?UR@43i zJMteSUm|&lWWd>y5_SFOxIm|Uu$s1Hmr6y&{6+i{f3#l?;um`1JIU*Zf^W)5A7950 zk|a2sC?6sHM{uGy_47M2*n)_enbX*K80|{7f?dfO`OeGhC2EhQVc=&su!k+! zmHf51j(9ydb`Mgr&lnt=mozvwh@@vFl0JT~arix#@?1CHJ~^Y^0QKv9&hG>1+Stc< z7Z;0nZEj|~YpK)GGs=v&s6PO~R!2--CF9kb1#O;8u+SAGT<;K5{L?(WhU8k3>p()C z|J%&K#MPDp%X*7iVxixQxLShf3?_y{hi@??hK<9= z|78Lb1{Q;dkMnHjwwrXcJ@H*Km{8C(*O?!L1j50*RR)vF?9Oli2|ox0ge4A12@dyz zQbGe1w{(L=Fx+`D&t>6u1W0ta5z2GH9l| zc;Tp3UhSwAUdgBh=p5gtQ3^+|#L>GrHplN9q2&nz3ZLJ@&Q_8l$pA$LTK;7`>zDBx zX!)#dl`@L?wX0$L)Sb>EzF}a8p9>Q`P_?High9C2CI?G%Uh3?@Go$ubmaiPQDsg(B>cXwH8axejJiUwZ7?>{Yq>$1j>cT zMh_=0?1*dPCwSey;cIxUnU%y+yO@ZDL@Z+OgqMi5nl6iY8|i@Tirf0xvo+D1{o>=@ zt@&CwclPWY6&Gt^qF-LXMdJ+IBeK}5dSMvIi2EEzA2c4{ElPq?Qd;hAMc7E!lgS~s z?lqgP47#%3X>~mrBo*9t*ESo?sL>7+tJy)1F!_`lZge)AK6iE3o1HbU>9)OAkQ7}4 z0?m+ApW&xAI_>AB?8xLk7m#UrksG|#ZF+4l!kwCxm|@VoC@b`GGqj*GJy9)qK~>bS zW7*1yT}^%#y>Fj7y3uI`N5d%SZUpT&y~{y!>u4zJN1LACXs^57t)ujIAyX_j1!i|E zd4S(K#G>7%w-z)*tQ=*alkgJ~kPeE2;vLa@I0f2~sUva_r1M+G#SpxWf-m}_q4o7h zBTxZ^fM4H4O93rrj8oVL=wj=#(YIqeYylX$Z^SlWXi_`FurLJKQv|X|1IQK$WK;sk z3REUY@-KkLzs3&T836ceXj3Dh0Jf&q z7r1m>1}+h=PltAB&;m{YMlrf~9a#kf~1a0GhJkQ{Puy& zFOkSbjI2dpSjRgbvi;J3DVo5_k|o}C-!F>b`;1khe=X+ zLtXxRR3hyf#7!)=1sOX#fGNuQj`m%n51|Sju1AIsQF~2)fV8dNFOYbJcDG_Hd^R$- z3lOUoi&eA}GSOzmgdX~Z=p(BDBF`q1D7IPTIS_d!G_{ivc_iYjet|`tvyzw_jVQcX zzzQV^e~38N%nKvU41u-F8hL@m(Cw^7hNO{Sq+vx#`ipT>X7)jD85i)V==}soX&5EOW`4siYh^|HXnPH0(K6|9iVFvARIqsSl zHtL5l>J~x85*1N}vgp+&aZ)MEv9Z(HyNwAavX@wG+?Ca4j@2(Qs|(C(uy+cp?UdCp z;P%Pt(uySD-=vd&laP6n*v&kb=cx2Fk!OeyL*=tXo+mOocPz#q9NSp0Uf*nOHa#>G zBX7~Pw7uyDAKlLeXXco`&A-FM*=^ntALtA{!yEgtoFVbnO|s+fVKbFz9U;vRE_$2I z$i3<{HvOAL1GHS^C z4=T4_`S|44>sYURmB?9;n`{uX@cKY<>rs|l0~Ovl5uw`r zif{%}>?_j`-d$(RF@n}j(wD?=_;(Ou#LwAceOrTe$05A#M6eUCiSybu*srmEUVFnL zAD^rRp4p-(9bmT@?l9pmC%uh3OLBhl7RIP3s@24oL!8pZF^|nqMfY>3-$py zOmyEGgTKrbOLLnPtj~z!QACD&{;mt9E>?!KIlGrIX z_^{T5d=7nBf$a+U!k{c!k2z#w`gS*t#^JppH-}gLj=**kuxfqa1I5a zh$0SDR1@xs#ELw*9z?L4ifSDE`NX=w;U+ZdN9N^Aw4Am4no6VDzD%oPQ$^GD+4rsY zdw0O#UDqZ(dJutc5PHcNc*o!_3Pif(GVYUuc`O%oIiaF1Cu&t?;}xXWUJsH2B%zEF z^HQhLmgi}D!n7Q-^T^_opCEESk)hdFOP$g-S`%BXAFGJui`0$x=Brft8r^5hPuY3L zvy~-foA4$wPiOgz=XQf!hmf9a2@`76(yjL#vxJjPD)pzL_t-u;zEUg7Q?#PfM2I`` z6(Zjx@+~6YA@V&UZxI=pOaQ+32whFp~)H+&|;_v`4VB5wm9QEk8SD~5e@qA}+%ZexOp z8hWxvG0qtyXBR1t!em>Foyv*Mqm|lX6ffJK^H_vk)CyXPGupS8(?Vl9i)2h2_(LLN zyLdnEB5%Fhxf;kfWh07$Hf^MQ4J7ZUWp{JWnCZKE0~1Z$RaUGycYJ(D^X~WyAEPU= zo^Iped)dZEGTWFMIN=@(hwRvpQ~`Nuj&dwc?v)h^tZI9w&NY3rKZJXcZvOw+yWHqlw%*SX602CS;Si`njWt}6lQO+V%^I7s~`2W3$2NZ;KUy!$@% z8pb||u^GKq8V=vTZukoQ=sh`^MM{E<56_7VGKR>=B|AO*!=8x5 zm|r@GK!Abk6eI=HkcqM)BSyRonJ73pv2j)O5%BC6V<%k7Su7=tDaWO_40os!SJ0yv z{v~C!!+}+d=8%7Z93^&zzK;7yq9R}Q8cIl(ruizADdb9RY<2e=9)(89< zzZxIlSq}0n2Wb|}P>xOiz>eO~@T^1rLH|&Eh$ccIIZ0PT2S-n1sK7KV1{It#&7xYqa4_oTIcCQdug!g+iL$LMn?hVhA6rI+sUvfkTj8G!lE@2qJhI1> z9y`JGlp48+2T^@-o*lzpHe0qx?hoBNmO41{m-L899?h6DLxJqX^19ui?I+bihIp;h zX?D69Pv^d|qbTWRIUC1uZFz*Mi$o~)z=mfo<90k(OH*!XZf+@;a!WIHX*%v$?p~VH zQ}-5gJr@_VJr|E>WH_6+pJy>Jlpn4yy+%9m-A;QLHo1(IXQC!m@t){vz0-G+3wN9^ zCnU<4+{VR1;_oOZ>Fy&!@`i&53FTA}Cm~^R1>ZQNSrz0|`X*u_NLZ(_kU~TWYtgsX z5i|IsfVi+?8^QqMB3z3K+ZHk`N)Vtu6$JLS!-C*x+%HDRw#f4+m8istYm`|K&@TxB z`ei|I!^F`ma{9T*N3Rr>akoOpSba5t<5j>#N6txOCdHMSA%BB6OtphHFshwp`3{-~ zz@I1J4^zscjB+LmO-C*LwBjTHZiO8g1%Az(kW@aIbN5dxcmH&4hSksbY1pA+@(4(4zGHVWeZ$Y8I%1Yxu*p=L#kts6>{Ye*IXdev<4Xw*(Aboy}STW z>hERT%^v*Iiod+qtbW_3y9cgJaCd%9udq$zPelGqWRkn{bKD>- z%AXOTTJ1n$yRP4<;~NKO@I{&=*_k|m&%7dhO>jhMbbMEKLL$)c$YK zz)ugq5`&%HM9!T)S{!^NrdUbxi9d$m;3B(Qyqrm^PMkC9GmLXMxhgeu(NMVYA2?1gX2~v=)f;=P;LLMNPTL7ovxr;X*hzg1q6))!cegEm{>7MPC zR!N{>x90ED-RGP>ea`=X|Lt^ZWTcqF-`n3Szw+MQsnkF4B>HFJP9M^M83oTVTqgu<3x8$t2CC|Cy zmD*M1I;vD{+NIN1+Lx*dSG=W?a@ZZYvb;#D;z_`YkAd1>73AK&Ye4hC+Rz=e(vLEPA68c zwN{#*qlyV#UFF-f?76Y0PoEn*7}5v9#jV8+Etp)mzQpQ2P}Yt1F96v41`}esm`6XW3b9snyDbRnKut z&&f$i_gin4bzxJb{faDow2hxX6&4u$1!Ucz{W*?8`<~_8TMJ> zyP6T{IfpWTPvGZX1F?-Y!%KCI*Nru^o?5fIX4krCuBE$a_bJb~X4+;qy#Pt$^#${K z+D?1cwe*Y`@-_2gf^FM=^OsCci{7n4PYnGkD3+3ni)VlGWIXAa%bS57@*j_zcc8gboLTRqV z(q!2^$m4X`Ef?2HbNcGhva1Lz>uKLY9g=C1!z4%F8?Cw!YEL>iUUqr$_otqavg zrCD8e{6fX8E;#JLAAL-o3IRE-R7?50Dgd12s#kHYv>Vl?h|YS>PrFWI!5`vt^f0HQ zCo|ELBhiyteWLq3rU$A|rY6@@Y7Eoy-pPq2h`@yFIqfB%s9(!ZCyElK3M6DVMOCM^k!&Z~L9&m8v!%%al>8x?AQw~Ka3015mlOFF zqjPWKMCy96narYoD9vx;#KT-dlA${Z5|B7+&FmUo6B2E8E%%hnWEPTaKn8JLBsw3T zNzR#)nPdqPotK$}B-@3zO-OB;y0e=W3C;u(oQX&3+2V7n;d2*cRGKmFIlh)NcuyP5>Z8EKpw=;B@E>;lDw6-hSPGr zhD)(OBF-uN+%Cx04Gp^pgCl1T*~2*I?Gbwv$AZ1h-i~9@-eK>=ame0f@5XW1zRTW& zit(Ewkjxfr_60rcq?Pr%&uXcn9#qqv;U_oX`TkG;w`tXf&$ z@e|roIEg~9nRHgR1h)K~#yVGh1pVm8{1{8}tbDH<@})d!r_tGy{G`}aXhzUR-_7r$ zv6u3+n&1q=_~Q%`7hWMkaB46ixRSBcYuRq5n}zzwb#v}ZR1{FbIn2FW7xj_zGS{+# zPE)v_BVwr|DCJp#`p8Kc2A~|%!4V2W&SCo5`L{DrD^SOFv6~mQQt%46TkIC1YK2B2 zei{67Ygth-L&sQ5!ThN*Yvs`V`p5~eW}h& ze6~GWh=JQx@mAYTWP!u>{;mXv%+CeQMDqdaK1_T&(z9$)2rOGzwA$E3p>OPt!y+rL ziOCCVIAZeBjvXS8x|NIdrtPk0L~E*HzT*f8f%AyiCGTyE*uPUiSr{{`Q&a0hiZAOA zoq6H;=PG9(d+yA$r+o|0i{R-1lnT6tc)DKHD)>1*@?#`az3P6F{UA}bd@Oo$R1vmf zChPRzy6o66#~gR&PPBCXZtPii27S} z&}bBoY^{T+flRG|%7!9->Ge$Fv1f7jar|ylP9I8jW)rxq2TGT!4Mdb8uoUW@MKn&O zo_Xt(Zy_k;TL=ch#6F5^N*I{1r#=Yc4|m=lST2GY7uoP62^HP#46yi{ov$Rr!``61lTdDM{9mEdI2kO+u`HRf1D zbm-%(v6&8q&SWqJF(2s6tmw>Kpfhs=Ix_=FH0_~#5bV4MTGktSWf+=$cn!{Pw}5wm z&g~Wh2k{$tCdW@dWQ5nn58*iSkk`?0lwaT|y&=5E7dc9npy@|t#68Up-7V)uwA>~d ze#qW#@8}N6XorK*4o60thi10|4PU^x^r$V48lfqUXCxTszo6&dICt`Tbe!_k5j=I5 zpvS1a+rFziDo@P@PmKng z2*F`=epW}*V2$+0GR;wql1P*!RB2JaFL*d?oOM$-r418%1H7@~Yn4YLu=6Ur?PJew zp7ve8xV@q-kF(g(!^LJ7Z6K_ecLU0i<2G1tfnsn*6vmGSR}- z$aHK|4!T_TmTrMfs>MFYOsY*nN%JiXLAgBjNLEM)D1Mr`7(XMq7`4jF9g?dg6xzUk z_|9Wx?R99Q;ZCF%4ew=d;%&UnQYf%$ji;X^`4owm6cj@B8p(jYyeA@jkvU+$5%E(L zzlXLmc`*Tudf-1YKSSJs7cjm=3ouM;ZSHH_m6-7I3t`hA)c4S#v9KE%!8y*k_#a6Y zS-kFsxeLMTpn#VSa(jrdF>Ybg=jf-cUKER;2DOf*>a>r4 zWj%r(bd>c`@d|W}tdwD~$&vm`QX*lacM>GCx`thZ4Tj$u0%vRaZmydLRKbpP-$akF zB;g$y@N;lopo;zoY&m%vt*;S*ssjC-Zb6_b8+Hnvp>{{^FDBr#lC;Q( zfM+TAA}&F)MA8#lIH}n)-1QNL4wzCwI5XmML^I7uPwZZ4UTn55H-l_v>F_i~gQ`r+ zhr<#%B9ZyCgVP!}G|N#Zx37$Zc~hX|?y zwq=wAb|oZ?GDAST)^~;-dL6?0tF_+dnW1CLfBwkDr>Cu_9!{VJ3~y zFPvC(n%GsKPSP0_1~FgTlj=-tajd=8okNj8@ohI>_*S*KirJ&j(GNTKAmCkYwVa5?G~*N^8b?tQ zAqlc2h#X1m1P~V-ArL{9BqtG{oaq025Y2#f5N@9#pk;$NN*eG-(HWZW+#QI{a-jB? zq4Z~+8I5f>$bXu#+Fv@Q@$C{WiD#^?u+;a?14Erov+9!cwTP@?e{p49DQ!BE`X&oM8!r3@Z@chYPmgioY5XyOCTX%TMV7Gm zkIalu9S#)RTX^He{h$4(kACZcf?oWx(RqF#{4<3eRyx?j(iQ_ikFXOR0TNa!Cx(9{ z?KIhmWuuVnlFQTT7cn|mm3<#Cfpwxuq~qfM#ga%NeZ5k6=m2!#AS3bwI`JvjsrYEW<&lB?94N3q+@gX?h(8$Kw)Yo-S$07vZI z(a-`&VOV*AT9heQKuRtQE9XNfIYO=J0W{R3C?Rz!E)6W7u*TD(R~dMImNj|_7Ab2z zTo65648qDxxb(uxMbsLiD-wc?1Ze}kJkkf3-X_&j5R;G13NePWB09K)g6QC3>{UVn z`pcLj&d=`Dms7u%4)P#YXLqm1tK2U;H#Sji*C#sG*jniZg*K`IeWO0Gu%e?qRCD?0 zQmS)20UKo3PNUv*fG3rjs=7-_%KFAMdhm^+hQ=`Nk>nqhhy9GIHWwYS1^x6Zt$I^} zBoUk(fXD&WmltODGaQwxA0g1l8026`V~p2?9N-A=7LeTdP6{_AE0Q(#Ad2{Pqx0~V z=3>6QL!IIy9wT|2WS-;#$pQ%lMSYrtrlR^h$O=B+$>(=m#{h2sBg`dk1{r~)47AV%edb8~ddK>r| z@Mw(X4U8j$A4lt!Ko-H~zacOnoAXSX%5`dX6I<&nbXc6nrnpQMmY3JbJyL5m7wU_h zqMVPfR+k$$j!&jiH~8i^ED&rs&z)VVVFMDj$;oz7*DpR*Z#eD~+zWn#@`gIY=fII} zwMKP`?E$QC)@vzri>Kvk%Qjif&#tt+I`^t#f86D&vK=MMgp`MXZ-2=SpjSUrv8!J7 zlPJo5oUCaYrezxG|4e7h|4RQ`_CGpzk;NiYGcGM+6GUT+g3JlJ`F)UX>J?xj!aDT& zfP?TW;NQKD{m2ONz#IyVtc4(CLL&=ahc>bz8|B`Q`#Q)o!2n7^kcXk2Y?r&2Pq79| zFu!z=2WTwUBEcRxr#Z&DG{uq&h$$;9 z5hXE_o%s?@;~Pfc2#AV4jQt*8XJzSZCri6XZV7VOW-Ux5z!tXywXzQOJO`>fIGa?z zik@TId_xXJhjG1DyV7tjIoN)^g8f?c8jt0FsN_M_C>83RWstnmTXprfQFStHo{9u>tC(g)q3$D|MK z24$o+V=q~8ZY)Give;XSyV{|_XrRs!XinRCl(AbTaJxQ(dP58!100IHE)G4PO+$c3 z_Btea8H*|CEQNUd$d_pwPtX*>bVZVaUp^tTUb?msO@H-HRtVMj>TN&0BM%2USP`+3>&u$Bc;?0>~ue@)WY>7wT)z!Udg zMECas-4krnPq>QbE*hU^j=NWdkK2P3cta_b+TYYC=TMjPE|hA(>SnVqe5|9YS7L-DK)g6tMyaF zz72#!J|+zFCEA`HWMgg;*Ae;mj0rIa5HNa@hO2~z^tX2jTbUC=$Kc+mp(?B^mSK-` z`%L>17tbT%_5y^21gwQYw`g>3WJ__W?_;0Bk)L8|!2Z(PXm$FEL5Q!Sj)WUWO}Npr z*?0pp^lQgBAjXhA?Npoy{I7{g^}{G4e5Z`P=o}%^aYzhtpC>kLYH6Z@@Vk7u?}6xC z#fCxOqUn5jbTAAc^ZrNd@;fAdLh=_Re@XIPlD{E&m*j6r{*I(Sxl%g7YZNZJ0Fw9+ zzkeIQSPB2>@n@t~d552?RP0s_-^S#7UtbC+`S;+XMb5yuhEF)UK^>r@De|- zkZU>?a~Tiy4}{-5jt_q57=8n;(|Q)G@87gK5-s>HTyy2is4CyvNT5Q-p7iQ|{)GP4 b=OiNq?skxLYRuSf3>)d(=$_FXJ3sQ@w6R$B literal 0 HcmV?d00001 diff --git a/lading_py/tests/smoke_test.py b/lading_py/tests/smoke_test.py new file mode 100644 index 000000000..ea7ff7ccd --- /dev/null +++ b/lading_py/tests/smoke_test.py @@ -0,0 +1,96 @@ +""" +Smoke test: spin up a Unix datagram socket server, run lading-py for 3 seconds, +assert bytes were received and the output file was written. +""" +import asyncio +import os +import socket +import tempfile +import threading +import time +import pytest +import yaml + + +SOCKET_PATH = "/tmp/lading_smoke_test.socket" + + +def _socket_server(sock_path: str, received: list, stop_event: threading.Event): + if os.path.exists(sock_path): + os.unlink(sock_path) + s = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM) + s.bind(sock_path) + s.settimeout(0.1) + while not stop_event.is_set(): + try: + data, _ = s.recvfrom(65536) + received.append(len(data)) + except socket.timeout: + pass + s.close() + + +@pytest.fixture +def smoke_config(tmp_path): + out_path = str(tmp_path / "output") + cfg = { + "generator": [{ + "unix_datagram": { + "seed": list(range(32)), + "path": SOCKET_PATH, + "bytes_per_second": "1 MiB", + "parallel_connections": 1, + "variant": { + "dogstatsd": { + "contexts": {"inclusive": {"min": 10, "max": 10}}, + "tags_per_msg": {"inclusive": {"min": 2, "max": 2}}, + "multivalue_count": {"inclusive": {"min": 2, "max": 4}}, + "multivalue_pack_probability": 0.1, + "kind_weights": {"metric": 90, "event": 5, "service_check": 5}, + "metric_weights": {"count": 1, "gauge": 1, "distribution": 3, "set": 0, "timer": 0, "histogram": 0}, + "metric_names": ["test.metric{{0-4}}"], + "tag_names": ["env", "host"], + "tag_values": ["prod{{0-2}}"], + } + }, + } + }], + "telemetry": {"path": out_path}, + "target_metrics": [], + "warmup_duration_secs": 0, + "experiment_duration_secs": 3, + } + cfg_path = str(tmp_path / "config.yaml") + with open(cfg_path, "w") as f: + yaml.dump(cfg, f) + return cfg_path, out_path + + +def test_smoke(smoke_config): + cfg_path, out_path = smoke_config + + received: list[int] = [] + stop = threading.Event() + srv = threading.Thread(target=_socket_server, args=(SOCKET_PATH, received, stop), daemon=True) + srv.start() + time.sleep(0.1) + + from lading_py.config import RootConfig + import lading_py.main as lm + + with open(cfg_path) as f: + raw = yaml.safe_load(f) + config = RootConfig.model_validate(raw) + asyncio.run(lm.inner_main(config)) + + stop.set() + srv.join(timeout=2) + + assert sum(received) > 0, "no bytes received at socket" + assert os.path.exists(out_path), "output file not created" + with open(out_path) as f: + lines = [l for l in f if l.strip()] + assert len(lines) > 0, "output file is empty" + + if os.path.exists(SOCKET_PATH): + os.unlink(SOCKET_PATH) diff --git a/lading_py/tests/test_capture.py b/lading_py/tests/test_capture.py new file mode 100644 index 000000000..1dc3a4909 --- /dev/null +++ b/lading_py/tests/test_capture.py @@ -0,0 +1,321 @@ +"""Tests for capture output: accumulator, JSONL writer, Parquet writer.""" +import json +import os +import tempfile +import time +import pytest +import pyarrow.parquet as pq + +from lading_py.capture.line import Line, MetricKind +from lading_py.capture.jsonl_writer import JsonlWriter +from lading_py.capture.parquet_writer import ParquetWriter +from lading_py.capture.accumulator import Accumulator, _parse_key +from lading_py.telemetry.registry import Registry, _key + + +# --------------------------------------------------------------------------- +# Line serialization +# --------------------------------------------------------------------------- + +class TestLine: + def _make_line(self, **kwargs) -> Line: + defaults = dict( + run_id="test-run-id", + time=1_700_000_000_000, + fetch_index=0, + metric_name="bytes_written", + metric_kind=MetricKind.Counter, + value=1024.0, + labels={"generator": "dogstatsd"}, + ) + defaults.update(kwargs) + return Line(**defaults) + + def test_to_dict_fields(self): + line = self._make_line() + d = line.to_dict() + assert d["run_id"] == "test-run-id" + assert d["time"] == 1_700_000_000_000 + assert d["fetch_index"] == 0 + assert d["metric_name"] == "bytes_written" + assert d["metric_kind"] == MetricKind.Counter + assert d["value"] == 1024.0 + assert d["labels"] == {"generator": "dogstatsd"} + + def test_to_dict_no_histogram_key_when_empty(self): + line = self._make_line() + d = line.to_dict() + assert "value_histogram" not in d + + def test_to_dict_base64_histogram(self): + import base64 + line = self._make_line(value_histogram=b"\x01\x02\x03") + d = line.to_dict() + assert d["value_histogram"] == base64.b64encode(b"\x01\x02\x03").decode() + + +# --------------------------------------------------------------------------- +# JSONL writer +# --------------------------------------------------------------------------- + +class TestJsonlWriter: + def _make_lines(self, n: int) -> list[Line]: + return [ + Line( + run_id="r", + time=1_000_000 + i, + fetch_index=i, + metric_name=f"metric.{i}", + metric_kind=MetricKind.Gauge, + value=float(i), + labels={"idx": str(i)}, + ) + for i in range(n) + ] + + def test_writes_and_reads_back(self, tmp_path): + path = str(tmp_path / "out.jsonl") + writer = JsonlWriter(path) + lines = self._make_lines(5) + writer.flush(lines) + with open(path) as f: + rows = [json.loads(l) for l in f if l.strip()] + assert len(rows) == 5 + assert rows[0]["metric_name"] == "metric.0" + assert rows[4]["value"] == 4.0 + + def test_multiple_flushes_append(self, tmp_path): + path = str(tmp_path / "out.jsonl") + writer = JsonlWriter(path) + writer.flush(self._make_lines(3)) + writer.flush(self._make_lines(2)) + with open(path) as f: + rows = [l for l in f if l.strip()] + assert len(rows) == 5 + + def test_empty_flush_noop(self, tmp_path): + path = str(tmp_path / "out.jsonl") + writer = JsonlWriter(path) + writer.flush([]) + assert os.path.getsize(path) == 0 + + def test_valid_json_per_line(self, tmp_path): + path = str(tmp_path / "out.jsonl") + writer = JsonlWriter(path) + writer.flush(self._make_lines(10)) + with open(path) as f: + for line in f: + if line.strip(): + json.loads(line) # should not raise + + def test_overwrites_on_new_writer(self, tmp_path): + path = str(tmp_path / "out.jsonl") + JsonlWriter(path).flush(self._make_lines(5)) + JsonlWriter(path).flush(self._make_lines(2)) + with open(path) as f: + rows = [l for l in f if l.strip()] + assert len(rows) == 2 + + def test_label_roundtrip(self, tmp_path): + path = str(tmp_path / "out.jsonl") + writer = JsonlWriter(path) + line = Line("r", 0, 0, "m", MetricKind.Gauge, 1.0, {"a": "x", "b": "y"}) + writer.flush([line]) + with open(path) as f: + row = json.loads(f.read()) + assert row["labels"] == {"a": "x", "b": "y"} + + +# --------------------------------------------------------------------------- +# Parquet writer +# --------------------------------------------------------------------------- + +class TestParquetWriter: + def _make_lines(self, n: int) -> list[Line]: + return [ + Line( + run_id="run1", + time=1_000 + i, + fetch_index=i, + metric_name="test.metric", + metric_kind=MetricKind.Counter, + value=float(i * 10), + labels={"env": "test"}, + ) + for i in range(n) + ] + + def test_writes_parquet_file(self, tmp_path): + path = str(tmp_path / "out.parquet") + writer = ParquetWriter(path) + writer.flush(self._make_lines(5)) + writer.finalize() + assert os.path.exists(path) + table = pq.read_table(path) + assert table.num_rows == 5 + + def test_schema_columns_present(self, tmp_path): + path = str(tmp_path / "out.parquet") + writer = ParquetWriter(path) + writer.flush(self._make_lines(1)) + writer.finalize() + table = pq.read_table(path) + for col in ("run_id", "time", "fetch_index", "metric_name", "metric_kind", "value"): + assert col in table.schema.names + + def test_values_correct(self, tmp_path): + path = str(tmp_path / "out.parquet") + writer = ParquetWriter(path) + writer.flush(self._make_lines(3)) + writer.finalize() + table = pq.read_table(path) + assert table["value"].to_pylist() == [0.0, 10.0, 20.0] + assert all(r == "run1" for r in table["run_id"].to_pylist()) + + def test_multiple_flushes_appended(self, tmp_path): + path = str(tmp_path / "out.parquet") + writer = ParquetWriter(path) + writer.flush(self._make_lines(3)) + writer.flush(self._make_lines(2)) + writer.finalize() + table = pq.read_table(path) + assert table.num_rows == 5 + + def test_empty_flush_noop(self, tmp_path): + path = str(tmp_path / "out.parquet") + writer = ParquetWriter(path) + writer.flush([]) + writer.finalize() + assert not os.path.exists(path) + + +# --------------------------------------------------------------------------- +# Accumulator +# --------------------------------------------------------------------------- + +class TestParseKey: + def test_simple(self): + name, labels = _parse_key(("metric", (("a", "1"),))) + assert name == "metric" + assert labels == {"a": "1"} + + def test_empty_labels(self): + name, labels = _parse_key(("m", ())) + assert name == "m" + assert labels == {} + + +class TestAccumulatorFlush: + def _acc(self, registry: Registry, writers: list) -> Accumulator: + return Accumulator("run-1", registry, writers, flush_seconds=3600) + + def test_counter_delta_first_flush(self): + registry = Registry() + registry.increment("bytes", 100) + lines = [] + acc = self._acc(registry, [_ListWriter(lines)]) + acc._flush() + counter_lines = [l for l in lines if l.metric_kind == MetricKind.Counter] + assert any(l.metric_name == "bytes" and l.value == 100.0 for l in counter_lines) + + def test_counter_delta_second_flush(self): + registry = Registry() + registry.increment("bytes", 100) + lines = [] + acc = self._acc(registry, [_ListWriter(lines)]) + acc._flush() + lines.clear() + registry.increment("bytes", 50) + acc._flush() + counter_lines = [l for l in lines if l.metric_name == "bytes"] + assert any(l.value == 50.0 for l in counter_lines) + + def test_counter_delta_zero_if_no_new_increments(self): + registry = Registry() + registry.increment("x", 10) + lines = [] + acc = self._acc(registry, [_ListWriter(lines)]) + acc._flush() + lines.clear() + acc._flush() + counter_lines = [l for l in lines if l.metric_name == "x"] + assert any(l.value == 0.0 for l in counter_lines) + + def test_gauge_passthrough(self): + registry = Registry() + registry.set_gauge("cpu", 55.5) + lines = [] + acc = self._acc(registry, [_ListWriter(lines)]) + acc._flush() + gauge_lines = [l for l in lines if l.metric_kind == MetricKind.Gauge] + assert any(l.metric_name == "cpu" and l.value == 55.5 for l in gauge_lines) + + def test_histogram_mean(self): + registry = Registry() + for v in [10.0, 20.0, 30.0]: + registry.record_histogram("latency", v) + lines = [] + acc = self._acc(registry, [_ListWriter(lines)]) + acc._flush() + hist_lines = [l for l in lines if l.metric_kind == MetricKind.Histogram] + assert any(l.metric_name == "latency" and l.value == 20.0 for l in hist_lines) + + def test_histogram_drained_after_flush(self): + registry = Registry() + registry.record_histogram("h", 5.0) + lines = [] + acc = self._acc(registry, [_ListWriter(lines)]) + acc._flush() + lines.clear() + acc._flush() + hist_lines = [l for l in lines if l.metric_kind == MetricKind.Histogram] + assert hist_lines == [] + + def test_fetch_index_increments(self): + registry = Registry() + registry.set_gauge("g", 1.0) + lines = [] + acc = self._acc(registry, [_ListWriter(lines)]) + acc._flush() + idx0 = lines[0].fetch_index + lines.clear() + registry.set_gauge("g", 2.0) + acc._flush() + idx1 = lines[0].fetch_index + assert idx1 == idx0 + 1 + + def test_labels_preserved(self): + registry = Registry() + registry.set_gauge("g", 1.0, {"env": "prod"}) + lines = [] + acc = self._acc(registry, [_ListWriter(lines)]) + acc._flush() + assert any(l.labels == {"env": "prod"} for l in lines) + + def test_run_id_in_lines(self): + registry = Registry() + registry.set_gauge("g", 1.0) + lines = [] + acc = Accumulator("my-run-id", registry, [_ListWriter(lines)]) + acc._flush() + assert all(l.run_id == "my-run-id" for l in lines) + + def test_multiple_writers_both_receive_lines(self): + registry = Registry() + registry.set_gauge("g", 1.0) + lines1, lines2 = [], [] + acc = self._acc(registry, [_ListWriter(lines1), _ListWriter(lines2)]) + acc._flush() + assert len(lines1) == len(lines2) == 1 + + +class _ListWriter: + """Test double that collects flushed Lines in a list.""" + def __init__(self, lines: list): + self._lines = lines + + def flush(self, lines: list[Line]) -> None: + self._lines.extend(lines) + + def finalize(self) -> None: + pass diff --git a/lading_py/tests/test_config.py b/lading_py/tests/test_config.py new file mode 100644 index 000000000..5232c46a1 --- /dev/null +++ b/lading_py/tests/test_config.py @@ -0,0 +1,140 @@ +"""Tests for config parsing.""" +import pytest +import yaml +from pydantic import ValidationError +from lading_py.config import ( + RootConfig, DogStatsDConfig, ConfRange, InclusiveRange, + parse_bytes, TelemetryConfig, +) + + +class TestParseBytes: + def test_mib(self): + assert parse_bytes("1 MiB") == 1024 ** 2 + + def test_gib(self): + assert parse_bytes("2 GiB") == 2 * 1024 ** 3 + + def test_mb(self): + assert parse_bytes("100 MB") == 100 * 1000 ** 2 + + def test_b(self): + assert parse_bytes("8192 B") == 8192 + + def test_fractional(self): + assert parse_bytes("0.5 GiB") == int(0.5 * 1024 ** 3) + + def test_plain_int(self): + assert parse_bytes(12345) == 12345 + + def test_case_insensitive(self): + assert parse_bytes("4 mib") == 4 * 1024 ** 2 + + def test_500_mib(self): + assert parse_bytes("500 MiB") == 500 * 1024 ** 2 + + +class TestConfRange: + def test_inclusive_lo_hi(self): + r = ConfRange(inclusive=InclusiveRange(min=3, max=7)) + assert r.lo == 3 + assert r.hi == 7 + + def test_sample_int_in_range(self): + import random + rng = random.Random(42) + r = ConfRange(inclusive=InclusiveRange(min=2, max=5)) + for _ in range(100): + v = r.sample_int(rng) + assert 2 <= v <= 5 + + def test_sample_float_in_range(self): + import random + rng = random.Random(42) + r = ConfRange(inclusive=InclusiveRange(min=0.1, max=1.0)) + for _ in range(100): + v = r.sample(rng) + assert 0.1 <= v <= 1.0 + + +class TestDogStatsDConfig: + def test_defaults(self): + cfg = DogStatsDConfig() + assert cfg.multivalue_pack_probability == 0.08 + assert cfg.sampling_probability == 0.5 + assert cfg.length_prefix_framed is False + + def test_length_prefix_framed_rejected(self): + with pytest.raises(ValidationError, match="length_prefix_framed"): + DogStatsDConfig(length_prefix_framed=True) + + def test_metric_names_default(self): + cfg = DogStatsDConfig() + assert cfg.metric_names == ["metric{{0-9}}"] + + +class TestTelemetryConfig: + def test_short_form_path(self): + tel = TelemetryConfig(path="nong") + assert tel.output_path == "nong" + assert tel.format == "jsonl" + assert tel.flush_seconds == 60 + + def test_long_form_log(self): + tel = TelemetryConfig(log={"path": "out", "format": {"jsonl": {"flush_seconds": 30}}}) + assert tel.output_path == "out" + assert tel.format == "jsonl" + assert tel.flush_seconds == 30 + + def test_parquet_format(self): + tel = TelemetryConfig(log={"path": "out", "format": {"parquet": {"flush_seconds": 120}}}) + assert tel.format == "parquet" + + def test_prometheus_addr(self): + tel = TelemetryConfig(prometheus={"addr": "0.0.0.0:9000"}) + assert tel.prometheus_addr == "0.0.0.0:9000" + assert tel.output_path is None + + +class TestRootConfigFromYaml: + def test_parse_lading_yaml(self): + with open("/home/stephenwakely/src/lading/lading.yaml") as f: + raw = yaml.safe_load(f) + config = RootConfig.model_validate(raw) + + assert len(config.generator) == 1 + gen = config.generator[0] + assert gen.unix_datagram is not None + assert gen.unix_datagram.path == "/tmp/dsd.socket" + assert gen.unix_datagram.bytes_per_second_int == 1024 ** 2 + + dsd = gen.unix_datagram.dogstatsd + assert dsd.contexts.lo == 50 + assert dsd.contexts.hi == 50 + assert dsd.metric_weights.distribution == 5 + assert dsd.metric_names == ["name{{0-2}}"] + assert dsd.tag_names == ["tag1", "tag2", "tag3"] + + assert len(config.target_metrics) == 3 + assert config.target_metrics[0].prometheus is not None + assert config.target_metrics[2].expvar is not None + + def test_minimal_config(self): + raw = { + "generator": [{ + "unix_datagram": { + "seed": list(range(32)), + "path": "/tmp/test.socket", + "variant": {"dogstatsd": {}}, + } + }], + } + config = RootConfig.model_validate(raw) + assert config.experiment_duration_secs == 60 + assert config.sample_period_milliseconds == 1000 + + def test_empty_config(self): + config = RootConfig.model_validate({}) + assert config.generator == [] + assert config.blackhole == [] + assert config.target_metrics == [] diff --git a/lading_py/tests/test_generator.py b/lading_py/tests/test_generator.py new file mode 100644 index 000000000..d492a698b --- /dev/null +++ b/lading_py/tests/test_generator.py @@ -0,0 +1,176 @@ +"""Tests for the DogStatsD generator: dispatch to dogstatsd-py and rate limiter.""" +import time +from unittest.mock import MagicMock, call, patch +from lading_py.generator.dogstatsd import TokenBucket, _send_block, _send_metric +from lading_py.payload.dogstatsd import MetricCall, EventCall, ServiceCheckCall + + +# --------------------------------------------------------------------------- +# TokenBucket +# --------------------------------------------------------------------------- + +class TestTokenBucket: + def test_acquires_immediately_when_tokens_available(self): + tb = TokenBucket(rate=1_000_000) + t0 = time.monotonic() + tb.acquire(100) + elapsed = time.monotonic() - t0 + assert elapsed < 0.05 # should be near-instant + + def test_throttles_when_rate_exceeded(self): + rate = 500 # 500 bytes/sec + tb = TokenBucket(rate=rate) + tb.acquire(rate) # drain the bucket + t0 = time.monotonic() + tb.acquire(250) # should wait ~0.5s + elapsed = time.monotonic() - t0 + assert elapsed >= 0.4, f"expected >= 0.4s, got {elapsed:.3f}s" + assert elapsed < 2.0, "took too long" + + def test_multiple_acquires_accumulate(self): + rate = 1000 + tb = TokenBucket(rate=rate) + tb.acquire(rate) # drain + t0 = time.monotonic() + tb.acquire(500) + tb.acquire(500) + elapsed = time.monotonic() - t0 + assert elapsed >= 0.9 + + +# --------------------------------------------------------------------------- +# _send_block dispatch +# --------------------------------------------------------------------------- + +def _make_client() -> MagicMock: + client = MagicMock() + buf = MagicMock() + client.open_buffer.return_value.__enter__ = MagicMock(return_value=buf) + client.open_buffer.return_value.__exit__ = MagicMock(return_value=False) + return client + + +class TestSendMetric: + def test_gauge(self): + client = _make_client() + m = MetricCall("my.gauge", 42.0, "gauge", ["env:prod"], 1.0) + _send_metric(client, m) + client.gauge.assert_called_once_with("my.gauge", 42.0, tags=["env:prod"], sample_rate=1.0) + + def test_count(self): + client = _make_client() + m = MetricCall("hits", 7.0, "count", [], None) + _send_metric(client, m) + client.increment.assert_called_once_with("hits", value=7, tags=[], sample_rate=1) + + def test_histogram(self): + client = _make_client() + m = MetricCall("h", 3.14, "histogram", [], 0.5) + _send_metric(client, m) + client.histogram.assert_called_once_with("h", 3.14, tags=[], sample_rate=0.5) + + def test_distribution(self): + client = _make_client() + m = MetricCall("d", 1.0, "distribution", ["a:b"], 1.0) + _send_metric(client, m) + client.distribution.assert_called_once_with("d", 1.0, tags=["a:b"], sample_rate=1.0) + + def test_timing(self): + client = _make_client() + m = MetricCall("t", 99.9, "timing", [], None) + _send_metric(client, m) + client.timing.assert_called_once_with("t", 99.9, tags=[], sample_rate=1) + + def test_set(self): + client = _make_client() + m = MetricCall("s", 42.0, "set", [], None) + _send_metric(client, m) + client.set.assert_called_once_with("s", 42, tags=[], sample_rate=1) + + def test_sample_rate_none_defaults_to_one(self): + client = _make_client() + m = MetricCall("g", 1.0, "gauge", [], None) + _send_metric(client, m) + _, kwargs = client.gauge.call_args + assert kwargs["sample_rate"] == 1 + + def test_sample_rate_passed_through(self): + client = _make_client() + m = MetricCall("g", 1.0, "gauge", [], 0.25) + _send_metric(client, m) + _, kwargs = client.gauge.call_args + assert kwargs["sample_rate"] == 0.25 + + +class TestSendBlock: + def test_single_metric_call(self): + client = _make_client() + m = MetricCall("g", 1.0, "gauge", [], None) + _send_block(client, m) + client.gauge.assert_called_once() + client.open_buffer.assert_not_called() + + def test_batch_uses_open_buffer(self): + client = _make_client() + batch = [ + MetricCall("a", 1.0, "gauge", [], None), + MetricCall("b", 2.0, "gauge", [], None), + ] + _send_block(client, batch) + client.open_buffer.assert_called_once() + buf = client.open_buffer.return_value.__enter__.return_value + assert buf.gauge.call_count == 2 + + def test_batch_all_metrics_sent(self): + client = _make_client() + batch = [ + MetricCall("a", 1.0, "gauge", [], None), + MetricCall("b", 2.0, "count", [], None), + MetricCall("c", 3.0, "distribution", [], None), + ] + _send_block(client, batch) + buf = client.open_buffer.return_value.__enter__.return_value + assert buf.gauge.call_count == 1 + assert buf.increment.call_count == 1 + assert buf.distribution.call_count == 1 + + def test_event_call(self): + client = _make_client() + e = EventCall("My Title", "Some text", ["env:prod"], "error", "normal") + _send_block(client, e) + client.event.assert_called_once_with( + "My Title", "Some text", + tags=["env:prod"], + alert_type="error", + priority="normal", + ) + + def test_event_no_alert_type(self): + client = _make_client() + e = EventCall("T", "B", [], None, None) + _send_block(client, e) + client.event.assert_called_once() + + def test_service_check_ok(self): + client = _make_client() + sc = ServiceCheckCall("check.name", 0, ["host:foo"], None) + _send_block(client, sc) + client.service_check.assert_called_once_with( + "check.name", 0, + tags=["host:foo"], + message=None, + ) + + def test_service_check_with_message(self): + client = _make_client() + sc = ServiceCheckCall("check", 2, [], "something is broken") + _send_block(client, sc) + _, kwargs = client.service_check.call_args + assert kwargs["message"] == "something is broken" + + def test_tags_passed_to_metric(self): + client = _make_client() + m = MetricCall("g", 1.0, "gauge", ["a:1", "b:2"], None) + _send_block(client, m) + _, kwargs = client.gauge.call_args + assert kwargs["tags"] == ["a:1", "b:2"] diff --git a/lading_py/tests/test_payload.py b/lading_py/tests/test_payload.py new file mode 100644 index 000000000..7fddede26 --- /dev/null +++ b/lading_py/tests/test_payload.py @@ -0,0 +1,270 @@ +"""Tests for DogStatsD payload generation.""" +import random +import pytest +from lading_py.config import ( + DogStatsDConfig, ConfRange, InclusiveRange, KindWeights, MetricWeights, +) +from lading_py.payload.dogstatsd import ( + expand_template, expand_list, + build_context_pool, generate_block, + BlockCache, MetricCall, EventCall, ServiceCheckCall, + _estimate_block_bytes, +) + + +# --------------------------------------------------------------------------- +# Template expansion +# --------------------------------------------------------------------------- + +class TestExpandTemplate: + def test_no_template(self): + assert expand_template("metric.name") == ["metric.name"] + + def test_simple_range(self): + assert expand_template("name{{0-2}}") == ["name0", "name1", "name2"] + + def test_suffix(self): + assert expand_template("m{{1-3}}.count") == ["m1.count", "m2.count", "m3.count"] + + def test_single_value(self): + assert expand_template("x{{5-5}}") == ["x5"] + + def test_expand_list_multiple(self): + result = expand_list(["a{{0-1}}", "b{{0-1}}"]) + assert result == ["a0", "a1", "b0", "b1"] + + def test_expand_list_no_templates(self): + assert expand_list(["tag1", "tag2"]) == ["tag1", "tag2"] + + def test_ten_values(self): + result = expand_template("value{{0-9}}") + assert len(result) == 10 + assert result[0] == "value0" + assert result[9] == "value9" + + +# --------------------------------------------------------------------------- +# Context pool +# --------------------------------------------------------------------------- + +def _make_cfg(**kwargs) -> DogStatsDConfig: + base = dict( + contexts=ConfRange(inclusive=InclusiveRange(min=10, max=10)), + tags_per_msg=ConfRange(inclusive=InclusiveRange(min=2, max=2)), + metric_names=["metric{{0-2}}"], + tag_names=["env", "host"], + tag_values=["prod{{0-1}}"], + ) + base.update(kwargs) + return DogStatsDConfig(**base) + + +class TestContextPool: + def test_correct_count(self): + cfg = _make_cfg() + rng = random.Random(1) + pool = build_context_pool(cfg, rng) + assert len(pool) == 10 + + def test_names_from_expanded_templates(self): + cfg = _make_cfg() + rng = random.Random(1) + pool = build_context_pool(cfg, rng) + valid_names = {"metric0", "metric1", "metric2"} + for ctx in pool: + assert ctx.name in valid_names + + def test_tags_are_key_value_pairs(self): + cfg = _make_cfg() + rng = random.Random(1) + pool = build_context_pool(cfg, rng) + for ctx in pool: + assert len(ctx.base_tags) == 2 + for tag in ctx.base_tags: + assert ":" in tag + + def test_tag_names_from_config(self): + cfg = _make_cfg() + rng = random.Random(1) + pool = build_context_pool(cfg, rng) + valid_tag_names = {"env", "host"} + valid_tag_values = {"prod0", "prod1"} + for ctx in pool: + for tag in ctx.base_tags: + k, v = tag.split(":", 1) + assert k in valid_tag_names + assert v in valid_tag_values + + +# --------------------------------------------------------------------------- +# Block generation +# --------------------------------------------------------------------------- + +class TestGenerateBlock: + def _metric_only_cfg(self) -> DogStatsDConfig: + return _make_cfg( + kind_weights=KindWeights(metric=1, event=0, service_check=0), + metric_weights=MetricWeights(distribution=1, gauge=0, count=0, timer=0, set=0, histogram=0), + multivalue_pack_probability=0.0, + ) + + def test_single_metric_call(self): + cfg = self._metric_only_cfg() + rng = random.Random(42) + contexts = build_context_pool(cfg, rng) + block = generate_block(rng, cfg, contexts) + assert isinstance(block, MetricCall) + assert block.metric_type == "distribution" + + def test_all_metric_types_reachable(self): + cfg = _make_cfg( + kind_weights=KindWeights(metric=1, event=0, service_check=0), + metric_weights=MetricWeights(count=1, gauge=1, timer=1, distribution=1, set=1, histogram=1), + multivalue_pack_probability=0.0, + ) + rng = random.Random(0) + contexts = build_context_pool(cfg, rng) + seen_types = set() + for _ in range(500): + b = generate_block(rng, cfg, contexts) + if isinstance(b, MetricCall): + seen_types.add(b.metric_type) + # All six types should appear across 500 samples + assert seen_types >= {"count", "gauge", "timing", "distribution", "set", "histogram"} + + def test_event_call_generated(self): + cfg = _make_cfg( + kind_weights=KindWeights(metric=0, event=1, service_check=0), + ) + rng = random.Random(1) + contexts = build_context_pool(cfg, rng) + block = generate_block(rng, cfg, contexts) + assert isinstance(block, EventCall) + assert len(block.title) > 0 + assert len(block.text) > 0 + + def test_service_check_generated(self): + cfg = _make_cfg( + kind_weights=KindWeights(metric=0, event=0, service_check=1), + ) + rng = random.Random(1) + contexts = build_context_pool(cfg, rng) + block = generate_block(rng, cfg, contexts) + assert isinstance(block, ServiceCheckCall) + assert block.status in (0, 1, 2, 3) + + def test_multivalue_batch(self): + cfg = _make_cfg( + kind_weights=KindWeights(metric=1, event=0, service_check=0), + multivalue_pack_probability=1.0, + multivalue_count=ConfRange(inclusive=InclusiveRange(min=5, max=5)), + ) + rng = random.Random(7) + contexts = build_context_pool(cfg, rng) + block = generate_block(rng, cfg, contexts) + assert isinstance(block, list) + assert len(block) == 5 + assert all(isinstance(m, MetricCall) for m in block) + + def test_sample_rate_present_and_absent(self): + cfg = _make_cfg( + kind_weights=KindWeights(metric=1, event=0, service_check=0), + sampling_probability=0.5, + ) + rng = random.Random(0) + contexts = build_context_pool(cfg, rng) + with_rate = without_rate = 0 + for _ in range(200): + b = generate_block(rng, cfg, contexts) + if isinstance(b, MetricCall): + if b.sample_rate is not None: + with_rate += 1 + else: + without_rate += 1 + assert with_rate > 0 + assert without_rate > 0 + + def test_sample_rate_in_range(self): + cfg = _make_cfg( + kind_weights=KindWeights(metric=1, event=0, service_check=0), + sampling_probability=1.0, + sampling_range=ConfRange(inclusive=InclusiveRange(min=0.1, max=0.5)), + ) + rng = random.Random(0) + contexts = build_context_pool(cfg, rng) + for _ in range(50): + b = generate_block(rng, cfg, contexts) + if isinstance(b, MetricCall): + assert b.sample_rate is not None + assert 0.1 <= b.sample_rate <= 0.5 + + +# --------------------------------------------------------------------------- +# Block cache +# --------------------------------------------------------------------------- + +class TestBlockCache: + def test_deterministic_with_same_seed(self): + cfg = _make_cfg() + seed = list(range(32)) + cache1 = BlockCache(cfg, seed, max_count=20) + cache2 = BlockCache(cfg, seed, max_count=20) + for _ in range(20): + b1, b2 = cache1.next(), cache2.next() + assert type(b1) == type(b2) + if isinstance(b1, MetricCall) and isinstance(b2, MetricCall): + assert b1.name == b2.name + assert b1.metric_type == b2.metric_type + + def test_different_seeds_differ(self): + cfg = _make_cfg() + cache1 = BlockCache(cfg, list(range(32)), max_count=50) + cache2 = BlockCache(cfg, list(reversed(range(32))), max_count=50) + blocks1 = [cache1.next() for _ in range(50)] + blocks2 = [cache2.next() for _ in range(50)] + # Very unlikely all would match with different seeds + assert any( + (isinstance(b1, MetricCall) and isinstance(b2, MetricCall) and b1.name != b2.name) + for b1, b2 in zip(blocks1, blocks2) + ) + + def test_wraps_around(self): + cfg = _make_cfg() + cache = BlockCache(cfg, list(range(32)), max_count=3) + b0a = cache.next() + b1a = cache.next() + b2a = cache.next() + b0b = cache.next() # wraps + if isinstance(b0a, MetricCall) and isinstance(b0b, MetricCall): + assert b0a.name == b0b.name + assert b0a.metric_type == b0b.metric_type + + def test_count_respected(self): + cfg = _make_cfg() + cache = BlockCache(cfg, list(range(32)), max_count=17) + assert len(cache._blocks) == 17 + + +# --------------------------------------------------------------------------- +# Byte estimation +# --------------------------------------------------------------------------- + +class TestEstimateBlockBytes: + def test_single_metric(self): + m = MetricCall("my.metric", 1.0, "gauge", ["env:prod", "host:foo"], None) + est = _estimate_block_bytes(m) + assert est > 0 + + def test_batch_larger_than_single(self): + m = MetricCall("x", 1.0, "gauge", [], None) + single = _estimate_block_bytes(m) + batch = _estimate_block_bytes([m, m, m]) + assert batch == single * 3 + + def test_event(self): + e = EventCall("title", "text", [], None, None) + assert _estimate_block_bytes(e) > 0 + + def test_service_check(self): + sc = ServiceCheckCall("check.name", 0, []) + assert _estimate_block_bytes(sc) > 0 diff --git a/lading_py/tests/test_registry.py b/lading_py/tests/test_registry.py new file mode 100644 index 000000000..7730a585b --- /dev/null +++ b/lading_py/tests/test_registry.py @@ -0,0 +1,133 @@ +"""Tests for the thread-safe metric registry.""" +import threading +from lading_py.telemetry.registry import Registry, _key + + +class TestKey: + def test_same_labels_same_key(self): + k1 = _key("metric", {"a": "1", "b": "2"}) + k2 = _key("metric", {"b": "2", "a": "1"}) # different insertion order + assert k1 == k2 + + def test_different_names_differ(self): + assert _key("a", {}) != _key("b", {}) + + def test_different_labels_differ(self): + assert _key("m", {"a": "1"}) != _key("m", {"a": "2"}) + + +class TestRegistryCounters: + def test_increment_basic(self): + r = Registry() + r.increment("hits", 5) + counters, _, _ = r.snapshot() + assert counters[_key("hits", {})] == 5 + + def test_increment_accumulates(self): + r = Registry() + r.increment("hits", 3) + r.increment("hits", 7) + counters, _, _ = r.snapshot() + assert counters[_key("hits", {})] == 10 + + def test_increment_default_value(self): + r = Registry() + r.increment("x") + counters, _, _ = r.snapshot() + assert counters[_key("x", {})] == 1 + + def test_increment_with_labels(self): + r = Registry() + r.increment("req", 1, {"status": "200"}) + r.increment("req", 2, {"status": "500"}) + counters, _, _ = r.snapshot() + assert counters[_key("req", {"status": "200"})] == 1 + assert counters[_key("req", {"status": "500"})] == 2 + + def test_counters_persist_across_snapshots(self): + r = Registry() + r.increment("x", 10) + r.snapshot() + r.increment("x", 5) + counters, _, _ = r.snapshot() + assert counters[_key("x", {})] == 15 # cumulative total + + def test_thread_safety(self): + r = Registry() + N = 1000 + threads = [ + threading.Thread(target=lambda: r.increment("counter", 1)) + for _ in range(N) + ] + for t in threads: + t.start() + for t in threads: + t.join() + counters, _, _ = r.snapshot() + assert counters[_key("counter", {})] == N + + +class TestRegistryGauges: + def test_set_gauge(self): + r = Registry() + r.set_gauge("cpu", 42.5) + _, gauges, _ = r.snapshot() + assert gauges[_key("cpu", {})] == 42.5 + + def test_gauge_overwritten(self): + r = Registry() + r.set_gauge("mem", 100.0) + r.set_gauge("mem", 200.0) + _, gauges, _ = r.snapshot() + assert gauges[_key("mem", {})] == 200.0 + + def test_gauge_persists_across_snapshots(self): + r = Registry() + r.set_gauge("g", 5.0) + r.snapshot() + _, gauges, _ = r.snapshot() + assert gauges[_key("g", {})] == 5.0 + + def test_gauge_with_labels(self): + r = Registry() + r.set_gauge("temp", 37.0, {"zone": "a"}) + r.set_gauge("temp", 22.0, {"zone": "b"}) + _, gauges, _ = r.snapshot() + assert gauges[_key("temp", {"zone": "a"})] == 37.0 + assert gauges[_key("temp", {"zone": "b"})] == 22.0 + + +class TestRegistryHistograms: + def test_record_histogram(self): + r = Registry() + r.record_histogram("latency", 12.5) + _, _, histograms = r.snapshot() + assert histograms[_key("latency", {})] == [12.5] + + def test_histogram_drained_on_snapshot(self): + r = Registry() + r.record_histogram("h", 1.0) + r.snapshot() + _, _, histograms = r.snapshot() + assert _key("h", {}) not in histograms or histograms[_key("h", {})] == [] + + def test_multiple_samples_collected(self): + r = Registry() + for v in [1.0, 2.0, 3.0]: + r.record_histogram("h", v) + _, _, histograms = r.snapshot() + assert sorted(histograms[_key("h", {})]) == [1.0, 2.0, 3.0] + + def test_histogram_thread_safety(self): + r = Registry() + N = 500 + threads = [ + threading.Thread(target=lambda: r.record_histogram("h", 1.0)) + for _ in range(N) + ] + for t in threads: + t.start() + for t in threads: + t.join() + _, _, histograms = r.snapshot() + assert len(histograms[_key("h", {})]) == N diff --git a/lading_py/tests/test_target_metrics.py b/lading_py/tests/test_target_metrics.py new file mode 100644 index 000000000..f288edddc --- /dev/null +++ b/lading_py/tests/test_target_metrics.py @@ -0,0 +1,175 @@ +"""Tests for Prometheus text parser and Expvar path resolver.""" +import pytest +from lading_py.target_metrics.prometheus import _parse_text, _parse_labels +from lading_py.target_metrics.expvar import _resolve_path + + +# --------------------------------------------------------------------------- +# Prometheus text format parser +# --------------------------------------------------------------------------- + +PROM_SAMPLE = """ +# HELP http_requests_total Total HTTP requests +# TYPE http_requests_total counter +http_requests_total{method="GET",status="200"} 1234 +http_requests_total{method="POST",status="500"} 5 + +# HELP cpu_usage CPU utilisation +# TYPE cpu_usage gauge +cpu_usage 0.73 + +# HELP memory_bytes Memory usage in bytes +# TYPE memory_bytes gauge +memory_bytes{host="web01"} 536870912 + +# TYPE some_histogram histogram +some_histogram_bucket{le="0.1"} 10 +some_histogram_bucket{le="+Inf"} 100 +some_histogram_sum 55.5 +some_histogram_count 100 +""" + + +class TestParseLabels: + def test_empty(self): + assert _parse_labels("") == {} + + def test_single(self): + assert _parse_labels('{env="prod"}') == {"env": "prod"} + + def test_multiple(self): + result = _parse_labels('{a="1",b="2",c="3"}') + assert result == {"a": "1", "b": "2", "c": "3"} + + def test_spaces_ignored(self): + result = _parse_labels('{method="GET", status="200"}') + assert "method" in result + assert "status" in result + + +class TestParseText: + def _by_name(self, results, name): + return [(k, v, l) for n, k, v, l in results if n == name] + + def test_counter_type(self): + results = _parse_text(PROM_SAMPLE) + names_kinds = {n: k for n, k, v, l in results} + assert names_kinds.get("http_requests_total") == "counter" + + def test_gauge_type(self): + results = _parse_text(PROM_SAMPLE) + names_kinds = {n: k for n, k, v, l in results} + assert names_kinds.get("cpu_usage") == "gauge" + + def test_counter_values(self): + results = _parse_text(PROM_SAMPLE) + req_results = [(v, l) for n, k, v, l in results if n == "http_requests_total"] + assert (1234.0, {"method": "GET", "status": "200"}) in req_results + assert (5.0, {"method": "POST", "status": "500"}) in req_results + + def test_gauge_no_labels(self): + results = _parse_text(PROM_SAMPLE) + cpu = [(v, l) for n, k, v, l in results if n == "cpu_usage"] + assert len(cpu) == 1 + assert cpu[0][0] == pytest.approx(0.73) + assert cpu[0][1] == {} + + def test_gauge_with_labels(self): + results = _parse_text(PROM_SAMPLE) + mem = [(v, l) for n, k, v, l in results if n == "memory_bytes"] + assert len(mem) == 1 + assert mem[0][1] == {"host": "web01"} + assert mem[0][0] == 536870912.0 + + def test_histogram_type(self): + results = _parse_text(PROM_SAMPLE) + names_kinds = {n: k for n, k, v, l in results} + assert names_kinds.get("some_histogram_bucket") == "histogram" + + def test_comments_skipped(self): + results = _parse_text("# HELP foo bar\n# TYPE foo gauge\nfoo 1.0\n") + assert len(results) == 1 + assert results[0][0] == "foo" + + def test_empty_input(self): + assert _parse_text("") == [] + + def test_unknown_type_defaults_to_gauge(self): + results = _parse_text("unknown_metric 42.0\n") + assert results[0][1] == "gauge" + + def test_inf_value(self): + results = _parse_text("# TYPE b histogram\nb_bucket{le=\"+Inf\"} 100\n") + vals = [v for _, _, v, _ in results] + assert any(v == 100.0 for v in vals) + + def test_scientific_notation(self): + results = _parse_text("# TYPE m gauge\nm 1.5e3\n") + assert results[0][2] == pytest.approx(1500.0) + + def test_negative_value(self): + results = _parse_text("# TYPE m gauge\nm -3.14\n") + assert results[0][2] == pytest.approx(-3.14) + + def test_multiline_no_crash(self): + lines = ["# TYPE requests counter"] + lines += [f'requests{{path="/api/{i}"}} {i * 10}' for i in range(50)] + results = _parse_text("\n".join(lines)) + assert len(results) == 50 + + +# --------------------------------------------------------------------------- +# Expvar path resolver +# --------------------------------------------------------------------------- + +class TestResolvePath: + def _data(self): + return { + "cmdline": ["agent", "-config", "agent.yaml"], + "uptime": 12345, + "forwarder": { + "Transactions": { + "Success": 99, + "Errors": 3, + }, + "FileStorage": { + "FilesCount": 7, + }, + }, + } + + def test_top_level_numeric(self): + assert _resolve_path(self._data(), "/uptime") == 12345 + + def test_nested_two_levels(self): + assert _resolve_path(self._data(), "/forwarder/FileStorage/FilesCount") == 7 + + def test_nested_three_levels(self): + assert _resolve_path(self._data(), "/forwarder/Transactions/Success") == 99 + + def test_missing_top_level(self): + assert _resolve_path(self._data(), "/nonexistent") is None + + def test_missing_nested(self): + assert _resolve_path(self._data(), "/forwarder/Transactions/Missing") is None + + def test_missing_mid_path(self): + assert _resolve_path(self._data(), "/forwarder/NoSuchKey/Count") is None + + def test_returns_dict(self): + result = _resolve_path(self._data(), "/forwarder/Transactions") + assert isinstance(result, dict) + assert result["Success"] == 99 + + def test_leading_slash_handled(self): + # Both with and without leading slash should work + assert _resolve_path(self._data(), "/uptime") == 12345 + + def test_empty_path(self): + # Empty path should return the whole dict + result = _resolve_path(self._data(), "/") + assert result == self._data() + + def test_path_into_list_returns_none(self): + # /cmdline points to a list; trying to go deeper should return None + assert _resolve_path(self._data(), "/cmdline/0") is None diff --git a/plans/pythonport.md b/plans/pythonport.md new file mode 100644 index 000000000..7284d8f9c --- /dev/null +++ b/plans/pythonport.md @@ -0,0 +1,617 @@ +# Lading Python Port Plan + +Port lading to Python with DogStatsD emission only. All telemetry collection, +reporting, and output functionality must be preserved. + +--- + +## Scope + +**In scope:** +- DogStatsD generator (emit via `dogstatsd-py` library) +- Telemetry collection from Datadog agent (Prometheus scrape + Expvar poll) +- Capture output (JSONL, Parquet, both) +- Prometheus exporter (passive HTTP `/metrics` endpoint) +- Observer (Linux `/proc` sampling) +- Config parsing (same YAML schema as Rust lading) +- Graceful lifecycle (warmup, experiment, shutdown) +- Blackhole (HTTP sink for target output) + +**Out of scope:** +- All non-DogStatsD generators (TCP, UDP, HTTP, Unix stream, Fluent, OTLP, etc.) +- All non-DogStatsD payload types +- Windows support + +--- + +## Reference Files + +| Concern | Rust source | +|---------|-------------| +| DogStatsD payload generation | `lading_payload/src/dogstatsd.rs` | +| Unix datagram transport | `lading/src/generator/unix_datagram.rs` | +| Capture line schema | `lading_capture/src/line.rs` | +| Capture accumulator | `lading_capture/src/accumulator.rs` | +| Prometheus target metrics | `lading/src/target_metrics/prometheus.rs` | +| Expvar target metrics | `lading/src/target_metrics/expvar.rs` | +| Config schema | `lading/src/config.rs` | +| Example config | `lading.yaml` | + +--- + +## Technology Choices + +| Concern | Library | Rationale | +|---------|---------|-----------| +| DogStatsD emission | `dogstatsd-py` (`datadog`) | Required by spec | +| Async runtime | `asyncio` | Standard; matches Tokio concurrency model | +| HTTP client | `aiohttp` | Async Prometheus/Expvar scraping | +| Config parsing | `pydantic` + `PyYAML` | Validated schema, matches Rust serde | +| Prometheus export | `prometheus-client` | Passive `/metrics` scrape endpoint | +| JSONL output | stdlib `json` + `gzip` | Zero dependencies | +| Parquet output | `pyarrow` | Industry standard; schema matches Rust | +| Protobuf (DDSketch) | `protobuf` + datadog proto | Histogram serialization | +| `/proc` parsing | stdlib only (Linux) | Avoids psutil divergence from Rust impl | +| Structured logging | `structlog` | JSON-friendly | + +--- + +## Project Layout + +``` +lading_py/ +├── pyproject.toml +├── lading_py/ +│ ├── __init__.py +│ ├── main.py # Entry point; lifecycle orchestration +│ ├── config.py # Pydantic config models (mirrors lading/src/config.rs) +│ ├── signal.py # asyncio.Event wrappers for lifecycle signals +│ │ +│ ├── generator/ +│ │ ├── __init__.py +│ │ └── dogstatsd.py # DogStatsD generator (uses dogstatsd-py) +│ │ +│ ├── payload/ +│ │ ├── __init__.py +│ │ └── dogstatsd.py # Payload construction (context pool, tag pool) +│ │ +│ ├── blackhole/ +│ │ ├── __init__.py +│ │ └── http.py # HTTP blackhole (aiohttp server) +│ │ +│ ├── target_metrics/ +│ │ ├── __init__.py +│ │ ├── prometheus.py # Prometheus scraper +│ │ └── expvar.py # Expvar poller +│ │ +│ ├── observer/ +│ │ ├── __init__.py +│ │ └── proc.py # /proc/{pid}/ sampler (Linux) +│ │ +│ ├── capture/ +│ │ ├── __init__.py +│ │ ├── line.py # Line dataclass (mirrors lading_capture/src/line.rs) +│ │ ├── accumulator.py # Rolling metric accumulator +│ │ ├── jsonl_writer.py # JSONL output +│ │ └── parquet_writer.py # Parquet output +│ │ +│ └── telemetry/ +│ ├── __init__.py +│ ├── registry.py # Thread-safe counter/gauge/histogram registry +│ └── prometheus_exporter.py # Passive HTTP /metrics endpoint +│ +└── tests/ + ├── test_config.py + ├── test_payload.py + ├── test_capture.py + └── test_target_metrics.py +``` + +--- + +## Step-by-Step Implementation Plan + +### Phase 1: Foundation + +#### Step 1 — Project scaffold + +Create `pyproject.toml` with dependencies, entry point `lading-py`, and Python >= 3.11 requirement. Pin all deps. + +```toml +[project] +name = "lading-py" +requires-python = ">=3.11" +dependencies = [ + "datadog>=0.49", # dogstatsd-py + "pydantic>=2", + "PyYAML>=6", + "aiohttp>=3.9", + "prometheus-client>=0.20", + "pyarrow>=15", + "protobuf>=4", + "structlog>=24", +] + +[project.scripts] +lading-py = "lading_py.main:main" +``` + +#### Step 2 — Config models (`config.py`) + +Pydantic models mirroring Rust config structs. Must parse the same YAML that Rust lading accepts. + +Key models: +- `ConfRange` — `{inclusive: {min, max}}` or `{exclusive: ...}` +- `KindWeights` — `{metric, event, service_check}` +- `MetricWeights` — `{count, gauge, timer, distribution, set, histogram}` +- `DogStatsDConfig` — full dogstatsd variant config +- `UnixDatagramConfig` — transport config (path, bytes_per_second, etc.) +- `GeneratorConfig` — wrapper with optional `id` +- `BlackholeConfig` — HTTP blackhole +- `TargetMetricsConfig` — list of prometheus/expvar entries +- `TelemetryConfig` — log/prometheus/prometheus_socket variant +- `ObserverConfig` +- `RootConfig` — top-level; holds all above + +Validation rules to replicate from Rust: +- `bytes_per_second` parsed from human-readable string ("1 MiB" → 1048576) +- `seed` must be 32 bytes +- `kind_weights` values must not all be zero +- `tag_length.end > MIN_TAG_LENGTH` check (PR #1875) + +#### Step 3 — Signals (`signal.py`) + +```python +class Signals: + experiment_started: asyncio.Event + shutdown: asyncio.Event + target_pid: asyncio.Event # set when target PID is known +``` + +Wraps broadcast-style coordination. All tasks await `experiment_started` before operating, all tasks check `shutdown` to terminate. + +--- + +### Phase 2: DogStatsD Generator + +#### Step 4 — Payload context pool (`payload/dogstatsd.py`) + +This is the most complex piece. Must replicate lading's weighted random generation. + +**Context pool:** +- Pre-generate N contexts (N = `contexts.max`) +- Each context: a fixed `(metric_name, tag_set)` tuple +- Tag sets sampled from tag name/value pools +- `unique_tag_ratio` controls how much tag reuse happens + +**Metric name templates:** +- `name{{0-2}}` expands to `name0`, `name1`, `name2` +- Expand all templates at startup, store as flat list +- Sample uniformly from expanded list + +**Payload generation produces call descriptors, not raw bytes.** All serialization +is delegated to dogstatsd-py at send time. + +```python +@dataclass +class MetricCall: + name: str + value: float + metric_type: str # "gauge" | "count" | "histogram" | "distribution" | "timing" | "set" + tags: list[str] # ["tag1:val1", "tag2:val2"] + sample_rate: float | None + timestamp: int | None # unix seconds, maps to |T field + +@dataclass +class EventCall: + title: str + text: str + tags: list[str] + alert_type: str | None # "error" | "warning" | "info" | "success" + priority: str | None + +@dataclass +class ServiceCheckCall: + name: str + status: int # 0=OK 1=WARNING 2=CRITICAL 3=UNKNOWN + tags: list[str] + message: str | None + +# A "block" is either a single call or a batch (multi-value) +Block = MetricCall | EventCall | ServiceCheckCall | list[MetricCall] + +def generate_block(rng, config, contexts) -> Block: + kind = weighted_choice(rng, config.kind_weights) + if kind == "metric": + count = 1 + if rng.random() < config.multivalue_pack_probability: + count = rng.randint(config.multivalue_count.min, config.multivalue_count.max) + calls = [_gen_metric_call(rng, config, contexts) for _ in range(count)] + return calls if count > 1 else calls[0] + elif kind == "event": + return _gen_event_call(rng, config) + else: + return _gen_service_check_call(rng, config) +``` + +**Multi-value packing:** +- With probability `multivalue_pack_probability`, generate `multivalue_count` metric + calls returned as a `list[MetricCall]`; the generator sends them inside a single + `client.open_buffer()` context so dogstatsd-py packs them into one datagram + +#### Step 5 — Block cache (`payload/dogstatsd.py`) + +Pre-build a cache of `Block` descriptors (call parameter tuples) before the run starts. +`maximum_prebuild_cache_size_bytes` bounds the cache: estimate each `MetricCall` at +~200 bytes of Python object overhead (not wire bytes) for the purposes of capping count. + +```python +class BlockCache: + def __init__(self, config, seed, contexts, max_count): + rng = Random(seed) # deterministic + self._blocks: list[Block] = [] + for _ in range(max_count): + self._blocks.append(generate_block(rng, config, contexts)) + self._idx = 0 + + def next(self) -> Block: + block = self._blocks[self._idx] + self._idx = (self._idx + 1) % len(self._blocks) + return block +``` + +Use `random.Random` with a seed derived from the `seed` config field (32-byte array). +Python's Mersenne Twister differs from Rust's StdRng but exact RNG parity is not +required — statistical properties matter, not bit-for-bit reproducibility. + +#### Step 6 — Generator task (`generator/dogstatsd.py`) + +All emission goes through `dogstatsd-py` (`datadog.dogstatsd.DogStatsd`). No raw socket +fallback. Each `Block` from the cache is dispatched to the appropriate client method. +Multi-value batches use `client.open_buffer()` so dogstatsd-py packs them into one +datagram internally. + +```python +_DISPATCH = { + "gauge": lambda c, m: c.gauge(m.name, m.value, tags=m.tags, sample_rate=m.sample_rate or 1), + "count": lambda c, m: c.increment(m.name, m.value, tags=m.tags, sample_rate=m.sample_rate or 1), + "histogram": lambda c, m: c.histogram(m.name, m.value, tags=m.tags, sample_rate=m.sample_rate or 1), + "distribution": lambda c, m: c.distribution(m.name, m.value, tags=m.tags, sample_rate=m.sample_rate or 1), + "timing": lambda c, m: c.timing(m.name, m.value, tags=m.tags, sample_rate=m.sample_rate or 1), + "set": lambda c, m: c.set(m.name, m.value, tags=m.tags), +} + +def _send_block(client: DogStatsd, block: Block) -> int: + """Send block via dogstatsd-py; return estimated wire bytes for rate limiting.""" + if isinstance(block, list): + # Multi-value batch — pack into one datagram + with client.open_buffer() as buf: + for m in block: + _DISPATCH[m.metric_type](buf, m) + return sum(_estimate_bytes(m) for m in block) + elif isinstance(block, MetricCall): + _DISPATCH[block.metric_type](client, block) + return _estimate_bytes(block) + elif isinstance(block, EventCall): + client.event(block.title, block.text, tags=block.tags, + alert_type=block.alert_type, priority=block.priority) + return _estimate_bytes(block) + else: # ServiceCheckCall + client.service_check(block.name, block.status, tags=block.tags, message=block.message) + return _estimate_bytes(block) + +class DogStatsDGenerator: + async def run(self, signals: Signals): + await signals.experiment_started.wait() + # One DogStatsd client per parallel connection (each has its own socket) + clients = [DogStatsd(socket_path=self.config.path) + for _ in range(self.config.parallel_connections)] + rate_limiter = TokenBucket(self.config.bytes_per_second) + tasks = [ + asyncio.create_task(self._send_loop(client, rate_limiter, signals)) + for client in clients + ] + await asyncio.gather(*tasks) + + async def _send_loop(self, client: DogStatsd, rate_limiter: TokenBucket, signals: Signals): + while not signals.shutdown.is_set(): + block = self.cache.next() + est_bytes = _estimate_bytes_block(block) + await rate_limiter.acquire(est_bytes) + try: + actual = _send_block(client, block) + self.registry.increment("bytes_written", actual) + self.registry.increment("packets_sent", 1) + except Exception as exc: + self.registry.increment("request_failure", 1, {"error": type(exc).__name__}) +``` + +**Limitation:** `length_prefix_framed: true` is unsupported. dogstatsd-py does not +expose length-prefix framing and there is no compliant way to implement it without +bypassing the library. Config validation will reject `length_prefix_framed: true` +with a clear error message. + +**Rate limiter:** Token bucket on estimated wire bytes (`len(name) + len(tags) + ~20`). +Async sleep to yield when bucket is empty. + +--- + +### Phase 3: Telemetry Collection + +#### Step 7 — Prometheus target metrics scraper (`target_metrics/prometheus.py`) + +```python +class PrometheusScraper: + async def run(self, signals: Signals): + await signals.experiment_started.wait() + async with aiohttp.ClientSession() as session: + while not signals.shutdown.is_set(): + text = await session.get(self.config.uri) + metrics = parse_prometheus_text(text) + for m in metrics: + self.registry.record(m.name, m.kind, m.value, m.labels | self.config.tags) + await asyncio.sleep(self.sample_period) +``` + +Prometheus text format parser: parse `# TYPE`, `# HELP`, metric lines. Handle counter, +gauge, histogram, summary. Map to lading's `MetricKind` (Counter/Gauge/Histogram). + +#### Step 8 — Expvar target metrics poller (`target_metrics/expvar.py`) + +```python +class ExpvarPoller: + async def run(self, signals: Signals): + await signals.experiment_started.wait() + async with aiohttp.ClientSession() as session: + while not signals.shutdown.is_set(): + data = await session.get(self.config.uri) # JSON + for var_path in self.config.vars: + value = jsonpath_get(data, var_path) # e.g. "/forwarder/Transactions/Success" + self.registry.record(var_path, MetricKind.Gauge, value, self.config.tags) + await asyncio.sleep(self.sample_period) +``` + +Path resolution: split `/foo/bar/baz` → nested dict lookup `data["foo"]["bar"]["baz"]`. + +--- + +### Phase 4: Observer + +#### Step 9 — `/proc` observer (`observer/proc.py`) + +Linux only. Samples `/proc/{pid}/smaps_rollup` and optionally `/proc/{pid}/smaps` +every `sample_period` seconds after `experiment_started`. + +Key metrics from `smaps_rollup`: +- `Rss` → gauge `smaps_rollup.Rss` +- `Pss` → gauge `smaps_rollup.Pss` +- `Private_Clean`, `Private_Dirty` → gauge +- `Anonymous` → gauge + +Parse format: `FieldName: kB` lines. + +Record all fields as gauges with label `pid=`. + +--- + +### Phase 5: Capture Output + +#### Step 10 — Metric registry (`telemetry/registry.py`) + +Thread-safe in-process registry. All generator/collector/observer code calls into this. + +```python +class Registry: + def increment(self, name: str, value: int, labels: dict): ... + def set_gauge(self, name: str, value: float, labels: dict): ... + def record_histogram(self, name: str, value: float, labels: dict): ... + def snapshot(self) -> list[Line]: ... # drain for flush +``` + +Internal storage: `threading.Lock` guarding dicts of `Counter`, `Gauge`, `DDSketch`. + +#### Step 11 — Line model (`capture/line.py`) + +Mirrors `lading_capture/src/line.rs`: + +```python +@dataclass +class Line: + run_id: str # UUID + time: int # ms since epoch + fetch_index: int # flush counter + metric_name: str + metric_kind: str # "counter" | "gauge" | "histogram" + value: float | int + labels: dict[str, str] + value_histogram: bytes # protobuf DDSketch, empty if not histogram +``` + +#### Step 12 — JSONL writer (`capture/jsonl_writer.py`) + +```python +class JsonlWriter: + def flush(self, lines: list[Line], fetch_index: int): + with open(self.path, "a") as f: + for line in lines: + f.write(json.dumps(dataclasses.asdict(line)) + "\n") +``` + +Flush every `flush_seconds` seconds via `asyncio.sleep` loop. +`value_histogram` bytes field: base64-encode in JSON output (matches Rust behavior). + +#### Step 13 — Parquet writer (`capture/parquet_writer.py`) + +Schema mirrors Rust Parquet output: + +```python +SCHEMA = pa.schema([ + ("run_id", pa.string()), + ("time", pa.int64()), + ("fetch_index", pa.int64()), + ("metric_name", pa.string()), + ("metric_kind", pa.string()), + ("value", pa.float64()), + ("labels", pa.map_(pa.string(), pa.string())), + ("value_histogram", pa.binary()), +]) +``` + +Accumulate rows in memory, flush to Parquet file at `flush_seconds` interval using +`pyarrow.parquet.write_table`. Append row groups (open file in append mode or write +separate files per flush and concatenate on shutdown). + +#### Step 14 — Accumulator (`capture/accumulator.py`) + +60-tick rolling window matching Rust accumulator behavior. + +Tracks per-metric history for computing rates (counters are differenced across ticks). +On each flush tick: +1. Snapshot registry +2. Diff counters vs previous snapshot +3. Pass gauge/histogram values through directly +4. Write `Line` objects to writer(s) + +--- + +### Phase 6: Telemetry Export + +#### Step 15 — Prometheus exporter (`telemetry/prometheus_exporter.py`) + +Passive HTTP endpoint. Uses `prometheus_client` library. + +```python +class PrometheusExporter: + async def run(self, signals: Signals): + # aiohttp handler for GET /metrics + # prometheus_client.generate_latest() for text format + # Periodically syncs from Registry to prometheus_client collectors +``` + +#### Step 16 — Blackhole HTTP (`blackhole/http.py`) + +`aiohttp` server that accepts all POST/PUT requests and discards the body. +Records bytes received as a counter. Binds to configured address. + +--- + +### Phase 7: Lifecycle Orchestration + +#### Step 17 — Main (`main.py`) + +```python +async def inner_main(config: RootConfig): + signals = Signals() + run_id = str(uuid.uuid4()) + + # Build telemetry output + registry = Registry() + writer = build_writer(config.telemetry, run_id) + + # Build and start all components as asyncio tasks + tasks = [] + + if config.generator: + for gen_cfg in config.generator: + dsd_cfg = gen_cfg.unix_datagram.variant.dogstatsd + contexts = build_context_pool(dsd_cfg) + cache = BlockCache(dsd_cfg, gen_cfg.unix_datagram.seed, contexts, max_count=10_000) + tasks.append(asyncio.create_task( + DogStatsDGenerator(gen_cfg, cache, registry).run(signals) + )) + + for bh_cfg in config.blackhole or []: + tasks.append(asyncio.create_task(BlackholeHttp(bh_cfg).run(signals))) + + for tm_cfg in config.target_metrics or []: + tasks.append(asyncio.create_task(build_target_metrics(tm_cfg, registry, signals))) + + if config.observer: + tasks.append(asyncio.create_task(Observer(config.observer, registry).run(signals))) + + tasks.append(asyncio.create_task(accumulate_and_flush(registry, writer, signals))) + + # Lifecycle + await asyncio.sleep(config.warmup_seconds or 0) + signals.experiment_started.set() + await asyncio.sleep(config.experiment_duration_seconds) + signals.shutdown.set() + + await asyncio.gather(*tasks, return_exceptions=True) + writer.finalize() + +def main(): + import argparse, yaml + parser = argparse.ArgumentParser() + parser.add_argument("--config", required=True) + args = parser.parse_args() + with open(args.config) as f: + raw = yaml.safe_load(f) + config = RootConfig.model_validate(raw) + asyncio.run(inner_main(config)) +``` + +Signal handling: install `SIGTERM`/`SIGINT` handler that sets `signals.shutdown`. + +--- + +### Phase 8: Testing + +#### Step 18 — Unit tests + +- `test_config.py`: load `lading.yaml`, verify all fields parse correctly +- `test_payload.py`: generate 1000 messages, verify wire format parses as valid DogStatsD +- `test_capture.py`: write Lines to JSONL and Parquet, read back, verify schema +- `test_target_metrics.py`: mock aiohttp responses, verify Prometheus text parse + +#### Step 19 — Integration smoke test + +`tests/smoke_test.py`: +1. Spin up a UDP socket server (mimicking the agent's DogStatsD socket) +2. Run `lading-py --config tests/smoke.yaml` for 5 seconds +3. Assert bytes received > 0 +4. Assert JSONL output file exists and has lines + +--- + +## Implementation Order + +1. `config.py` — foundation everything else depends on +2. `signal.py` — trivial, needed early +3. `payload/dogstatsd.py` — context pool, block cache +4. `generator/dogstatsd.py` — core feature +5. `telemetry/registry.py` — all metrics recording +6. `capture/line.py` + `capture/jsonl_writer.py` — minimum viable output +7. `main.py` — wire everything together; can test end-to-end here +8. `target_metrics/prometheus.py` + `target_metrics/expvar.py` +9. `observer/proc.py` +10. `capture/parquet_writer.py` + `capture/accumulator.py` +11. `telemetry/prometheus_exporter.py` + `blackhole/http.py` +12. Tests + +--- + +## Key Fidelity Decisions + +| Behavior | Rust | Python | +|----------|------|--------| +| RNG | SeededStdRng (ChaCha) | `random.Random(seed)` | +| RNG parity | Bit-exact reproducibility | Not required; stats parity only | +| Concurrency | Tokio async | asyncio | +| Socket type | Raw Unix datagram | dogstatsd-py exclusively; `open_buffer()` for batches | +| Histogram sketch | DDSketch (protobuf) | Same protobuf schema | +| Time unit | ms (u128) | `int(time.time() * 1000)` | +| Byte sizes | `bytesize` crate parsing | Manual parser: "1 MiB" → 1048576 | +| Config YAML | serde_yaml | PyYAML + pydantic | + +--- + +## Risks and Mitigations + +| Risk | Mitigation | +|------|-----------| +| `length_prefix_framed: true` unsupported by dogstatsd-py | Reject at config validation with clear error; all other wire formats work | +| Python throughput lower than Rust | Pre-built block cache + asyncio avoids per-message allocation; accept perf trade-off | +| DDSketch protobuf schema not public | Extract `.proto` from datadog-agent repo; codegen with `protoc` | +| Prometheus text format edge cases | Use `prometheus_client`'s own parser instead of hand-rolling | +| `/proc` parsing changes across kernel versions | Mirror Rust's exact field-by-field parsing; skip unknown fields | From 8a995ae4fe6f6066db7fc9686da149d243a170d8 Mon Sep 17 00:00:00 2001 From: Stephen Wakely Date: Thu, 11 Jun 2026 11:53:22 +0100 Subject: [PATCH 2/5] Ensure rust isnt being used --- lading/src/bin/lading.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/lading/src/bin/lading.rs b/lading/src/bin/lading.rs index 6161fac55..983103215 100644 --- a/lading/src/bin/lading.rs +++ b/lading/src/bin/lading.rs @@ -1,3 +1,5 @@ +#![allow(unused_imports)] +#![allow(dead_code)] //! Main lading binary for load testing. use std::{ @@ -730,6 +732,9 @@ fn init_tracing(json_output: bool) { } fn main() -> Result<(), Error> { + panic!("Rust is forbidden."); + + /* // Two-parser fallback logic until CliFlatLegacy is removed let (json_output, args) = match CliWithSubcommands::try_parse() { Ok(cli) => match cli.command { @@ -799,6 +804,7 @@ fn main() -> Result<(), Error> { runtime.shutdown_timeout(max_shutdown_delay); info!("Bye. :)"); res + */ } #[cfg(test)] From 5fc947b95ae83abc22efafe78964fda5b9a765bb Mon Sep 17 00:00:00 2001 From: Stephen Wakely Date: Thu, 11 Jun 2026 13:30:18 +0100 Subject: [PATCH 3/5] fix(lading-py): match Rust lading CLI interface Replaces --config (required) with --config-path (default: /etc/lading/lading.yaml) and adds all flags the Rust binary exposes: run/config-check subcommands, --no-target, --target-pid, --capture-path/format/flush-seconds, --prometheus-addr, --experiment-duration-seconds/infinite, --warmup-duration-seconds, --global-labels, and compat stubs for unsupported flags. LADING_CONFIG env var is also honoured. Co-Authored-By: Claude Sonnet 4.6 (1M context) --- lading_py/lading_py/main.py | 202 ++++++++++++++++++++++++++++++++++-- 1 file changed, 193 insertions(+), 9 deletions(-) diff --git a/lading_py/lading_py/main.py b/lading_py/lading_py/main.py index 86b832aa7..1ac355d72 100644 --- a/lading_py/lading_py/main.py +++ b/lading_py/lading_py/main.py @@ -1,11 +1,19 @@ """ lading-py entry point. +CLI is compatible with Rust lading: + lading-py [--config-path PATH] [--no-target] [flags...] + lading-py run [--config-path PATH] [--no-target] [flags...] + lading-py config-check [--config-path PATH] + +Config is also accepted via the LADING_CONFIG environment variable (raw YAML). + Lifecycle: warmup → experiment_started → experiment → shutdown → drain """ import argparse import asyncio +import os import signal import sys import uuid @@ -25,6 +33,159 @@ from lading_py.observer.proc import ProcObserver from lading_py.telemetry.prometheus_exporter import PrometheusExporter +DEFAULT_CONFIG_PATH = "/etc/lading/lading.yaml" + + +# --------------------------------------------------------------------------- +# CLI +# --------------------------------------------------------------------------- + +def _add_run_args(p: argparse.ArgumentParser) -> None: + """Attach all runtime flags to a parser (shared between flat and `run`).""" + p.add_argument( + "--config-path", + default=os.environ.get("LADING_CONFIG_PATH", DEFAULT_CONFIG_PATH), + metavar="PATH", + help=f"path to lading YAML config (default: {DEFAULT_CONFIG_PATH})", + ) + p.add_argument("--global-labels", default=None, metavar="KEY=VAL,...", + help="additional labels applied to all captures") + + # Target group — one is required but lading-py only supports --no-target + # and --target-pid; others are accepted and ignored for compat + tgt = p.add_mutually_exclusive_group(required=False) + tgt.add_argument("--no-target", action="store_true", + help="disable target measurement (default behaviour)") + tgt.add_argument("--target-pid", type=int, default=None, metavar="PID", + help="measure an externally-launched process by PID") + tgt.add_argument("--target-path", default=None, metavar="PATH", + help="(accepted for compat; target execution not supported)") + tgt.add_argument("--target-container", default=None, metavar="NAME", + help="(accepted for compat; container targeting not supported)") + + # Telemetry overrides + p.add_argument("--capture-path", default=None, metavar="PATH", + help="override telemetry output path from config") + p.add_argument("--capture-format", default=None, choices=["jsonl", "parquet", "multi"], + help="override capture format (jsonl|parquet|multi)") + p.add_argument("--capture-flush-seconds", type=int, default=None, metavar="N", + help="override capture flush interval") + p.add_argument("--capture-compression-level", type=int, default=3, metavar="N", + help="parquet compression level 1-22 (default: 3)") + p.add_argument("--capture-expiration-seconds", type=int, default=None, metavar="N", + help="(accepted for compat; not implemented)") + p.add_argument("--prometheus-addr", default=None, metavar="ADDR", + help="override prometheus exporter bind address") + p.add_argument("--prometheus-path", default=None, metavar="PATH", + help="override prometheus exporter socket path") + + # Lifecycle overrides + p.add_argument("--experiment-duration-seconds", type=int, default=None, metavar="N", + help="override experiment duration from config") + p.add_argument("--experiment-duration-infinite", action="store_true", + help="run indefinitely (until SIGTERM/SIGINT)") + p.add_argument("--warmup-duration-seconds", type=int, default=None, metavar="N", + help="override warmup duration from config") + p.add_argument("--max-shutdown-delay", type=int, default=30, metavar="N", + help="maximum seconds to wait for graceful shutdown (default: 30)") + + # Misc + p.add_argument("--disable-inspector", action="store_true", + help="(accepted for compat; inspector not implemented)") + + +def _build_parser() -> argparse.ArgumentParser: + root = argparse.ArgumentParser( + prog="lading-py", + description="lading-py: DogStatsD load generator", + ) + root.add_argument("--json-logs", action="store_true", + help="output logs in JSON format") + + subs = root.add_subparsers(dest="subcommand") + + # `run` subcommand + run_p = subs.add_parser("run", help="run lading with the specified configuration") + _add_run_args(run_p) + + # `config-check` subcommand + check_p = subs.add_parser("config-check", help="validate configuration file and exit") + check_p.add_argument( + "--config-path", + default=os.environ.get("LADING_CONFIG_PATH", DEFAULT_CONFIG_PATH), + metavar="PATH", + help=f"path to lading YAML config (default: {DEFAULT_CONFIG_PATH})", + ) + + # Legacy flat mode (no subcommand) — add run args directly to root + _add_run_args(root) + + return root + + +# --------------------------------------------------------------------------- +# Config loading +# --------------------------------------------------------------------------- + +def _load_raw_config(config_path: str) -> dict: + lading_config_env = os.environ.get("LADING_CONFIG") + if lading_config_env: + return yaml.safe_load(lading_config_env) + with open(config_path) as f: + return yaml.safe_load(f) + + +def _apply_cli_overrides(config: RootConfig, args: argparse.Namespace) -> RootConfig: + """Return a new RootConfig with CLI flag overrides applied.""" + raw = config.model_dump() + + # Experiment / warmup duration + if getattr(args, "experiment_duration_seconds", None) is not None: + raw["experiment_duration_secs"] = args.experiment_duration_seconds + if getattr(args, "warmup_duration_seconds", None) is not None: + raw["warmup_duration_secs"] = args.warmup_duration_seconds + + # Telemetry + capture_path = getattr(args, "capture_path", None) + capture_format = getattr(args, "capture_format", None) + capture_flush = getattr(args, "capture_flush_seconds", None) + prom_addr = getattr(args, "prometheus_addr", None) + prom_path = getattr(args, "prometheus_path", None) + + if any(x is not None for x in (capture_path, capture_format, capture_flush, prom_addr, prom_path)): + tel = raw.get("telemetry") or {} + if capture_path: + tel["path"] = capture_path + if capture_format: + tel.setdefault("log", {})["format"] = {capture_format: { + "flush_seconds": capture_flush or 60 + }} + elif capture_flush and not capture_format: + tel.setdefault("log", {}).setdefault("format", {}).setdefault( + "jsonl", {})["flush_seconds"] = capture_flush + if prom_addr: + tel["prometheus"] = {"addr": prom_addr} + if prom_path: + tel["prometheus_socket"] = {"path": prom_path} + raw["telemetry"] = tel + + # Global labels + global_labels_str = getattr(args, "global_labels", None) + if global_labels_str: + pairs = {} + for token in global_labels_str.split(","): + if "=" in token: + k, _, v = token.partition("=") + pairs[k.strip()] = v.strip() + tel = raw.setdefault("telemetry", {}) + tel["global_labels"] = pairs + + return RootConfig.model_validate(raw) + + +# --------------------------------------------------------------------------- +# Run +# --------------------------------------------------------------------------- def _build_writers(tel: TelemetryConfig | None) -> list: if tel is None or tel.output_path is None: @@ -39,7 +200,7 @@ def _build_writers(tel: TelemetryConfig | None) -> list: return [JsonlWriter(path)] -async def inner_main(config: RootConfig) -> None: +async def inner_main(config: RootConfig, target_pid: int | None = None) -> None: run_id = str(uuid.uuid4()) signals = Signals() registry = Registry() @@ -93,8 +254,7 @@ def _on_signal(): poller = ExpvarPoller(tm.expvar, registry, period_secs) tasks.append(asyncio.create_task(poller.run(signals), name="expvar_poller")) - # Observer (target PID must be provided for proc observer to be useful) - target_pid: int | None = None + # Observer if config.observer and target_pid is not None: obs = ProcObserver(config.observer, registry, period_secs) tasks.append(asyncio.create_task(obs.run(signals, target_pid), name="observer")) @@ -105,7 +265,11 @@ def _on_signal(): signals.experiment_started.set() - await asyncio.sleep(config.experiment_duration_secs) + if config.experiment_duration_secs > 0: + await asyncio.sleep(config.experiment_duration_secs) + else: + # infinite mode — wait for shutdown signal + await signals.shutdown.wait() signals.set_shutdown() @@ -116,15 +280,35 @@ def _on_signal(): def main() -> None: - parser = argparse.ArgumentParser(description="lading-py: DogStatsD load generator") - parser.add_argument("--config", required=True, help="Path to lading YAML config") + parser = _build_parser() args = parser.parse_args() - with open(args.config) as f: - raw = yaml.safe_load(f) + subcommand = getattr(args, "subcommand", None) + # config-check: validate and exit + if subcommand == "config-check": + try: + raw = _load_raw_config(args.config_path) + RootConfig.model_validate(raw) + print(f"Config OK: {args.config_path}") + sys.exit(0) + except Exception as exc: + print(f"Config invalid: {exc}", file=sys.stderr) + sys.exit(1) + + # run or legacy flat mode + config_path = getattr(args, "config_path", DEFAULT_CONFIG_PATH) + raw = _load_raw_config(config_path) config = RootConfig.model_validate(raw) - asyncio.run(inner_main(config)) + config = _apply_cli_overrides(config, args) + + # --experiment-duration-infinite → set duration to 0 (signals infinite loop) + if getattr(args, "experiment_duration_infinite", False): + config = config.model_copy(update={"experiment_duration_secs": 0}) + + target_pid = getattr(args, "target_pid", None) + + asyncio.run(inner_main(config, target_pid=target_pid)) if __name__ == "__main__": From be339ac27db60269a31bd80b730759ed5454fcde Mon Sep 17 00:00:00 2001 From: Stephen Wakely Date: Thu, 11 Jun 2026 14:52:34 +0100 Subject: [PATCH 4/5] fix(lading-py): handle directory config path like Rust lading Co-Authored-By: Claude Sonnet 4.6 (1M context) --- lading_py/lading_py/main.py | 36 +++++++++++++++++++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/lading_py/lading_py/main.py b/lading_py/lading_py/main.py index 1ac355d72..2a93a23f0 100644 --- a/lading_py/lading_py/main.py +++ b/lading_py/lading_py/main.py @@ -127,11 +127,45 @@ def _build_parser() -> argparse.ArgumentParser: # Config loading # --------------------------------------------------------------------------- +_SINGLETON_KEYS = {"telemetry", "sample_period_milliseconds", "inspector", "observer"} +_LIST_KEYS = {"generator", "blackhole", "target_metrics"} + + +def _merge_raw_configs(base: dict, overlay: dict) -> dict: + for key, val in overlay.items(): + if key in _SINGLETON_KEYS: + if key in base: + raise ValueError(f"'{key}' defined in multiple config files") + base[key] = val + elif key in _LIST_KEYS: + base.setdefault(key, []).extend(val if isinstance(val, list) else [val]) + else: + base[key] = val + return base + + def _load_raw_config(config_path: str) -> dict: lading_config_env = os.environ.get("LADING_CONFIG") if lading_config_env: return yaml.safe_load(lading_config_env) - with open(config_path) as f: + p = os.path.abspath(config_path) + if os.path.isdir(p): + yaml_files = sorted( + entry.path + for entry in os.scandir(p) + if entry.is_file() + and entry.name.endswith(".yaml") + and not entry.name.startswith(".") + ) + if not yaml_files: + raise ValueError(f"No .yaml config files found in directory: {p}") + merged: dict = {} + for path in yaml_files: + with open(path) as f: + partial = yaml.safe_load(f) or {} + merged = _merge_raw_configs(merged, partial) + return merged + with open(p) as f: return yaml.safe_load(f) From d38228daeebf5aa0fffa5a207b8f9d4f8596115d Mon Sep 17 00:00:00 2001 From: Stephen Wakely Date: Thu, 11 Jun 2026 17:02:05 +0100 Subject: [PATCH 5/5] fix(lading-py): add backports.zstd for aiohttp zstd content-encoding aiohttp fails at parse time when receiving zstd-compressed requests without this backport of Python 3.14's compression.zstd module. Co-Authored-By: Claude Sonnet 4.6 (1M context) --- lading_py/pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/lading_py/pyproject.toml b/lading_py/pyproject.toml index 34f9c6750..523cbbc0f 100644 --- a/lading_py/pyproject.toml +++ b/lading_py/pyproject.toml @@ -11,6 +11,7 @@ dependencies = [ "pydantic>=2", "PyYAML>=6", "aiohttp>=3.9", + "backports.zstd>=1.5", "prometheus-client>=0.20", "pyarrow>=15", "structlog>=24",