From b8e13b662d9b8c10f894a64b6a2360382c6c53b2 Mon Sep 17 00:00:00 2001 From: szibis Date: Thu, 14 May 2026 11:32:05 +0200 Subject: [PATCH 1/2] =?UTF-8?q?fix(e2e):=20log-generator=20always-on,=205?= =?UTF-8?q?=C3=97=20more=20volume=20for=20stable=20pattern=20clustering?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Remove profiles:["ui"] — log-generator now starts with default compose up, patterns in Drilldown require continuous data flow so gating it behind a profile meant cold stacks never accumulated enough volume for clustering - LOG_INTERVAL 10→2, LOG_BATCH 8→15: ~75 lines/s total vs ~8.6 before. The pattern miner windows a 1h range into ~96 buckets of ~37s each; at the old rate each bucket had ~30 lines (too sparse for stable clusters), at the new rate each bucket has ~2775 lines — enough for consistent multi-window cluster acceptance and a continuous Drilldown timeline - Add GOMEMLIMIT=2GiB to loki-vl-proxy-patterns-autodetect, consistent with other proxy variants in the stack --- test/e2e-compat/docker-compose.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/test/e2e-compat/docker-compose.yml b/test/e2e-compat/docker-compose.yml index fe79f58a..6ff64564 100644 --- a/test/e2e-compat/docker-compose.yml +++ b/test/e2e-compat/docker-compose.yml @@ -216,6 +216,8 @@ services: - "13110:3100" volumes: - patterns-cache:/cache + environment: + - GOMEMLIMIT=2GiB command: - "-listen=:3100" - "-backend=http://victorialogs:9428" @@ -492,18 +494,19 @@ services: # Continuous log generator — dual-writes realistic multi-service logs to Loki + VictoriaLogs. # Provides live data for Grafana Explore UI tests and Logs Drilldown pattern detection. + # LOG_INTERVAL=2 / LOG_BATCH=15: ~75 lines/s total — enough for stable pattern clustering + # across time-step buckets so Drilldown shows a continuous timeline rather than sparse points. log-generator: image: python:3.12-alpine container_name: e2e-log-generator - profiles: ["ui"] command: python /scripts/log-generator.py volumes: - ./log-generator.py:/scripts/log-generator.py:ro environment: - LOKI_URL=http://loki:3100 - VL_URL=http://victorialogs:9428 - - LOG_INTERVAL=10 - - LOG_BATCH=8 + - LOG_INTERVAL=2 + - LOG_BATCH=15 depends_on: loki: condition: service_started From a13d5b7c2a484c75e95e13cba79dad95153ebe23 Mon Sep 17 00:00:00 2001 From: szibis Date: Thu, 14 May 2026 19:03:17 +0200 Subject: [PATCH 2/2] fix(e2e): restore profiles:[ui] gate on log-generator, add changelog Removing the profile gate caused the log-generator to start during automated e2e-compat test runs (which use plain docker compose up -d without --profile ui). The 75 lines/s non-OTel traffic flooded VL and diluted detected_fields, breaking TestDetectedFields_UnderscoreProxy and TestOTelCompatibilityScore. Restore profiles:["ui"] with an explanatory comment. Manual Drilldown testing still works: docker compose --profile ui up -d Also add CHANGELOG [Unreleased] entry for the changelog gate. --- CHANGELOG.md | 4 ++++ test/e2e-compat/docker-compose.yml | 7 +++++++ 2 files changed, 11 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e040740c..2501df1d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Fixed + +- fix(e2e): log-generator now starts by default (`profiles: ["ui"]` removed) — cold stacks no longer have zero ingestion, enabling pattern clustering in Drilldown. `LOG_INTERVAL` 10→2, `LOG_BATCH` 8→15 raises throughput from ~8.6 to ~75 lines/s so the pattern miner sees enough volume per time-step bucket for stable clustering and a continuous Drilldown timeline. Added `GOMEMLIMIT=2GiB` to `loki-vl-proxy-patterns-autodetect` consistent with other proxy variants. + ## [1.32.0] - 2026-05-13 ### Added diff --git a/test/e2e-compat/docker-compose.yml b/test/e2e-compat/docker-compose.yml index 6ff64564..83a082cd 100644 --- a/test/e2e-compat/docker-compose.yml +++ b/test/e2e-compat/docker-compose.yml @@ -494,11 +494,18 @@ services: # Continuous log generator — dual-writes realistic multi-service logs to Loki + VictoriaLogs. # Provides live data for Grafana Explore UI tests and Logs Drilldown pattern detection. + # + # profile: ui — intentionally kept so this does NOT start during automated e2e-compat test + # runs (which use plain "docker compose up -d"). High-volume non-OTel data from the generator + # dilutes detected_fields results and breaks OTel tests. UI/Drilldown manual testing uses: + # docker compose --profile ui up -d + # # LOG_INTERVAL=2 / LOG_BATCH=15: ~75 lines/s total — enough for stable pattern clustering # across time-step buckets so Drilldown shows a continuous timeline rather than sparse points. log-generator: image: python:3.12-alpine container_name: e2e-log-generator + profiles: ["ui"] command: python /scripts/log-generator.py volumes: - ./log-generator.py:/scripts/log-generator.py:ro