refactor: simplified the framework

Signed-off-by: bwplotka <bwplotka@gmail.com>

Author: bwplotka

.github/workflows/compliance-tests.yml

@@ -1,21 +1,21 @@
-name: Compliance Tests
-
+---
+name: Compliance
 on:
   pull_request:
+  push:
+    branches: [main]
+
+permissions:
+  contents: read
 
 jobs:
-  remotewrite-compliance:
+  remotewrite-sender:
+    name: Remote Write Sender Compliance Tests
     runs-on: ubuntu-latest
-
+    container:
+      image: quay.io/prometheus/golang-builder:1.25-base
     steps:
-    - uses: actions/checkout@v4
-
-    - name: Set up Go
-      uses: actions/setup-go@v4
-      with:
-        go-version: '1.22'
-
-    - name: Run Remote Write Prometheus Compliance Tests
-      run: |
-        cd remotewrite/sender
-        go test --tags=compliance -run "TestRemoteWrite/prometheus/.+" -v ./
+      - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
+        with:
+          persist-credentials: false
+      - run: cd remotewrite && make sender

remotewrite/.gitignore

@@ -0,0 +1,4 @@
+results.json
+config.yml
+index.preloaded.html
+bin/

remotewrite/Makefile

@@ -1,7 +1,20 @@
 # Set the default target to 'help' so it runs when you just type 'make'
 .DEFAULT_GOAL := help
 
-DEBUG=? ""
+mkfile_path := $(abspath $(lastword $(MAKEFILE_LIST)))
+current_dir := $(notdir $(patsubst %/,%,$(dir $(mkfile_path))))
+ROOT_DIR := $(dir $(mkfile_path))
+
+PROMETHEUS_RW_COMPLIANCE_TEST_RE=""
+PROMETHEUS_RW_COMPLIANCE_SKIP_TEST_RE=""
+TIMEOUT="10m"
+SED=$(shell which gsed 2>/dev/null || which sed)
+RESULT_FILE="$(ROOT_DIR)/results.json"
+
+## Common vars.
+DEBUG=""
+## Sender vars.
+PROMETHEUS_COMPLIANCE_RW_SENDERS="prometheus"
 
 .PHONY: help
 help: ## Show this help message
@@ -10,14 +23,30 @@ help: ## Show this help message
 	@echo "Available commands:"
 	@awk 'BEGIN {FS = ":.*?## "} /^[a-zA-Z_-]+:.*?## / {printf "  \033[36m%-15s\033[0m %s\n", $$1, $$2}' $(MAKEFILE_LIST)
 
-PROMETHEUS_RW2_COMPLIANCE_SENDERS =? "prometheus"
 
 .PHONY: sender
-sender: ## Run sender compliance tests for a given PROMETHEUS_RW2_COMPLIANCE_SENDERS with a HTML results view.
-	cd ./sender && DEBUG=$(DEBUG) PROMETHEUS_RW2_COMPLIANCE_SENDERS="$(PROMETHEUS_RW2_COMPLIANCE_SENDERS)" go test -json | tee results.json
-	@echo "Opening HTML result view; load the `results.json` file"
-	@open ./index.html
-
-.PHONY: sender-check
-sender-ci: ## Run sender compliance tests for a given PROMETHEUS_RW2_COMPLIANCE_SENDERS suitable for CI.
-	cd ./sender && DEBUG=$(DEBUG) PROMETHEUS_RW2_COMPLIANCE_SENDERS="$(PROMETHEUS_RW2_COMPLIANCE_SENDERS)" go test -v
+sender: ## Run sender compliance tests for a given senders suitable for humans and CI.
+	cd ./sender && DEBUG=$(DEBUG) PROMETHEUS_COMPLIANCE_RW_SENDERS=$(PROMETHEUS_COMPLIANCE_RW_SENDERS) go test -run $(PROMETHEUS_RW_COMPLIANCE_TEST_RE) -skip $(PROMETHEUS_RW_COMPLIANCE_SKIP_TEST_RE) -v -timeout=$(TIMEOUT)
+
+.PHONY: sender-html
+sender-html: ## Run sender compliance tests for a given senders with a HTML results view.
+	@cd ./sender && DEBUG=$(DEBUG) PROMETHEUS_COMPLIANCE_RW_SENDERS=$(PROMETHEUS_COMPLIANCE_RW_SENDERS) go test -run $(PROMETHEUS_RW_COMPLIANCE_TEST_RE) -skip $(PROMETHEUS_RW_COMPLIANCE_SKIP_TEST_RE) -json | tee $(RESULT_FILE)
+	@$(MAKE) open-preloaded-results
+
+PROMETHEUS_RW2_COMPLIANCE_RECEIVERS="local-prometheus"
+
+.PHONY: receiver
+receiver: ## Run receiver compliance tests for a given receivers suitable for humans and CI.
+	cd ./receiver && DEBUG=$(DEBUG) PROMETHEUS_RW2_COMPLIANCE_RECEIVERS=$(PROMETHEUS_RW2_COMPLIANCE_RECEIVERS) go test -run $(PROMETHEUS_RW_COMPLIANCE_TEST_RE) -skip $(PROMETHEUS_RW_COMPLIANCE_SKIP_TEST_RE) -v -timeout=$(TIMEOUT)
+
+.PHONY: receiver-html
+receiver-html: ## Run receiver compliance tests for a given receivers with a HTML results view.
+	@cd ./receiver && DEBUG=$(DEBUG) PROMETHEUS_RW2_COMPLIANCE_RECEIVERS=$(PROMETHEUS_RW2_COMPLIANCE_RECEIVERS) go test -run $(PROMETHEUS_RW_COMPLIANCE_TEST_RE) -skip $(PROMETHEUS_RW_COMPLIANCE_SKIP_TEST_RE) -json | tee $(RESULT_FILE)
+	@$(MAKE) open-preloaded-results
+
+
+.PHONY: open-preloaded-results
+open-preloaded-results: # Open index.html with the preloaded results from results.json file.
+	@# Preload to avoid CORS issue on secure browsers.
+	@cat index.html | perl -pe 'BEGIN{local $$/; open(F,"<results.json"); $$c=<F>; close F} s/const PRELOADED_RESULT_DATA = ``;/const PRELOADED_RESULT_DATA = `$$c`;/' > index.preloaded.html
+	@open index.preloaded.html

remotewrite/README.md

@@ -7,35 +7,39 @@ Both [sender](#sender-compliance-sender) and [receiver](#receiver-compliance-rec
 To generate and view results, add `-json` flag to `go test` command and `tee` the results to a file called `results.json`.
 
 Tests are marked with compliance levels according to RFC specifications:
-- **MUST**: Required by specification
-- **SHOULD**: Recommended by specification
-- **MAY**: Could have by specification
-- **RECOMMENDED**: Not in specification but recommended for performance
+- **MUST**: Required by Remote Write specification
+- **SHOULD**: Recommended by Remote Write specification
+- **MAY**: Could have by Remote Write specification
+- **RECOMMENDED**: Not in Remote Write specification, but recommended for performance or Prometheus compatibility reasons
 
 ## Sender Compliance (`/sender`)
 
-Tests that Remote-Write senders properly implement the protocol by forking sender instances (e.g., Prometheus), examining generated requests, and validating them against the specification.
+Tests that Remote-Write senders implement the RW2 protocol by running tests cases with:
+
+1. Scraper exposing OpenMetrics 1.0
+2. Target sender to test (e.g., Prometheus),
+3. Special receiver that tests various elements of the request(s).
 
 Tests cover:
+- Series encoding
 - Float samples encoding
 - Native Histograms encoding
 - Exemplars encoding
-- Protocol headers and content negotiation
-- Error handling and retry logic
-- Backoff and batching behavior
-- Metadata and symbol table management
+- Metadata encoding
+- Protocol headers
+- Error handling, basic batching and retry logic
 - Request formatting and compression
-
-### Limitations
-
-The test suite validates the format and structure of requests sent by the sender but does not verify end-to-end data flow or persistence. Because senders may have different configuration options and capabilities, passing all tests does not guarantee a sender supports every Remote-Write feature (such as Native Histograms). Some senders may not expose certain features or may require specific configuration.
+- Various Prometheus / OpenMetrics 1.0 semantics as "RECOMMENDED" (staleness, upness). Because this test depends on scraping behaviour we
+are testing some elements of the exposition format support too.
 
 ### Prerequisites
 
 - Go 1.23 or later
-- The sender binary to test (e.g., Prometheus)
+- The sender binary you want to test (e.g., Prometheus)
+  - Requires OpenMetrics 1.0 scrape capability.
+  - Required Remote Write sending capability.
 
-The test suite automatically downloads and runs Prometheus as the reference sender implementation. For testing custom senders, place the binary in the `bin/` directory.
+The test suite automatically downloads and runs Prometheus as the reference sender implementation.
 
 ### Configuration
 
@@ -47,6 +51,15 @@ The test suite uses environment variables:
 PROMETHEUS_RW2_COMPLIANCE_SENDERS="prometheus"
 ```
 
+Currently supported senders:
+- `prometheus` - The reference Prometheus implementation (automatically downloaded).
+- `process`  - For custom sender that is a local binary.
+
+For testing custom senders:
+
+* Add target running code and register it the `sender/main_test.go`.
+* Use custom process target via `PROMETHEUS_COMPLIANCE_RW_SENDERS="process"` and `PROMETHEUS_COMPLIANCE_RW_PROCESS_BINARY=<path>` envvars.
+
 **Debug output:**
 
 Debug variable controls if the tested process suppresses output (empty DEBUG) or not. 
@@ -55,28 +68,20 @@ Debug variable controls if the tested process suppresses output (empty DEBUG) or
 DEBUG="1"
 ```
 
-Currently supported senders:
-- `prometheus` - The reference Prometheus implementation (automatically downloaded)
-
 ### Running Tests
 
 ```bash
-pushd ./sender
-# Run compliance tests e.g. for the default prometheus binary ...
-PROMETHEUS_RW2_COMPLIANCE_SENDERS="prometheus" go test -v
-popd
+make sender
 ```
 
-To use visualization HTML page, export data to JSON and generate `results.json` e.g.
+To use visualization HTML page:
 
 ```bash
-pushd ./sender
-PROMETHEUS_RW2_COMPLIANCE_SENDERS="prometheus" go test -json | tee results.json
-popd
-
-# Open ./index.html in your browser and see results.
+make sender-html
 ```
 
+See Makefile for detailed invocation.
+
 ## Receiver Compliance (`/receiver`)
 
 Tests that Remote-Write endpoints properly handle incoming requests by sending various Remote-Write v2 requests and validating responses.
@@ -136,21 +141,14 @@ export PROMETHEUS_RW2_COMPLIANCE_RECEIVERS="local-prometheus,mimir"
 
 ### Running Tests
 
-Simply run `go test` in the `receiver` directory. For example:
-
 ```bash
-pushd ./receiver
-# Run compliance tests e.g. for the local-prometheus remote write endpoint defined in your PROMETHEUS_RW2_COMPLIANCE_CONFIG_FILE.
-PROMETHEUS_RW2_COMPLIANCE_RECEIVERS="local-prometheus" go test -v
-popd
+make receiver
 ```
 
-To use visualization HTML page, export data to JSON and generate `results.json` e.g.
+To use visualization HTML page:
 
 ```bash
-pushd ./receiver
-PROMETHEUS_RW2_COMPLIANCE_RECEIVERS="local-prometheus" go test -json | tee results.json
-popd
-
-# Open ./index.html in your browser and see results.
+make receiver-html
 ```
+
+See Makefile for detailed invocation.