Add compatibility test framework

Makefile

@@ -189,3 +189,27 @@ stop-dependencies:
 .PHONY: nuke-dependencies
 nuke-dependencies:
 	$(DOCKER_COMPOSE) down --timeout 60 --volumes --remove-orphans
+
+# Compatibility tests
+COMPAT_DIR     = tests/compatibility
+COMPAT_PARALLEL ?= 1
+
+.PHONY: test-compatibility
+test-compatibility: test-compatibility-pull
+	cd $(COMPAT_DIR) && GOFLAGS="" go test \
+		-tags testcompatibility \
+		-timeout 60m \
+		-parallel $(COMPAT_PARALLEL) \
+		./matrix/ 
+
+.PHONY: test-compatibility-pull
+test-compatibility-pull:
+	DOCKER_CLI_HINTS=false docker pull -q temporalio/auto-setup:1.29.2
+	DOCKER_CLI_HINTS=false docker pull -q temporalio/server:1.27.4
+	DOCKER_CLI_HINTS=false docker pull -q temporalio/admin-tools:1.27
+	DOCKER_CLI_HINTS=false docker pull -q postgres:15
+	DOCKER_CLI_HINTS=false docker pull -q postgres:12
+
+.PHONY: test-compatibility-tidy
+test-compatibility-tidy:
+	cd $(COMPAT_DIR) && GOFLAGS="" go mod tidy

tests/compatibility/go.mod

@@ -0,0 +1,69 @@
+module github.com/temporalio/s2s-proxy/tests/compatibility
+
+go 1.25.0
+
+require (
+	github.com/docker/go-connections v0.5.0
+	github.com/stretchr/testify v1.11.1
+	github.com/testcontainers/testcontainers-go v0.36.0
+	go.temporal.io/api v1.46.0
+	go.temporal.io/server v1.27.4
+	google.golang.org/grpc v1.72.2
+	google.golang.org/protobuf v1.36.8
+	gopkg.in/yaml.v3 v3.0.1
+)
+
+require (
+	dario.cat/mergo v1.0.1 // indirect
+	github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect
+	github.com/Microsoft/go-winio v0.6.2 // indirect
+	github.com/cenkalti/backoff/v4 v4.3.0 // indirect
+	github.com/containerd/log v0.1.0 // indirect
+	github.com/containerd/platforms v0.2.1 // indirect
+	github.com/cpuguy83/dockercfg v0.3.2 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/distribution/reference v0.6.0 // indirect
+	github.com/docker/docker v28.0.1+incompatible // indirect
+	github.com/docker/go-units v0.5.0 // indirect
+	github.com/ebitengine/purego v0.8.2 // indirect
+	github.com/felixge/httpsnoop v1.0.4 // indirect
+	github.com/go-logr/logr v1.4.2 // indirect
+	github.com/go-logr/stdr v1.2.2 // indirect
+	github.com/go-ole/go-ole v1.2.6 // indirect
+	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/google/uuid v1.6.0 // indirect
+	github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.3 // indirect
+	github.com/klauspost/compress v1.18.0 // indirect
+	github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect
+	github.com/magiconair/properties v1.8.9 // indirect
+	github.com/moby/docker-image-spec v1.3.1 // indirect
+	github.com/moby/patternmatcher v0.6.0 // indirect
+	github.com/moby/sys/sequential v0.5.0 // indirect
+	github.com/moby/sys/user v0.1.0 // indirect
+	github.com/moby/sys/userns v0.1.0 // indirect
+	github.com/moby/term v0.5.0 // indirect
+	github.com/morikuni/aec v1.0.0 // indirect
+	github.com/opencontainers/go-digest v1.0.0 // indirect
+	github.com/opencontainers/image-spec v1.1.1 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect
+	github.com/shirou/gopsutil/v4 v4.25.1 // indirect
+	github.com/sirupsen/logrus v1.9.3 // indirect
+	github.com/tklauser/go-sysconf v0.3.12 // indirect
+	github.com/tklauser/numcpus v0.6.1 // indirect
+	github.com/yusufpapurcu/wmi v1.2.4 // indirect
+	go.opentelemetry.io/auto/sdk v1.1.0 // indirect
+	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0 // indirect
+	go.opentelemetry.io/otel v1.35.0 // indirect
+	go.opentelemetry.io/otel/metric v1.35.0 // indirect
+	go.opentelemetry.io/otel/trace v1.35.0 // indirect
+	golang.org/x/crypto v0.36.0 // indirect
+	golang.org/x/net v0.37.0 // indirect
+	golang.org/x/sys v0.31.0 // indirect
+	golang.org/x/text v0.23.0 // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20250324211829-b45e905df463 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20250324211829-b45e905df463 // indirect
+)
+
+replace github.com/temporalio/s2s-proxy => ../..

tests/compatibility/matrix/matrix_test.go

@@ -0,0 +1,36 @@
+//go:build testcompatibility
+
+package matrix
+
+import (
+	"testing"
+
+	"github.com/temporalio/s2s-proxy/tests/compatibility/specifications"
+	"github.com/temporalio/s2s-proxy/tests/compatibility/specifications/temporal_1_27_4_postgres12"
+	"github.com/temporalio/s2s-proxy/tests/compatibility/specifications/temporal_1_29_2_postgres15"
+)
+
+func TestTopology_1_29_2_same(t *testing.T) {
+	t.Parallel()
+	run(t, []specifications.ClusterSpec{
+		temporal_1_29_2_postgres15.New(),
+		temporal_1_29_2_postgres15.New(),
+	})
+}
+
+func TestTopology_1_29_2_triplet(t *testing.T) {
+	t.Parallel()
+	run(t, []specifications.ClusterSpec{
+		temporal_1_29_2_postgres15.New(),
+		temporal_1_29_2_postgres15.New(),
+		temporal_1_29_2_postgres15.New(),
+	})
+}
+
+func TestTopology_1_29_2_vs_1_27_4(t *testing.T) {
+	t.Parallel()
+	run(t, []specifications.ClusterSpec{
+		temporal_1_29_2_postgres15.New(),
+		temporal_1_27_4_postgres12.New(),
+	})
+}

tests/compatibility/matrix/run.go

@@ -0,0 +1,63 @@
+//go:build testcompatibility
+
+package matrix
+
+import (
+	"fmt"
+	"math/rand"
+	"testing"
+
+	"github.com/temporalio/s2s-proxy/tests/compatibility/specifications"
+	"github.com/temporalio/s2s-proxy/tests/compatibility/suite"
+	"github.com/temporalio/s2s-proxy/tests/compatibility/topology"
+)
+
+// run runs the full test suite for a single topology of cluster specs.
+func run(t *testing.T, specs []specifications.ClusterSpec) {
+	t.Helper()
+
+	top := topology.NewTopology(t, newTopologyID(), specs)
+
+	healthPassed := t.Run("Health", func(t *testing.T) {
+		for _, cluster := range top.Clusters() {
+			if !t.Run(cluster.Name(), func(t *testing.T) { suite.RunClusterHealthSuite(t, top, cluster) }) {
+				return
+			}
+		}
+
+		for _, proxy := range top.Proxies() {
+			if !t.Run(proxy.Name(), func(t *testing.T) { suite.RunProxyHealthSuite(t, top, proxy) }) {
+				return
+			}
+		}
+	})
+
+	// If health checks fail, we skip the rest of the tests to avoid misleading failures.
+	// If health / Proxy tests fail, this is likely indication of cluster configuration issues.
+	if !healthPassed {
+		return
+	}
+
+	t.Run("Connectivity", func(t *testing.T) {
+		for _, proxy := range top.Proxies() {
+			t.Run(proxy.Name(), func(t *testing.T) {
+				suite.RunConnectivitySuite(t, top, proxy)
+			})
+		}
+	})
+
+	t.Run("Replication", func(t *testing.T) {
+		for _, cluster := range top.Clusters() {
+			t.Run(cluster.Name(), func(t *testing.T) {
+				suite.RunReplicationSuite(t, top, cluster)
+			})
+		}
+	})
+}
+
+// newTopologyID generates a random topology ID for test isolation.
+// Used to separate parallel topology runs in the global Docker runtime.
+// For example, to ensure that the "cluster-a" in "topology-1" and "topology-2" are isolated.
+func newTopologyID() string {
+	return fmt.Sprintf("%08x", rand.Uint32()) //nolint:gosec
+}

tests/compatibility/specifications/temporal_1_27_4_postgres12/config.go

@@ -0,0 +1,29 @@
+//go:build testcompatibility
+
+package temporal_1_27_4_postgres12
+
+import (
+	_ "embed"
+
+	"github.com/temporalio/s2s-proxy/tests/compatibility/specifications"
+)
+
+//go:embed server_config.yaml
+var serverConfigYAML string
+
+//go:embed server_setup_schema.sh
+var serverSetupSchemaTemplate string
+
+func New() specifications.ClusterSpec {
+	return specifications.ClusterSpec{
+		Server: specifications.ServerSpec{
+			Image:               "temporalio/server:1.27.4",
+			AdminToolsImage:     "temporalio/admin-tools:1.27",
+			ConfigTemplate:      serverConfigYAML,
+			SetupSchemaTemplate: serverSetupSchemaTemplate,
+		},
+		Database: specifications.DatabaseSpec{
+			Image: "postgres:12",
+		},
+	}
+}

tests/compatibility/specifications/temporal_1_27_4_postgres12/server_config.yaml

@@ -0,0 +1,91 @@
+log:
+  stdout: true
+  level: "warn"
+
+persistence:
+  defaultStore: default
+  visibilityStore: visibility
+  numHistoryShards: 1
+  datastores:
+    default:
+      sql:
+        pluginName: "postgres12"
+        databaseName: "temporal"
+        connectAddr: "[[.DatabaseAlias]]:5432"
+        connectProtocol: "tcp"
+        user: "temporal"
+        password: "temporal"
+        maxConns: 20
+        maxIdleConns: 20
+        maxConnLifetime: "1h"
+    visibility:
+      sql:
+        pluginName: "postgres12"
+        databaseName: "temporal_visibility"
+        connectAddr: "[[.DatabaseAlias]]:5432"
+        connectProtocol: "tcp"
+        user: "temporal"
+        password: "temporal"
+        maxConns: 10
+        maxIdleConns: 10
+        maxConnLifetime: "1h"
+
+global:
+  membership:
+    maxJoinDuration: 30s
+    broadcastAddress: "{{ default .Env.TEMPORAL_BROADCAST_ADDRESS "" }}"
+
+services:
+  frontend:
+    rpc:
+      grpcPort: 7233
+      membershipPort: 6933
+      bindOnIP: "0.0.0.0"
+      httpPort: 7243
+  matching:
+    rpc:
+      grpcPort: 7235
+      membershipPort: 6935
+      bindOnIP: "0.0.0.0"
+  history:
+    rpc:
+      grpcPort: 7234
+      membershipPort: 6934
+      bindOnIP: "0.0.0.0"
+  worker:
+    rpc:
+      grpcPort: 7239
+      membershipPort: 6939
+      bindOnIP: "0.0.0.0"
+
+clusterMetadata:
+  enableGlobalNamespace: true
+  failoverVersionIncrement: 10
+  masterClusterName: "[[.ClusterName]]"
+  currentClusterName: "[[.ClusterName]]"
+  clusterInformation:
+    [[.ClusterName]]:
+      enabled: true
+      initialFailoverVersion: [[.InitialFailoverVersion]]
+      rpcName: "frontend"
+      rpcAddress: "[[.NetworkAlias]]:7233"
+
+dcRedirectionPolicy:
+  policy: "all-apis-forwarding"
+
+dynamicConfigClient:
+  filepath: "config/dynamicconfig/docker.yaml"
+  pollInterval: "10s"
+
+archival:
+  history:
+    state: "disabled"
+  visibility:
+    state: "disabled"
+
+namespaceDefaults:
+  archival:
+    history:
+      state: "disabled"
+    visibility:
+      state: "disabled"

tests/compatibility/specifications/temporal_1_27_4_postgres12/server_setup_schema.sh

@@ -0,0 +1,12 @@
+set -e
+export SQL_PASSWORD=temporal
+T="temporal-sql-tool --plugin postgres12 --ep [[.DatabaseAlias]] -u temporal -p 5432"
+
+$T --db temporal       setup-schema -v 0.0
+$T --db temporal       update-schema -d /etc/temporal/schema/postgresql/v12/temporal/versioned
+
+$T --db temporal_visibility create
+$T --db temporal_visibility setup-schema -v 0.0
+$T --db temporal_visibility update-schema -d /etc/temporal/schema/postgresql/v12/visibility/versioned
+
+echo "temporal schema setup complete"

tests/compatibility/specifications/temporal_1_29_2_postgres15/config.go

@@ -0,0 +1,29 @@
+//go:build testcompatibility
+
+package temporal_1_29_2_postgres15
+
+import (
+	_ "embed"
+
+	"github.com/temporalio/s2s-proxy/tests/compatibility/specifications"
+)
+
+//go:embed server_config.yaml
+var serverConfigTemplate string
+
+//go:embed server_setup_schema.sh
+var setupSchemaTemplate string
+
+func New() specifications.ClusterSpec {
+	return specifications.ClusterSpec{
+		Server: specifications.ServerSpec{
+			Image:               "temporalio/server:1.29.2",
+			AdminToolsImage:     "temporalio/admin-tools:1.29",
+			ConfigTemplate:      serverConfigTemplate,
+			SetupSchemaTemplate: setupSchemaTemplate,
+		},
+		Database: specifications.DatabaseSpec{
+			Image: "postgres:15",
+		},
+	}
+}