[DX-2789] Observability stack updated (#2401)

Tofel · web-flow · commit dc70cecc1596 · 2026-03-12T08:21:33.000+01:00
* Observability stack: update Tempo version to latest, enable traces-to-profiles, run Pyroscope with bigger and higher resolution profiles
diff --git a/.github/workflows/framework-cli-tests.yml b/.github/workflows/framework-cli-tests.yml
@@ -21,13 +21,16 @@ jobs:
       - name: Configure AWS credentials using OIDC
         uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # v4.0.2
         with:
-          role-to-assume: ${{ secrets.PUBLIC_AWS_ECR_ROLE }}
-          aws-region: us-east-1
-      - name: Authenticate to ECR Public
-        id: login-ecr-public
+          role-to-assume: ${{ secrets.AWS_CTF_READ_ACCESS_ROLE_ARN }}
+          aws-region: us-west-2
+
+      - name: Login to Amazon ECR
+        id: login-ecr-private
         uses: aws-actions/amazon-ecr-login@062b18b96a7aff071d4dc91bc00c4c1a7945b076 # v2.0.1
         with:
-          registry-type: public
+          registries: ${{ format('{0},{1}', secrets.AWS_ACCOUNT_ID_SDLC, secrets.AWS_ACCOUNT_ID_PROD) }}
+        env:
+          AWS_REGION: us-west-2
       - name: Check for changes in Framework
         uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2
         id: changes
diff --git a/.github/workflows/framework-golden-tests-private.yml b/.github/workflows/framework-golden-tests-private.yml
@@ -10,6 +10,8 @@ jobs:
         working-directory: framework/examples/myproject
     env:
       CTF_JD_IMAGE: "${{secrets.AWS_ACCOUNT_ID_PROD}}.dkr.ecr.us-west-2.amazonaws.com/job-distributor:0.22.1"
+      # TODO: once a public image with support for field [Pyrscope] LinkTracesToProfiles is available, use it instead of the private image.
+      CTF_CHAINLINK_IMAGE: "${{secrets.AWS_ACCOUNT_ID_SDLC}}.dkr.ecr.us-west-2.amazonaws.com/chainlink:manual-9b82dc2"
     runs-on: ubuntu-latest
     permissions:
       id-token: write
@@ -22,19 +24,35 @@ jobs:
             config: jd.toml
             count: 1
             timeout: 10m
-            # TODO: sdlc auth
-#          - name: TestDockerFakes
-#            config: fake_docker.toml
-#            count: 1
-#            timeout: 10m
+          - name: TestSmoke
+            config: smoke.toml
+            count: 1
+            timeout: 10m
+          # TODO: modify fake_docker.toml to use private registry, make sure that image is there.
+          # - name: TestDockerFakes
+          #   config: fake_docker.toml
+          #   count: 1
+          #   timeout: 10m
+          - name: TestFakes
+            config: fake.toml
+            count: 1
+            timeout: 10m
+          - name: TestPerformanceBaseline
+            config: performance_baseline.toml
+            count: 1
+            timeout: 10m
+          - name: TestScalability
+            config: scalability.toml
+            count: 1
+            timeout: 10m
     name: ${{ matrix.test.name }}
     steps:
       - name: Checkout repo
         uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4
       - name: Configure AWS credentials using OIDC
         uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # v4.0.2
         with:
-          role-to-assume: ${{ secrets.AWS_CTF_READ_ACCESS_ROLE_ARN }}
+          role-to-assume: ${{ secrets.AWS_GHA_SMARTCONTRACTKIT_ROLE_ARN }}
           aws-region: us-west-2
       - name: Login to Amazon ECR
         id: login-ecr-private
diff --git a/.github/workflows/framework-golden-tests.yml b/.github/workflows/framework-golden-tests.yml
@@ -18,10 +18,6 @@ jobs:
       fail-fast: false
       matrix:
         test:
-          - name: TestSmoke
-            config: smoke.toml
-            count: 1
-            timeout: 10m
           - name: TestSmokeLimitedResources
             config: smoke_limited_resources.toml
             count: 1
@@ -60,31 +56,14 @@ jobs:
           #   config: upgrade.toml
           #   count: 1
           #   timeout: 10m
-          - name: TestPerformanceBaseline
-            config: performance_baseline.toml
-            count: 1
-            timeout: 10m
           - name: TestChaos
             config: chaos.toml
             count: 1
             timeout: 10m
-          - name: TestScalability
-            config: scalability.toml
-            count: 1
-            timeout: 10m
           - name: TestLocalS3
             config: local_s3.toml
             count: 1
             timeout: 10m
-          - name: TestFakes
-            config: fake.toml
-            count: 1
-            timeout: 10m
-            # TODO: sdlc auth (move to framework-golden-tests-private.yml, which has that auth set up)
-#          - name: TestDockerFakes
-#            config: fake_docker.toml
-#            count: 1
-#            timeout: 10m
     name: ${{ matrix.test.name }}
     steps:
       - name: Checkout repo
diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml
@@ -13,9 +13,6 @@ jobs:
       fail-fast: false
       matrix:
         test:
-          - path: framework
-            vm: ubuntu-latest
-            regex: TestSmoke
           - path: framework
             vm: ubuntu-latest
             regex: TestPrometheus
diff --git a/framework/.changeset/v0.14.2.md b/framework/.changeset/v0.14.2.md
@@ -1 +1 @@
-- Expose P2P only in K8s
+- Expose P2P only in K8s
diff --git a/framework/.changeset/v0.15.5.md b/framework/.changeset/v0.15.5.md
@@ -0,0 +1 @@
+- Observability stack: update Tempo version to latest, enable traces-to-profiles, run Pyroscope with bigger and higher resolution profiles
diff --git a/framework/components/blockchain/anvil.go b/framework/components/blockchain/anvil.go
@@ -36,7 +36,7 @@ func defaultAnvil(in *Input) {
 		in.Port = "8545"
 	}
 	if in.ContainerName == "" {
-		in.ContainerName = "anvil"
+		in.ContainerName = "anvil-" + in.ChainID
 	}
 }
 
diff --git a/framework/components/clnode/clnode_test.go b/framework/components/clnode/clnode_test.go
@@ -1,6 +1,7 @@
 package clnode_test
 
 import (
+	"os"
 	"testing"
 
 	"github.com/stretchr/testify/require"
@@ -39,7 +40,7 @@ func TestSmokeComponentDockerNodeWithSharedDB(t *testing.T) {
 					VolumeName: "a",
 				},
 				Node: &clnode.NodeInput{
-					Image: "public.ecr.aws/chainlink/chainlink:v2.17.0",
+					Image: os.Getenv("CTF_CHAINLINK_IMAGE"),
 					Name:  "cl-node-1",
 				},
 			},
@@ -75,7 +76,7 @@ func TestSmokeComponentDockerNodeWithDB(t *testing.T) {
 					VolumeName: "b",
 				},
 				Node: &clnode.NodeInput{
-					Image: "public.ecr.aws/chainlink/chainlink:v2.17.0",
+					Image: os.Getenv("CTF_CHAINLINK_IMAGE"),
 					Name:  "cl-node-2",
 				},
 			},
diff --git a/framework/components/clnode/default.go b/framework/components/clnode/default.go
@@ -22,6 +22,7 @@ Level = 'debug'
 [Pyroscope]
 ServerAddress = 'http://host.docker.internal:4040'
 Environment = 'local'
+LinkTracesToProfiles = true
 
 [WebServer]
 HTTPWriteTimeout = '30s'
diff --git a/framework/components/simple_node_set/nodeset_test.go b/framework/components/simple_node_set/nodeset_test.go
@@ -1,6 +1,7 @@
 package simple_node_set_test
 
 import (
+	"os"
 	"testing"
 
 	"github.com/stretchr/testify/require"
@@ -61,7 +62,7 @@ func TestSmokeComponentDockerNodeSetSharedDB(t *testing.T) {
 				NodeSpecs: []*clnode.Input{
 					{
 						Node: &clnode.NodeInput{
-							Image: "public.ecr.aws/chainlink/chainlink:v2.17.0",
+							Image: os.Getenv("CTF_CHAINLINK_IMAGE"),
 							Name:  "cl-node-1",
 							UserConfigOverrides: `
 [Log]
@@ -71,7 +72,7 @@ level = 'info'
 					},
 					{
 						Node: &clnode.NodeInput{
-							Image: "public.ecr.aws/chainlink/chainlink:v2.17.0",
+							Image: os.Getenv("CTF_CHAINLINK_IMAGE"),
 							Name:  "cl-node-2",
 							UserConfigOverrides: `
 [Log]
diff --git a/framework/examples/myproject/upgrade_multi_nodeset_test.go b/framework/examples/myproject/upgrade_multi_nodeset_test.go
@@ -1,6 +1,7 @@
 package examples
 
 import (
+	"os"
 	"testing"
 	"time"
 
@@ -36,7 +37,7 @@ func TestMultiUpgrade(t *testing.T) {
 
 	// reboot both node sets and upgrade with new configs
 
-	in.NodeSetA.NodeSpecs[0].Node.Image = "public.ecr.aws/chainlink/chainlink:v2.17.0"
+	in.NodeSetA.NodeSpecs[0].Node.Image = os.Getenv("CTF_CHAINLINK_IMAGE")
 	in.NodeSetA.NodeSpecs[0].Node.UserConfigOverrides = `
 											[Log]
 											level = 'info'
@@ -45,7 +46,7 @@ func TestMultiUpgrade(t *testing.T) {
 	ns1, err = ns.UpgradeNodeSet(t, in.NodeSetA, bc, 3*time.Second)
 	require.NoError(t, err)
 
-	in.NodeSetB.NodeSpecs[0].Node.Image = "public.ecr.aws/chainlink/chainlink:v2.17.0"
+	in.NodeSetB.NodeSpecs[0].Node.Image = os.Getenv("CTF_CHAINLINK_IMAGE")
 	in.NodeSetB.NodeSpecs[0].Node.UserConfigOverrides = `
 											[Log]
 											level = 'info'
diff --git a/framework/examples/myproject/upgrade_test.go b/framework/examples/myproject/upgrade_test.go
@@ -2,6 +2,7 @@ package examples
 
 import (
 	"fmt"
+	"os"
 	"testing"
 	"time"
 
@@ -28,7 +29,7 @@ const (
 		                abi="submit(uint256 value)"
 		                data="{ \\"value\\": $(multiply) }"]
 		   submit_tx   [type="ethtx" to="0x859AAa51961284C94d970B47E82b8771942F1980" data="$(encode_tx)"]
-		
+
 		   fetch -> parse -> multiply -> encode_tx -> submit_tx
 		"""`
 )
@@ -57,12 +58,12 @@ func TestUpgrade(t *testing.T) {
 	_, _, err = c[0].CreateJobRaw(testJob)
 	require.NoError(t, err)
 
-	in.NodeSets[0].NodeSpecs[0].Node.Image = "public.ecr.aws/chainlink/chainlink:v2.17.0"
+	in.NodeSets[0].NodeSpecs[0].Node.Image = os.Getenv("CTF_CHAINLINK_IMAGE")
 	in.NodeSets[0].NodeSpecs[0].Node.UserConfigOverrides = `
 											[Log]
 											level = 'info'
 	`
-	in.NodeSets[0].NodeSpecs[4].Node.Image = "public.ecr.aws/chainlink/chainlink:v2.17.0"
+	in.NodeSets[0].NodeSpecs[4].Node.Image = os.Getenv("CTF_CHAINLINK_IMAGE")
 	in.NodeSets[0].NodeSpecs[4].Node.UserConfigOverrides = `
 											[Log]
 											level = 'info'
diff --git a/framework/observability/compose/conf/provisioning/datasources/loki.yaml b/framework/observability/compose/conf/provisioning/datasources/loki.yaml
@@ -3,22 +3,46 @@ apiVersion: 1
 datasources:
   - name: Tempo
     type: tempo
+    uid: tempo
     access: proxy
     url: http://tempo:3200
     isDefault: false
+    jsonData:
+      tracesToLogsV2:
+        datasourceUid: P8E80F9AEF21F6940
+        spanStartTimeShift: '-1h'
+        spanEndTimeShift: '1h'
+        filterByTraceID: true
+        filterBySpanID: true
+      tracesToProfiles:
+        datasourceUid: pyroscope
+        profileTypeId: 'process_cpu:cpu:nanoseconds:cpu:nanoseconds'
+        customQuery: true
+        query: '{service_name="chainlink-node"}'
+      serviceMap:
+        datasourceUid: PBFA97CFB590B2093
+      nodeGraph:
+        enabled: true
   - name: Loki
     type: loki
+    uid: P8E80F9AEF21F6940
     isDefault: true
     access: proxy
     url: http://loki:3100
     jsonData:
       maxLines: 5000
+      derivedFields:
+        - datasourceUid: tempo
+          matcherRegex: '"traceID":"([a-f0-9]+)"'
+          name: TraceID
+          url: '$${__value.raw}'
   - name: Prometheus
     type: prometheus
+    uid: PBFA97CFB590B2093
     access: proxy
-    url: http://prometheus:9090 # Replace with your Prometheus URL
+    url: http://prometheus:9090
     jsonData:
-      timeInterval: 5s # Adjust this interval as needed
+      timeInterval: 5s
 
   - name: PostgreSQL 0
     type: postgres
@@ -82,8 +106,11 @@ datasources:
 
   - name: Pyroscope
     type: grafana-pyroscope-datasource
+    uid: pyroscope
     access: proxy
     url: http://pyroscope:4040
     isDefault: false
     version: 1
     editable: true
+    jsonData:
+      maxNodes: 100000
diff --git a/framework/observability/compose/docker-compose.yaml b/framework/observability/compose/docker-compose.yaml
@@ -1,6 +1,7 @@
 services:
   tempo:
-    image: grafana/tempo:2.3.1
+    image: grafana/tempo:2.7.1
+    user: '0:0'
     command: [ "-config.file=/etc/tempo.yaml" ]
     volumes:
       - ./tempo.yaml:/etc/tempo.yaml
@@ -10,7 +11,11 @@ services:
       - "9411:9411"   # zipkin
       - "14268:14268" # jaeger ingest
       - "14250:14250" # jaeger grpc
-      - "14400:55680" # otlp http legacy
+    deploy:
+      resources:
+        limits:
+          memory: 2G
+    restart: unless-stopped
   otel-collector:
     image: otel/opentelemetry-collector-contrib:0.123.0
     volumes:
@@ -40,6 +45,10 @@ services:
   prometheus:
     image: prom/prometheus:v3.4.1
     user: '0:0'
+    command:
+      - '--config.file=/etc/prometheus/prometheus.yml'
+      - '--web.enable-remote-write-receiver'
+      - '--enable-feature=exemplar-storage'
     volumes:
       - /var/run/docker.sock:/var/run/docker.sock
       - ./conf/prometheus.yml:/etc/prometheus/prometheus.yml
diff --git a/framework/observability/compose/pyroscope-config.yaml b/framework/observability/compose/pyroscope-config.yaml
@@ -0,0 +1,25 @@
+# Pyroscope server configuration
+# Increased limits to preserve more profile data for comparison analysis
+# Mount as volume - ./pyroscope-config.yaml:/etc/pyroscope/config.yaml
+# and pass --config.file=/etc/pyroscope/config.yaml to pyroscope command
+# in docker-compose.yaml to apply these settings
+
+limits:
+  # Ingestion rate limits (default: 4 MiB/s rate, 2 MiB burst)
+  ingestion_rate_mb: 100
+  ingestion_burst_size_mb: 50
+
+  # Disable sample limit to prevent profile rejection (default: 16000)
+  max_profile_stacktrace_samples: 0
+
+  # Increase max stacktrace depth (default: 1000)
+  max_profile_stacktrace_depth: 2000
+
+  # Increase default flame graph nodes shown (default: 8192)
+  max_flamegraph_nodes_default: 100000
+
+  # Disable max flame graph nodes limit (default: 1048576)
+  max_flamegraph_nodes_max: 0
+
+  # Increase max profile size (default: 4MB)
+  max_profile_size_bytes: 16777216
diff --git a/framework/observability/compose/tempo.yaml b/framework/observability/compose/tempo.yaml

Original file line number	Diff line number	Diff line change
`@@ -1 +1 @@`
`1`		`-- Expose P2P only in K8s`
	`1`	`+- Expose P2P only in K8s`
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1 @@`
	`1`	`+- Observability stack: update Tempo version to latest, enable traces-to-profiles, run Pyroscope with bigger and higher resolution profiles`
Original file line number	Diff line number	Diff line change
`@@ -36,7 +36,7 @@ func defaultAnvil(in *Input) {`
`36`	`36`	`in.Port = "8545"`
`37`	`37`	`}`
`38`	`38`	`if in.ContainerName == "" {`
`39`		`- in.ContainerName = "anvil"`
	`39`	`+ in.ContainerName = "anvil-" + in.ChainID`
`40`	`40`	`}`
`41`	`41`	`}`
`42`	`42`