rane sot uniq versions (#2471)

Author: skudasov

.github/workflows/test.yaml

@@ -25,6 +25,9 @@ jobs:
           - path: framework
             vm: ubuntu-latest
             regex: TestDX
+          - path: framework
+            vm: ubuntu-latest
+            regex: TestChaos
           - path: wasp
             vm: ubuntu22.04-16cores-64GB # ghv-ignore!
             regex: TestSmoke

book/src/framework/compat.md

@@ -3,6 +3,7 @@
 ## Prerequisites
 
 Authorize in our SDLC ECR registry first. Get the creds and run
+
 ```bash
 aws ecr get-login-password --region us-west-2 | docker login --username AWS --password-stdin <sdlc_ecr_registry>
 ```
@@ -23,7 +24,7 @@ ctf compat backward \
 --refs 2.33.0 \
 --refs 2.34.0 \
 --refs 2.35.0 \
---nodes 3
+--upgrade-nodes 3
 ```
 
 Keep in mind that `refs` should be present in regsitry you are testing against, the first (oldest) `ref` should also have a valid end-to-end test that works.
@@ -32,27 +33,52 @@ In CI we detect SemVer tags automatically, whenever a new tag appears we select
 
 ```bash
 ctf compat backward \
---registry <sdlc_ecr_registry> \
+--registry <sdlc_ecr_chainlink_registry> \
 --buildcmd "just cli" \
---envcmd "cl r" \
+--envcmd "cl r env.toml,products/ocr2/basic.toml" \
 --testcmd "cl test ocr2 TestSmoke/rounds" \
---nodes 3 \
---versions-back 3
+--strip-image-suffix v \
+--upgrade-nodes 2 \
+--versions-back 2
 ```
 
 In case you have multiple DONs in your product and names of nodes are different please use `--node-name-template custom-cl-node-%d` option
 
-## Modelling Node Operators Cluster
+## Modelling Node Operators Cluster (WIP)
 
-It is possible to fetch versions node operators are currently running and model DON upgrade sequence locally. Logic is the same, get all the versions, rollback to the oldest one, setup product, verify, try to upgrade all the versions running the oldest test for each upgrade.
+It is possible to fetch versions node operators are currently running and model DON upgrade sequence locally. When `product` is specified, `compat` will fetch the current versions from the RANE SOT data source and model the upgrade sequence for versions found on real DONs up to the latest one, each node one at a time.
 
 ```bash
 ctf compat backward \
---registry <sdlc_ecr_registry>\
+--registry <sdlc_ecr_chainlink_registry> \
 --buildcmd "just cli" \
---envcmd "cl r" \
+--envcmd "cl r env.toml,products/ocr2/basic.toml" \
+--testcmd "cl test ocr2 TestSmoke/rounds" \
+--product data-feeds \
+--no-git-rollback \
+--don_nodes 5
+```
+
+The tool will check out earliest Git `ref` and setup environment and tests.
+
+If you don't have tests on this tag you can use `--no-git-rollback` to skip the rollback step.
+
+Since not all the versions from SOT are currently having corresponding Git tags or images, you can provide refs directly using `--refs` flag, useful for testing.
+
+```bash
+ctf compat backward \
+--registry <sdlc_ecr_chainlink_registry>\
+--buildcmd "just cli" \
+--envcmd "cl r env.toml,products/ocr2/basic.toml" \
 --testcmd "cl test ocr2 TestSmoke/rounds" \
---nop northwestnodes \
---versions-back 3 \
---nodes 3
+--product data-feeds \
+--refs "2.36.1-rc.0" \
+--refs "2.36.1-beta.0" \
+--refs "2.36.1-beta.2" \
+--refs "2.37.0-rc.0" \
+--refs "2.37.0-beta.0" \
+--refs "2.38.0-rc.0" \
+--refs "2.38.0-beta.0" \
+--no-git-rollback \
+--don_nodes 5
 ```

framework/.changeset/v0.15.6.md

@@ -0,0 +1 @@
+- Model DONs from SOT data source for upgrade testing

framework/chaos/chaos.go

@@ -58,7 +58,7 @@ func NewDockerChaos(ctx context.Context) (*DockerChaos, error) {
 		Image:      "lukaszlach/docker-tc",
 		Name:       dockerTCContainerName,
 		CapAdd:     []string{"NET_ADMIN"},
-		WaitingFor: wait.ForLog("Starting Docker Traffic Control"),
+		WaitingFor: wait.ForLog("Forwarding requests to /docker-tc"),
 		HostConfigModifier: func(h *container.HostConfig) {
 			h.Privileged = true
 			h.NetworkMode = "host"
@@ -75,7 +75,6 @@ func NewDockerChaos(ctx context.Context) (*DockerChaos, error) {
 	_, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{
 		ContainerRequest: req,
 		Started:          true,
-		Reuse:            true,
 	})
 	if err != nil {
 		return nil, fmt.Errorf("failed to start docker-tc container: %w", err)
@@ -87,13 +86,13 @@ func NewDockerChaos(ctx context.Context) (*DockerChaos, error) {
 
 // RemoveAll removes all the experiments
 func (m *DockerChaos) RemoveAll() error {
-	for containerName, experimentCmd := range m.Experiments {
+	for containerName, experimentRemoveCmd := range m.Experiments {
 		framework.L.Info().
 			Str("Container", containerName).
-			Str("Cmd", experimentCmd).
+			Str("Cmd", experimentRemoveCmd).
 			Msg("Removing chaos for container")
-		if _, err := framework.ExecCmd(experimentCmd); err != nil {
-			return fmt.Errorf("failed to remove chaos experiment: name: %s, command:%s, err: %w", containerName, experimentCmd, err)
+		if _, err := framework.ExecCmd(experimentRemoveCmd); err != nil {
+			return fmt.Errorf("failed to remove chaos experiment: name: %s, command:%s, err: %w", containerName, experimentRemoveCmd, err)
 		}
 	}
 	m.Experiments = make(map[string]string)

framework/chaos/chaos_test.go

@@ -6,21 +6,22 @@ import (
 
 	"github.com/stretchr/testify/require"
 
+	"github.com/smartcontractkit/chainlink-testing-framework/framework"
 	"github.com/smartcontractkit/chainlink-testing-framework/framework/chaos"
 	"github.com/smartcontractkit/chainlink-testing-framework/framework/rpc"
 )
 
-func TestSmokeChaos(t *testing.T) {
-	c, err := rpc.StartAnvil([]string{"--balance", "1", "--block-time", "5"})
+func TestChaos(t *testing.T) {
+	err := framework.DefaultNetwork(nil)
 	require.NoError(t, err)
-
-	i, err := c.Inspect(t.Context())
+	_, err = rpc.StartAnvil([]string{"--balance", "1", "--block-time", "5"})
 	require.NoError(t, err)
-	containerName := i.Name[1:]
 
 	dtc, err := chaos.NewDockerChaos(t.Context())
 	require.NoError(t, err)
 
+	containerName := "anvil"
+
 	tests := []struct {
 		name          string
 		containerName string

framework/cmd/main.go

@@ -281,18 +281,27 @@ Be aware that any TODO requires your attention before your run the final test!
 						Name:    "backward",
 						Aliases: []string{"b"},
 						Flags: []cli.Flag{
+							&cli.BoolFlag{
+								Name:  "no-git-rollback",
+								Usage: "Disable Git rollback, run current version of env and command",
+							},
 							&cli.IntFlag{
 								Name:    "versions-back",
 								Aliases: []string{"v"},
 								Usage:   "How many versions back to test",
 								Value:   1,
 							},
 							&cli.IntFlag{
-								Name:    "nodes",
+								Name:    "upgrade-nodes",
 								Aliases: []string{"n"},
 								Usage:   "How many nodes to upgrade",
 								Value:   3,
 							},
+							&cli.IntFlag{
+								Name:  "don_nodes",
+								Usage: "How many total nodes DON have",
+								Value: 5,
+							},
 							&cli.StringFlag{
 								Name:    "registry",
 								Aliases: []string{"r"},
@@ -301,7 +310,11 @@ Be aware that any TODO requires your attention before your run the final test!
 							},
 							&cli.StringFlag{
 								Name:  "strip-image-suffix",
-								Usage: "Stripts image suffix from ref to map it to registry images",
+								Usage: "Strips image suffix from ref to map it to registry images",
+							},
+							&cli.StringFlag{
+								Name:  "rane-add-git-tag-prefix",
+								Usage: "Add Git tag prefix to RANE's refs to match Git tags",
 							},
 							&cli.StringFlag{
 								Name:    "buildcmd",
@@ -320,8 +333,8 @@ Be aware that any TODO requires your attention before your run the final test!
 								Usage:   "Test verification command",
 							},
 							&cli.StringFlag{
-								Name:  "nop",
-								Usage: "Find specific NOPs ref upgrade sequence",
+								Name:  "product",
+								Usage: "Find specific NOPs ref upgrade sequence by product. See RANE SOT default URL, maps to 'jq '.nodes[].jobs[].product' from data source",
 							},
 							&cli.StringFlag{
 								Name:  "node-name-template",
@@ -352,6 +365,7 @@ Be aware that any TODO requires your attention before your run the final test!
 						},
 						Usage: "Rollbacks N versions back, runs the test the upgrades CL nodes with new versions",
 						Action: func(c *cli.Context) error {
+							noGitRollback := c.Bool("no-git-rollback")
 							versionsBack := c.Int("versions-back")
 							registry := c.String("registry")
 							refs := c.StringSlice("refs")
@@ -361,13 +375,15 @@ Be aware that any TODO requires your attention before your run the final test!
 							buildcmd := c.String("buildcmd")
 							envcmd := c.String("envcmd")
 							testcmd := c.String("testcmd")
-							nodes := c.Int("nodes")
+							upgradeNodes := c.Int("upgrade-nodes")
+							donNodes := c.Int("don_nodes")
 							nodeNameTemplate := c.String("node-name-template")
 
-							nop := c.String("nop")
-							sotURL := c.String("sot-url")
+							product := c.String("product")
 							skipPull := c.Bool("skip-pull")
 
+							sotURL := c.String("sot-url")
+
 							// test logic is:
 							// - rollback to selected ref
 							// - spin up the env and perform the initial smoke test
@@ -377,33 +393,51 @@ Be aware that any TODO requires your attention before your run the final test!
 
 							// if no refs provided find refs (tags) sequence SemVer sequence for last N versions_back
 							// else, use refs param slice
-							//
+
+							// Step 1: Find upgrade sequence either from Git refs or product unique versions found in SOT data source
+
 							var err error
-							if len(refs) == 0 && nop == "" {
+							if len(refs) == 0 && product == "" {
 								refs, err = framework.FindSemVerRefSequence(versionsBack, include, exclude)
 								if err != nil {
 									return err
 								}
 							}
-							if nop != "" {
-								refs, err = framework.FindNOPRefs(sotURL, nop, exclude)
+
+							if product != "" {
+								// TODO: Stub, until versions are mapped properly and not skipped this is just an example
+								// Both Git, ECR and "versions" from SOT data do not match
+								// so we fetch the data but do not use it for now
+								// test refs are populated from --refs, see docs and CI in core
+								_, err := framework.FindNOPsVersionsByProduct(sotURL, product, exclude)
 								if err != nil {
 									return err
 								}
 							}
+
 							framework.L.Info().
 								Int("TotalSemVerRefs", len(refs)).
 								Strs("SelectedRefs", refs).
 								Str("EarliestRefs", refs[0]).
 								Msg("Formed upgrade sequence")
+
 							// if no commands just show the tags and return
 							if buildcmd == "" || envcmd == "" || testcmd == "" {
 								framework.L.Info().Msg("No envcmd or testcmd or buildcmd provided, skipping")
 								return nil
 							}
-							// checkout the oldest ref
-							if err := framework.CheckOut(refs[0]); err != nil {
-								return err
+
+							// RANE doesn't have Git tags so we are mapping field "version" to Git tag
+							gitTagPrefix := c.String("rane-add-git-tag-prefix")
+							for i := range refs {
+								refs[i] = fmt.Sprintf("%s%s", gitTagPrefix, refs[i])
+							}
+
+							// checkout the oldest ref if we need to preserve test setup isolation
+							if !noGitRollback {
+								if err := framework.CheckOut(refs[0]); err != nil {
+									return err
+								}
 							}
 
 							// this is a hack allowing us to match what we have in Git to registry or NOP version
@@ -413,42 +447,25 @@ Be aware that any TODO requires your attention before your run the final test!
 								refs[i] = strings.ReplaceAll(refs[i], testImgSuffix, "")
 							}
 
-							// setup the env and verify with test command
-							framework.L.Info().Strs("Sequence", refs).Msg("Running upgrade sequence")
-							// first env var is used by devenv, second by simple nodeset to override the image
-							os.Setenv("CHAINLINK_IMAGE", fmt.Sprintf("%s:%s", registry, refs[0]))
-							os.Setenv("CTF_CHAINLINK_IMAGE", fmt.Sprintf("%s:%s", registry, refs[0]))
-							if _, err := framework.ExecCmdWithContext(c.Context, buildcmd); err != nil {
-								return err
-							}
-							if _, err := framework.ExecCmdWithContext(c.Context, envcmd); err != nil {
-								return err
+							// Step 2: Perform upgrade testing with either upgrading through last N versions
+							// or by modelling versions from SOT data source, then upgrading to the latest version
+							upgradeContext := framework.UpgradeContext{
+								ProductName:      product,
+								Refs:             refs,
+								DonNodes:         donNodes,
+								UpgradeNodes:     upgradeNodes,
+								NodeNameTemplate: nodeNameTemplate,
+								Registry:         registry,
+								Buildcmd:         buildcmd,
+								Envcmd:           envcmd,
+								Testcmd:          testcmd,
+								SkipPull:         skipPull,
 							}
-							if _, err := framework.ExecCmdWithContext(c.Context, testcmd); err != nil {
-								return err
-							}
-							// start upgrading nodes and verifying with tests, all the versions except the oldest one
-							for _, tag := range refs[1:] {
-								framework.L.Info().
-									Int("Nodes", nodes).
-									Str("Version", tag).
-									Msg("Upgrading nodes")
-								img := fmt.Sprintf("%s:%s", registry, tag)
-								if !skipPull {
-									if _, err := framework.ExecCmdWithContext(c.Context, fmt.Sprintf("docker pull %s", img)); err != nil {
-										return fmt.Errorf("failed to pull image %s: %w", img, err)
-									}
-								}
-								for i := range nodes {
-									if err := framework.UpgradeContainer(c.Context, fmt.Sprintf(nodeNameTemplate, i), img); err != nil {
-										return err
-									}
-								}
-								if _, err := framework.ExecCmd(testcmd); err != nil {
-									return err
-								}
+
+							if product != "" {
+								return framework.UpgradeNProductUniqueVersionsRolling(c.Context, upgradeContext)
 							}
-							return nil
+							return framework.UpgradeNRolling(c.Context, upgradeContext)
 						},
 					},
 				},