diff --git a/.github/workflows/test-cl-smoke.yaml b/.github/workflows/test-cl-smoke.yaml index d3d6724e9..67260dfcd 100644 --- a/.github/workflows/test-cl-smoke.yaml +++ b/.github/workflows/test-cl-smoke.yaml @@ -79,6 +79,34 @@ jobs: run_cmd: TestHA_CrossComponentDown config: "env.toml,env-HA.toml,env-cl.toml,env-cl-ci.toml" timeout: 10m + - name: TestEnvironmentChangeReconcile_CommitteeVerifierAllowlistDecoyExpectErrorThenDeployerHappyPath + run_cmd: TestEnvironmentChangeReconcile_CommitteeVerifierAllowlistDecoyExpectErrorThenDeployerHappyPath + config: "env.toml,env-cl.toml,env-cl-ci.toml" + timeout: 10m + - name: TestEnvironmentChangeReconcile_TestRouterLaneThenProductionRouterExpectMessagesSucceedEachStage + run_cmd: TestEnvironmentChangeReconcile_TestRouterLaneThenProductionRouterExpectMessagesSucceedEachStage + config: "env.toml,env-cl.toml,env-cl-ci.toml" + timeout: 10m + - name: TestEnvironmentChangeReconcile_RemoveDefaultCommitteeNOPAndLowerThresholdExpectMessageSuccessWithoutRemovedNOPVerification + run_cmd: TestEnvironmentChangeReconcile_RemoveDefaultCommitteeNOPAndLowerThresholdExpectMessageSuccessWithoutRemovedNOPVerification + config: "env.toml,env-cl.toml,env-cl-ci.toml" + timeout: 10m + - name: TestEnvironmentChangeReconcile_SignerKeyRotationExpectMessagesSucceedEachPhase + run_cmd: TestEnvironmentChangeReconcile_SignerKeyRotationExpectMessagesSucceedEachPhase + config: "env.toml,env-cl-rotation.toml,env-cl-rotation-ci.toml" + timeout: 15m + - name: TestEnvironmentChangeReconcile_SignerKeyRotationOffchainFirstExpectMessageSuccessAfterFullReconcile + run_cmd: TestEnvironmentChangeReconcile_SignerKeyRotationOffchainFirstExpectMessageSuccessAfterFullReconcile + config: "env.toml,env-cl-rotation.toml,env-cl-rotation-ci.toml" + timeout: 15m + - name: TestEnvironmentChangeReconcile_ThresholdDecreaseExpectMessageSuccess + run_cmd: TestEnvironmentChangeReconcile_ThresholdDecreaseExpectMessageSuccess + config: "env.toml,env-cl-rotation.toml,env-cl-rotation-ci.toml" + timeout: 15m + - name: TestEnvironmentChangeReconcile_ThresholdIncreaseRecoveryExpectMessageSuccessAfterFullReconcile + run_cmd: TestEnvironmentChangeReconcile_ThresholdIncreaseRecoveryExpectMessageSuccessAfterFullReconcile + config: "env.toml,env-cl-rotation.toml,env-cl-rotation-ci.toml" + timeout: 20m # We need to configure HeadTracker for the CL tests to have finality depth. Otherwise, it does instant finality. # - name: TestE2EReorg # config: "env.toml,env-src-auto-mine.toml,env-cl.toml,env-cl-ci.toml" diff --git a/build/devenv/env-cl-rotation-ci.toml b/build/devenv/env-cl-rotation-ci.toml new file mode 100644 index 000000000..c45fe6f5c --- /dev/null +++ b/build/devenv/env-cl-rotation-ci.toml @@ -0,0 +1,32 @@ +## CI-specific override for env-cl-rotation.toml: use the pre-built CL image. +[[nodesets]] + name = "don" + nodes = 3 + override_mode = "each" + + [nodesets.db] + image = "postgres:15.0" + + [[nodesets.node_specs]] + + [nodesets.node_specs.node] + image = "docker.io/library/local:latest" + + [nodesets.node_specs.node.env_vars] + CL_EVM_CMD="" + + [[nodesets.node_specs]] + + [nodesets.node_specs.node] + image = "docker.io/library/local:latest" + + [nodesets.node_specs.node.env_vars] + CL_EVM_CMD="" + + [[nodesets.node_specs]] + + [nodesets.node_specs.node] + image = "docker.io/library/local:latest" + + [nodesets.node_specs.node.env_vars] + CL_EVM_CMD="" diff --git a/build/devenv/env-cl-rotation.toml b/build/devenv/env-cl-rotation.toml new file mode 100644 index 000000000..a4c4f8123 --- /dev/null +++ b/build/devenv/env-cl-rotation.toml @@ -0,0 +1,458 @@ +## CL mode environment topology for key rotation tests. +## Default committee uses 2-of-3 so rotating one signer exercises quorum behavior. +[environment_topology] +indexer_address = ["http://indexer-1:8100"] +pyroscope_url = "http://host.docker.internal:4040" + +[environment_topology.monitoring] +Enabled = true +Type = "beholder" + +[environment_topology.monitoring.Beholder] +InsecureConnection = true +OtelExporterHTTPEndpoint = "host.docker.internal:4318" +LogStreamingEnabled = false +MetricReaderInterval = 5 +TraceSampleRatio = 1.0 +TraceBatchTimeout = 10 + +[[environment_topology.nop_topology.nops]] +alias = "node-0" +name = "node-0" + +[[environment_topology.nop_topology.nops]] +alias = "node-1" +name = "node-1" + +[[environment_topology.nop_topology.nops]] +alias = "node-2" +name = "node-2" + +[environment_topology.nop_topology.committees.default] +qualifier = "default" +verifier_version = "2.0.0" + +[[environment_topology.nop_topology.committees.default.aggregators]] +name = "default" +address = "default-aggregator:50051" +insecure_connection = true + +[environment_topology.nop_topology.committees.default.chain_configs.3379446385462418246] +nop_aliases = ["node-0", "node-1", "node-2"] +threshold = 2 + +[environment_topology.nop_topology.committees.default.chain_configs.12922642891491394802] +nop_aliases = ["node-0", "node-1", "node-2"] +threshold = 2 + +[environment_topology.nop_topology.committees.default.chain_configs.4793464827907405086] +nop_aliases = ["node-0", "node-1", "node-2"] +threshold = 2 + +[environment_topology.nop_topology.committees.secondary] +qualifier = "secondary" +verifier_version = "2.0.0" + +[[environment_topology.nop_topology.committees.secondary.aggregators]] +name = "default" +address = "secondary-aggregator:50051" +insecure_connection = true + +[environment_topology.nop_topology.committees.secondary.chain_configs.3379446385462418246] +nop_aliases = ["node-0", "node-1"] +threshold = 2 + +[environment_topology.nop_topology.committees.secondary.chain_configs.12922642891491394802] +nop_aliases = ["node-0", "node-1"] +threshold = 2 + +[environment_topology.nop_topology.committees.secondary.chain_configs.4793464827907405086] +nop_aliases = ["node-0", "node-1"] +threshold = 2 + +[environment_topology.nop_topology.committees.tertiary] +qualifier = "tertiary" +verifier_version = "2.0.0" + +[[environment_topology.nop_topology.committees.tertiary.aggregators]] +name = "default" +address = "tertiary-aggregator:50051" +insecure_connection = true + +[environment_topology.nop_topology.committees.tertiary.chain_configs.3379446385462418246] +nop_aliases = ["node-0", "node-1"] +threshold = 2 + +[environment_topology.nop_topology.committees.tertiary.chain_configs.12922642891491394802] +nop_aliases = ["node-0", "node-1"] +threshold = 2 + +[environment_topology.nop_topology.committees.tertiary.chain_configs.4793464827907405086] +nop_aliases = ["node-0", "node-1"] +threshold = 2 + +[environment_topology.executor_pools.default] +indexer_query_limit = 100 +backoff_duration = 15_000_000_000 +lookback_window = 3_600_000_000_000 +reader_cache_expiry = 300_000_000_000 +max_retry_duration = 28_800_000_000_000 +worker_count = 100 +ntp_server = "time.google.com" + +[environment_topology.executor_pools.default.chain_configs."3379446385462418246"] +nop_aliases = ["node-0", "node-1"] +execution_interval = 15_000_000_000 + +[environment_topology.executor_pools.default.chain_configs."12922642891491394802"] +nop_aliases = ["node-0", "node-1"] +execution_interval = 15_000_000_000 + +[environment_topology.executor_pools.default.chain_configs."4793464827907405086"] +nop_aliases = ["node-0", "node-1"] +execution_interval = 15_000_000_000 + +[environment_topology.executor_pools.custom] +indexer_query_limit = 100 +backoff_duration = 15_000_000_000 +lookback_window = 3_600_000_000_000 +reader_cache_expiry = 300_000_000_000 +max_retry_duration = 28_800_000_000_000 +worker_count = 100 +ntp_server = "time.google.com" + +[environment_topology.executor_pools.custom.chain_configs."3379446385462418246"] +nop_aliases = ["node-0"] +execution_interval = 15_000_000_000 + +[environment_topology.executor_pools.custom.chain_configs."12922642891491394802"] +nop_aliases = ["node-0"] +execution_interval = 15_000_000_000 + +[environment_topology.executor_pools.custom.chain_configs."4793464827907405086"] +nop_aliases = ["node-0"] +execution_interval = 15_000_000_000 + +[[nodesets]] + name = "don" + nodes = 3 + override_mode = "each" + + [nodesets.db] + image = "postgres:15.0" + + [[nodesets.node_specs]] + + [nodesets.node_specs.node] + docker_ctx = "../../../chainlink" + docker_file = "plugins/chainlink.Dockerfile" + + [nodesets.node_specs.node.env_vars] + CL_EVM_CMD="" + + [[nodesets.node_specs]] + + [nodesets.node_specs.node] + docker_ctx = "../../../chainlink" + docker_file = "plugins/chainlink.Dockerfile" + + [nodesets.node_specs.node.env_vars] + CL_EVM_CMD="" + + [[nodesets.node_specs]] + + [nodesets.node_specs.node] + docker_ctx = "../../../chainlink" + docker_file = "plugins/chainlink.Dockerfile" + + [nodesets.node_specs.node.env_vars] + CL_EVM_CMD="" + +## Default Verifier Setup (3 nodes) +[[verifier]] + mode = "cl" + image = "verifier:dev" + container_name = "default-verifier-1" + nop_alias = "node-0" + port = 8100 + source_code_path = "../verifier" + root_path = "../../" + committee_name = "default" + node_index = 0 + insecure_aggregator_connection = true + [verifier.db] + image = "postgres:16-alpine" + name = "default-verifier-1-db" + port = 8432 + +[[verifier]] + mode = "cl" + image = "verifier:dev" + container_name = "default-verifier-2" + nop_alias = "node-1" + port = 8200 + source_code_path = "../verifier" + root_path = "../../" + committee_name = "default" + node_index = 1 + insecure_aggregator_connection = true + [verifier.db] + image = "postgres:16-alpine" + name = "default-verifier-2-db" + port = 8433 + +[[verifier]] + mode = "cl" + image = "verifier:dev" + container_name = "default-verifier-3" + nop_alias = "node-2" + port = 8700 + source_code_path = "../verifier" + root_path = "../../" + committee_name = "default" + node_index = 0 + insecure_aggregator_connection = true + [verifier.db] + image = "postgres:16-alpine" + name = "default-verifier-3-db" + port = 8438 + +# Secondary Verifier Setup (shares same 2 nodes) +[[verifier]] + mode = "cl" + image = "verifier:dev" + container_name = "secondary-verifier-1" + nop_alias = "node-0" + port = 8300 + source_code_path = "../verifier" + root_path = "../../" + committee_name = "secondary" + node_index = 0 + insecure_aggregator_connection = true + [verifier.db] + image = "postgres:16-alpine" + name = "secondary-verifier-1-db" + port = 8434 + +[[verifier]] + mode = "cl" + image = "verifier:dev" + container_name = "secondary-verifier-2" + nop_alias = "node-1" + port = 8400 + source_code_path = "../verifier" + root_path = "../../" + committee_name = "secondary" + node_index = 1 + insecure_aggregator_connection = true + [verifier.db] + image = "postgres:16-alpine" + name = "secondary-verifier-2-db" + port = 8435 + +# Tertiary Verifier Setup (shares same 2 nodes) +[[verifier]] + mode = "cl" + image = "verifier:dev" + container_name = "tertiary-verifier-1" + nop_alias = "node-0" + port = 8500 + source_code_path = "../verifier" + root_path = "../../" + committee_name = "tertiary" + node_index = 0 + insecure_aggregator_connection = true + [verifier.db] + image = "postgres:16-alpine" + name = "tertiary-verifier-1-db" + port = 8436 + +[[verifier]] + mode = "cl" + image = "verifier:dev" + container_name = "tertiary-verifier-2" + nop_alias = "node-1" + port = 8600 + source_code_path = "../verifier" + root_path = "../../" + committee_name = "tertiary" + node_index = 1 + insecure_aggregator_connection = true + [verifier.db] + image = "postgres:16-alpine" + name = "tertiary-verifier-2-db" + port = 8437 + +[[executor]] + mode = "cl" + image = "executor:dev" + port = 8101 + source_code_path = "../executor" + root_path = "../../" + container_name = "default-executor-1" + nop_alias = "node-0" + executor_qualifier = "default" + +[[executor]] + mode = "cl" + image = "executor:dev" + port = 8102 + source_code_path = "../executor" + root_path = "../../" + container_name = "default-executor-2" + nop_alias = "node-1" + executor_qualifier = "default" + +[[executor]] + mode = "cl" + image = "executor:dev" + port = 8103 + source_code_path = "../executor" + root_path = "../../" + container_name = "custom-executor-1" + nop_alias = "node-0" + executor_qualifier = "custom" + +[[aggregator]] + committee_name = "default" + image = "aggregator:dev" + host_port = 50051 + redundant_aggregators = 1 + source_code_path = "../aggregator" + root_path = "../../" + + [aggregator.db] + image = "postgres:16-alpine" + host_port = 7432 + + [aggregator.redis] + image = "redis:7-alpine" + host_port = 6379 + + [[aggregator.api_clients]] + client_id = "default-verifier-1" + description = "Development default verifier 1" + enabled = true + groups = ["verifiers"] + + [[aggregator.api_clients]] + client_id = "default-verifier-2" + description = "Development default verifier 2" + enabled = true + groups = ["verifiers"] + + [[aggregator.api_clients]] + client_id = "default-verifier-3" + description = "Development default verifier 3" + enabled = true + groups = ["verifiers"] + + [[aggregator.api_clients]] + client_id = "monitoring" + description = "Monitoring and infrastructure client" + enabled = true + groups = ["monitoring"] + + [[aggregator.api_clients]] + client_id = "indexer" + description = "Development Indexer" + enabled = true + groups = ["indexer"] + + [aggregator.env] + storage_connection_url = "postgresql://aggregator:aggregator@default-aggregator-db:5432/aggregator?sslmode=disable" + redis_address = "default-aggregator-redis:6379" + redis_password = "" + redis_db = "0" + +[[aggregator]] + committee_name = "secondary" + image = "aggregator:dev" + host_port = 50052 + redundant_aggregators = 0 + source_code_path = "../aggregator" + root_path = "../../" + + [aggregator.db] + image = "postgres:16-alpine" + host_port = 7433 + + [aggregator.redis] + image = "redis:7-alpine" + host_port = 6380 + + [[aggregator.api_clients]] + client_id = "secondary-verifier-1" + description = "Development secondary verifier 1" + enabled = true + groups = ["verifiers"] + + [[aggregator.api_clients]] + client_id = "secondary-verifier-2" + description = "Development secondary verifier 2" + enabled = true + groups = ["verifiers"] + + [[aggregator.api_clients]] + client_id = "monitoring" + description = "Monitoring and infrastructure client" + enabled = true + groups = ["monitoring"] + + [[aggregator.api_clients]] + client_id = "indexer" + description = "Development Indexer" + enabled = true + groups = ["indexer"] + + [aggregator.env] + storage_connection_url = "postgresql://aggregator:aggregator@secondary-aggregator-db:5432/aggregator?sslmode=disable" + redis_address = "secondary-aggregator-redis:6379" + redis_password = "" + redis_db = "0" + +[[aggregator]] + committee_name = "tertiary" + image = "aggregator:dev" + host_port = 50053 + redundant_aggregators = 0 + source_code_path = "../aggregator" + root_path = "../../" + + [aggregator.db] + image = "postgres:16-alpine" + host_port = 7434 + + [aggregator.redis] + image = "redis:7-alpine" + host_port = 6381 + + [[aggregator.api_clients]] + client_id = "tertiary-verifier-1" + description = "Development tertiary verifier 1" + enabled = true + groups = ["verifiers"] + + [[aggregator.api_clients]] + client_id = "tertiary-verifier-2" + description = "Development tertiary verifier 2" + enabled = true + groups = ["verifiers"] + + [[aggregator.api_clients]] + client_id = "monitoring" + description = "Monitoring and infrastructure client" + enabled = true + groups = ["monitoring"] + + [[aggregator.api_clients]] + client_id = "indexer" + description = "Development Indexer" + enabled = true + groups = ["indexer"] + + [aggregator.env] + storage_connection_url = "postgresql://aggregator:aggregator@tertiary-aggregator-db:5432/aggregator?sslmode=disable" + redis_address = "tertiary-aggregator-redis:6379" + redis_password = "" + redis_db = "0" diff --git a/build/devenv/env-cl.toml b/build/devenv/env-cl.toml index 4d0a6e999..1c7ba0d18 100644 --- a/build/devenv/env-cl.toml +++ b/build/devenv/env-cl.toml @@ -34,15 +34,15 @@ insecure_connection = true [environment_topology.nop_topology.committees.default.chain_configs.3379446385462418246] nop_aliases = ["node-0", "node-1"] -threshold = 2 +threshold = 1 [environment_topology.nop_topology.committees.default.chain_configs.12922642891491394802] nop_aliases = ["node-0", "node-1"] -threshold = 2 +threshold = 1 [environment_topology.nop_topology.committees.default.chain_configs.4793464827907405086] nop_aliases = ["node-0", "node-1"] -threshold = 2 +threshold = 1 [environment_topology.nop_topology.committees.secondary] qualifier = "secondary" diff --git a/build/devenv/environment.go b/build/devenv/environment.go index 0b1f5538b..ba28ddf79 100644 --- a/build/devenv/environment.go +++ b/build/devenv/environment.go @@ -533,23 +533,22 @@ func buildEnvironmentTopology(in *Cfg, e *deployment.Environment) *ccvdeployment return &envCfg } +// BuildEnvironmentTopology is the exported entry point used by tests and harness code. +func BuildEnvironmentTopology(in *Cfg, e *deployment.Environment) *ccvdeployment.EnvironmentTopology { + return buildEnvironmentTopology(in, e) +} + // generateExecutorJobSpecs generates job specs for all executors using the changeset. -// It returns a map of container name -> job spec for use in CL mode. // For standalone mode, it also sets GeneratedConfig on each executor. // The ds parameter is a mutable datastore that will be updated with the changeset output. func generateExecutorJobSpecs( - ctx context.Context, e *deployment.Environment, in *Cfg, - selectors []uint64, - impls []cciptestinterfaces.CCIP17Configuration, topology *ccvdeployment.EnvironmentTopology, ds datastore.MutableDataStore, -) (map[string]bootstrap.JobSpec, error) { - executorJobSpecs := make(map[string]bootstrap.JobSpec) - +) error { if len(in.Executor) == 0 { - return executorJobSpecs, nil + return nil } // Group executors by qualifier @@ -576,80 +575,87 @@ func generateExecutorJobSpecs( TargetNOPs: ccvshared.ConvertStringToNopAliases(execNOPAliases), }) if err != nil { - return nil, fmt.Errorf("failed to generate executor configs for qualifier %s: %w", qualifier, err) + return fmt.Errorf("failed to generate executor configs for qualifier %s: %w", qualifier, err) } if err := ds.Merge(output.DataStore.Seal()); err != nil { - return nil, fmt.Errorf("failed to merge executor job specs datastore: %w", err) + return fmt.Errorf("failed to merge executor job specs datastore: %w", err) } for _, exec := range qualifierExecutors { jobSpecID := ccvshared.NewExecutorJobID(ccvshared.NOPAlias(exec.NOPAlias), ccvshared.ExecutorJobScope{ExecutorQualifier: qualifier}) job, err := ccvdeployment.GetJob(output.DataStore.Seal(), ccvshared.NOPAlias(exec.NOPAlias), jobSpecID.ToJobID()) if err != nil { - return nil, fmt.Errorf("failed to get executor job spec for %s: %w", exec.ContainerName, err) + return fmt.Errorf("failed to get executor job spec for %s: %w", exec.ContainerName, err) } - // TODO: Use bootstrap.JobSpec in CLD to avoid this conversion here var executorSpec ExecutorJobSpec - { - md, err := toml.Decode(job.Spec, &executorSpec) - if err != nil { - return nil, fmt.Errorf("failed to decode verifier job spec for %s: %w", exec.ContainerName, err) - } - if len(md.Undecoded()) > 0 { - L.Warn(). - Str("spec", job.Spec). - Str("undecoded fields", fmt.Sprintf("%v", md.Undecoded())). - Msg("Undecoded fields in executor job spec") - return nil, fmt.Errorf("unknown fields in executor job spec for %s: %v", exec.ContainerName, md.Undecoded()) - } - executorJobSpecs[exec.ContainerName] = executorSpec.ToBootstrapJobSpec() + md, err := toml.Decode(job.Spec, &executorSpec) + if err != nil { + return fmt.Errorf("failed to decode executor job spec for %s: %w", exec.ContainerName, err) + } + if len(md.Undecoded()) > 0 { + L.Warn(). + Str("spec", job.Spec). + Str("undecoded fields", fmt.Sprintf("%v", md.Undecoded())). + Msg("Undecoded fields in executor job spec") + return fmt.Errorf("unknown fields in executor job spec for %s: %v", exec.ContainerName, md.Undecoded()) } + exec.GeneratedJobSpecs = []bootstrap.JobSpec{executorSpec.ToBootstrapJobSpec()} - // Extract inner config from job spec for standalone mode var cfg executor.Configuration if err := toml.Unmarshal([]byte(executorSpec.ExecutorConfig), &cfg); err != nil { - return nil, fmt.Errorf("failed to parse executor config from job spec: %w", err) + return fmt.Errorf("failed to parse executor config from job spec: %w", err) } // Marshal the inner config back to TOML for standalone mode configBytes, err := toml.Marshal(cfg) if err != nil { - return nil, fmt.Errorf("failed to marshal executor config: %w", err) + return fmt.Errorf("failed to marshal executor config: %w", err) } exec.GeneratedConfig = string(configBytes) } } - // Set transmitter keys for standalone mode using family-specific key generation for _, exec := range in.Executor { + if exec.TransmitterPrivateKey != "" { + continue + } family := exec.ChainFamily if family == "" { family = chainsel.FamilyEVM } fac, facErr := GetImplFactory(family) if facErr != nil { - return nil, fmt.Errorf("no impl factory for executor chain family %q: %w", family, facErr) + return fmt.Errorf("no impl factory for executor chain family %q: %w", family, facErr) } pk, pkErr := fac.GenerateTransmitterKey() if pkErr != nil { - return nil, fmt.Errorf("failed to generate transmitter key for family %q: %w", family, pkErr) + return fmt.Errorf("failed to generate transmitter key for family %q: %w", family, pkErr) } exec.TransmitterPrivateKey = pk } - // Build executor transmitter addresses grouped by chain family so each chain - // only funds addresses in its native format. + return nil +} + +func fundStandaloneExecutorAddresses( + ctx context.Context, + in *Cfg, + impls []cciptestinterfaces.CCIP17Configuration, +) error { addressesByFamily := make(map[string][]protocol.UnknownAddress) for _, exec := range in.Executor { + if exec == nil || exec.Mode != services.Standalone { + continue + } family := exec.ChainFamily if family == "" { family = chainsel.FamilyEVM } fac, facErr := GetImplFactory(family) if facErr != nil { - return nil, fmt.Errorf("no impl factory for executor chain family %q: %w", family, facErr) + return fmt.Errorf("no impl factory for executor chain family %q: %w", family, facErr) } addressesByFamily[family] = append( addressesByFamily[family], @@ -657,6 +663,10 @@ func generateExecutorJobSpecs( ) } + if len(addressesByFamily) == 0 { + return nil + } + Plog.Info().Any("AddressesByFamily", addressesByFamily).Int("ImplsLen", len(impls)).Msg("Funding executors") for i, impl := range impls { family, famErr := blockchain.TypeToFamily(in.Blockchains[i].Type) @@ -674,12 +684,11 @@ func generateExecutorJobSpecs( Plog.Info().Int("ImplIndex", i).Msg("Funding executor") if err := impl.FundAddresses(ctx, in.Blockchains[i], addresses, big.NewInt(5)); err != nil { - return nil, fmt.Errorf("failed to fund addresses for executors: %w", err) + return fmt.Errorf("failed to fund addresses for executors: %w", err) } Plog.Info().Int("ImplIndex", i).Msg("Funded executors") } - - return executorJobSpecs, nil + return nil } // generateVerifierJobSpecs generates job specs for all verifiers using the changeset. @@ -690,7 +699,6 @@ func generateExecutorJobSpecs( func generateVerifierJobSpecs( e *deployment.Environment, in *Cfg, - selectors []uint64, topology *ccvdeployment.EnvironmentTopology, sharedTLSCerts *services.TLSCertPaths, ds datastore.MutableDataStore, @@ -722,11 +730,31 @@ func generateVerifierJobSpecs( } for family := range families { + activeNOPAliases, err := topologyVerifierNOPAliasesForCommitteeFamily(topology, committeeName, family) + if err != nil { + return nil, err + } + + activeFamilyVerifiers := make([]*committeeverifier.Input, 0, len(committeeVerifiers)) verNOPAliases := make([]ccvshared.NOPAlias, 0, len(committeeVerifiers)) for _, ver := range committeeVerifiers { - if ver.ChainFamily == family { - verNOPAliases = append(verNOPAliases, ccvshared.NOPAlias(ver.NOPAlias)) + if ver.ChainFamily != family { + continue + } + if _, ok := activeNOPAliases[ver.NOPAlias]; !ok { + ver.GeneratedJobSpecs = nil + ver.GeneratedConfig = "" + if ver.Out == nil { + ver.Out = &committeeverifier.Output{} + } + ver.Out.VerifierID = "" + continue } + activeFamilyVerifiers = append(activeFamilyVerifiers, ver) + verNOPAliases = append(verNOPAliases, ccvshared.NOPAlias(ver.NOPAlias)) + } + if len(activeFamilyVerifiers) == 0 { + continue } disableFinalityCheckers := disableFinalityCheckersPerFamily[family] @@ -755,16 +783,12 @@ func generateVerifierJobSpecs( // 1:1 verifier-to-aggregator mapping. For single-aggregator committees // this constraint doesn't apply — all verifiers share the one aggregator. if len(aggNames) > 1 { - if err := validateStandaloneVerifierNodeIndices(committeeName, committeeVerifiers, len(aggNames)); err != nil { + if err := validateStandaloneVerifierNodeIndices(committeeName, activeFamilyVerifiers, len(aggNames)); err != nil { return nil, err } } - for _, ver := range committeeVerifiers { - if ver.ChainFamily != family { - continue - } - + for _, ver := range activeFamilyVerifiers { allJobSpecs := make([]bootstrap.JobSpec, 0, len(aggNames)) for _, aggName := range aggNames { jobSpecID := ccvshared.NewVerifierJobID(ccvshared.NOPAlias(ver.NOPAlias), aggName, ccvshared.VerifierJobScope{CommitteeQualifier: committeeName}) @@ -783,7 +807,7 @@ func generateVerifierJobSpecs( L.Warn(). Str("spec", job.Spec). Str("undecoded fields", fmt.Sprintf("%v", md.Undecoded())). - Msg("Undecoded fields in executor job spec") + Msg("Undecoded fields in verifier job spec") return nil, fmt.Errorf("unknown fields in verifier job spec for %s aggregator: %v", ver.ContainerName, md.Undecoded()) } @@ -808,10 +832,13 @@ func generateVerifierJobSpecs( } ver.GeneratedConfig = string(configBytes) - // Store the VerifierID in the output for test access - if ver.Out != nil { - ver.Out.VerifierID = verCfg.VerifierID + // Store the VerifierID in the output for test access. CL-mode verifiers + // don't have an Out populated by launchStandaloneVerifiers, so initialize + // it here so VerifierID is observable regardless of mode. + if ver.Out == nil { + ver.Out = &committeeverifier.Output{} } + ver.Out.VerifierID = verCfg.VerifierID if sharedTLSCerts != nil && !ver.InsecureAggregatorConnection { ver.TLSCACertFile = sharedTLSCerts.CACertFile @@ -824,6 +851,48 @@ func generateVerifierJobSpecs( return verifierJobSpecs, nil } +func topologyVerifierNOPAliasesForCommitteeFamily( + topology *ccvdeployment.EnvironmentTopology, + committeeName string, + family string, +) (map[string]struct{}, error) { + activeNOPAliases := make(map[string]struct{}) + if topology == nil || topology.NOPTopology == nil { + return activeNOPAliases, nil + } + + var committee *ccvdeployment.CommitteeConfig + for _, candidate := range topology.NOPTopology.Committees { + if strings.EqualFold(candidate.Qualifier, committeeName) { + candidateCopy := candidate + committee = &candidateCopy + break + } + } + if committee == nil { + return nil, fmt.Errorf("committee %s not found in topology", committeeName) + } + + for selectorStr, chainCfg := range committee.ChainConfigs { + selector, err := strconv.ParseUint(selectorStr, 10, 64) + if err != nil { + return nil, fmt.Errorf("parse committee %s chain selector %q: %w", committeeName, selectorStr, err) + } + selectorFamily, err := chainsel.GetSelectorFamily(selector) + if err != nil { + return nil, fmt.Errorf("get chain family for selector %d: %w", selector, err) + } + if selectorFamily != family { + continue + } + for _, nopAlias := range chainCfg.NOPAliases { + activeNOPAliases[nopAlias] = struct{}{} + } + } + + return activeNOPAliases, nil +} + // NewEnvironment creates a new CCIP CCV environment locally in Docker. func NewEnvironment() (in *Cfg, err error) { ctx := context.Background() @@ -1089,7 +1158,7 @@ func NewEnvironment() (in *Cfg, err error) { } L.Info().Any("Selectors", selectors).Msg("Deploying for chain selectors") - topology := buildEnvironmentTopology(in, e) + topology := BuildEnvironmentTopology(in, e) if topology == nil { return nil, fmt.Errorf("failed to build environment topology") } @@ -1238,29 +1307,18 @@ func NewEnvironment() (in *Cfg, err error) { } } - // Generate aggregator configs using changesets (on-chain state as source of truth) - for _, aggregatorInput := range in.Aggregator { - aggregatorInput.SharedTLSCerts = sharedTLSCerts - - // Use changeset to generate committee config from on-chain state - instanceName := aggregatorInput.InstanceName() - cs := ccvchangesets.GenerateAggregatorConfig(ccvadapters.GetRegistry()) - output, err := cs.Apply(*e, ccvchangesets.GenerateAggregatorConfigInput{ - Topology: topology, - ServiceIdentifier: instanceName + "-aggregator", - CommitteeQualifier: aggregatorInput.CommitteeName, - }) - if err != nil { - return nil, fmt.Errorf("failed to generate aggregator config for %s (committee %s): %w", instanceName, aggregatorInput.CommitteeName, err) - } + verifierJobSpecs, err := configureOffchainFromTopology(e, in, topology, sharedTLSCerts, ds) + if err != nil { + return nil, fmt.Errorf("failed to regenerate offchain config: %w", err) + } + if err := fundStandaloneExecutorAddresses(ctx, in, impls); err != nil { + return nil, fmt.Errorf("failed to fund standalone executor addresses: %w", err) + } - // Get generated config from output datastore - aggCfg, err := ccvdeployment.GetAggregatorConfig(output.DataStore.Seal(), instanceName+"-aggregator") - if err != nil { - return nil, fmt.Errorf("failed to get aggregator config from output: %w", err) + for _, aggregatorInput := range in.Aggregator { + if sharedTLSCerts != nil { + aggregatorInput.SharedTLSCerts = sharedTLSCerts } - aggregatorInput.GeneratedCommittee = aggCfg - out, err := services.NewAggregator(aggregatorInput) if err != nil { return nil, fmt.Errorf("failed to create aggregator service for committee %s: %w", aggregatorInput.CommitteeName, err) @@ -1269,7 +1327,6 @@ func NewEnvironment() (in *Cfg, err error) { if out.TLSCACertFile != "" { in.AggregatorCACertFiles[aggregatorInput.CommitteeName] = out.TLSCACertFile } - e.DataStore = output.DataStore.Seal() } /////////////////////////////// @@ -1281,31 +1338,6 @@ func NewEnvironment() (in *Cfg, err error) { // start up the indexer(s) after the aggregators are up to avoid spamming of errors // in the logs when they start before the aggregators are up. /////////////////////////// - // Generate indexer config using changeset (on-chain state as source of truth). - // One shared config is generated; all indexers use the same config and duplicated secrets/auth. - if len(in.Aggregator) > 0 && len(in.Indexer) > 0 { - firstIdx := in.Indexer[0] - cs := ccvchangesets.GenerateIndexerConfig(ccvadapters.GetRegistry()) - output, err := cs.Apply(*e, ccvchangesets.GenerateIndexerConfigInput{ - ServiceIdentifier: "indexer", - CommitteeVerifierNameToQualifier: firstIdx.CommitteeVerifierNameToQualifier, - CCTPVerifierNameToQualifier: firstIdx.CCTPVerifierNameToQualifier, - LombardVerifierNameToQualifier: firstIdx.LombardVerifierNameToQualifier, - }) - if err != nil { - return nil, fmt.Errorf("failed to generate indexer config: %w", err) - } - - idxCfg, err := ccvdeployment.GetIndexerConfig(output.DataStore.Seal(), "indexer") - if err != nil { - return nil, fmt.Errorf("failed to get indexer config from output: %w", err) - } - e.DataStore = output.DataStore.Seal() - for _, idxIn := range in.Indexer { - idxIn.GeneratedCfg = idxCfg - } - } - if len(in.Indexer) < 1 { return nil, fmt.Errorf("at least one indexer is required") } @@ -1423,11 +1455,6 @@ func NewEnvironment() (in *Cfg, err error) { // START: Launch executors // ///////////////////////////// - executorJobSpecs, err := generateExecutorJobSpecs(ctx, e, in, selectors, impls, topology, ds) - if err != nil { - return nil, err - } - _, err = launchStandaloneExecutors(in.Executor, blockchainOutputs) if err != nil { return nil, fmt.Errorf("failed to create standalone executor: %w", err) @@ -1442,7 +1469,7 @@ func NewEnvironment() (in *Cfg, err error) { if err := registerExecutorsWithJD(ctx, in.Executor, jdInfra.OffchainClient); err != nil { return nil, err } - if err := proposeJobsToExecutors(ctx, in.Executor, executorJobSpecs, blockchainOutputs, jdInfra.OffchainClient); err != nil { + if err := proposeJobsToExecutors(ctx, in.Executor, blockchainOutputs, jdInfra.OffchainClient); err != nil { return nil, err } } @@ -1455,11 +1482,6 @@ func NewEnvironment() (in *Cfg, err error) { // START: Launch verifiers // ///////////////////////////// - verifierJobSpecs, err := generateVerifierJobSpecs(e, in, selectors, topology, sharedTLSCerts, ds) - if err != nil { - return nil, err - } - // Each verifier owns one aggregator (NodeIndex % numAggs). Select the // corresponding job spec so proposeJobsToStandaloneVerifiers gets a // single spec per container. @@ -1564,15 +1586,10 @@ func NewEnvironment() (in *Cfg, err error) { //////////////////////////////////////////////////// e.DataStore = ds.Seal() + in.CLDF.DataStore = e.DataStore - if in.JDInfra != nil { - if err := jobs.AcceptPendingJobs(ctx, in.ClientLookup); err != nil { - return nil, fmt.Errorf("failed to accept pending jobs: %w", err) - } - - if err := jobs.SyncAndVerifyJobProposals(e); err != nil { - return nil, fmt.Errorf("failed to sync/verify job proposals: %w", err) - } + if err := acceptPendingJobsAndSync(ctx, e, in); err != nil { + return nil, fmt.Errorf("failed to accept pending jobs: %w", err) } timeTrack.Print() @@ -1901,7 +1918,6 @@ func registerExecutorsWithJD(ctx context.Context, executors []*executorsvc.Input func proposeJobsToExecutors( ctx context.Context, executors []*executorsvc.Input, - executorJobSpecs map[string]bootstrap.JobSpec, blockchainOutputs []*blockchain.Output, jdClient offchain.Client, ) error { @@ -1935,10 +1951,10 @@ func proposeJobsToExecutors( return fmt.Errorf("failed to load chain config for family %s: %w", exec.ChainFamily, err) } - baseJobSpec, ok := executorJobSpecs[exec.ContainerName] - if !ok { + if len(exec.GeneratedJobSpecs) == 0 { return fmt.Errorf("no job spec found for executor %s", exec.ContainerName) } + baseJobSpec := exec.GeneratedJobSpecs[0] jobSpec, err := executorsvc.RebuildExecutorJobSpecWithBlockchainInfos(baseJobSpec, blockchainInfos) if err != nil { diff --git a/build/devenv/environment_change_requirements.go b/build/devenv/environment_change_requirements.go new file mode 100644 index 000000000..97560843a --- /dev/null +++ b/build/devenv/environment_change_requirements.go @@ -0,0 +1,45 @@ +package ccv + +import ( + "fmt" + + "github.com/smartcontractkit/chainlink-ccv/build/devenv/services" + ccvshared "github.com/smartcontractkit/chainlink-ccv/deployment/shared" +) + +// RequireFullCLModeForEnvironmentChangeReconcile returns an error if the loaded env-out is not a +// fully Chainlink-backed devenv (topology NOPs CL, verifier and executor services in CL mode, node sets present). +// Environment-change reconcile E2E tests use this to skip standalone smoke env-outs. +func RequireFullCLModeForEnvironmentChangeReconcile(in *Cfg) error { + if in == nil { + return fmt.Errorf("cfg is nil") + } + if in.EnvironmentTopology == nil || in.EnvironmentTopology.NOPTopology == nil { + return fmt.Errorf("environment_topology.nop_topology is required") + } + for _, nop := range in.EnvironmentTopology.NOPTopology.NOPs { + if nop.GetMode() != ccvshared.NOPModeCL { + ident := nop.Alias + if ident == "" { + ident = nop.Name + } + return fmt.Errorf("NOP %q must use topology mode %q for environment-change reconcile tests (got %q)", + ident, ccvshared.NOPModeCL, nop.GetMode()) + } + } + for _, v := range in.Verifier { + if v.Mode != services.CL { + return fmt.Errorf("verifier for nop_alias %q must use mode %q (got %q)", + v.NOPAlias, services.CL, v.Mode) + } + } + for _, ex := range in.Executor { + if ex.Mode != services.CL { + return fmt.Errorf("executor must use mode %q (got %q)", services.CL, ex.Mode) + } + } + if len(in.NodeSets) == 0 { + return fmt.Errorf("at least one nodeset is required for CL mode") + } + return nil +} diff --git a/build/devenv/evm/impl.go b/build/devenv/evm/impl.go index fdf631ad7..ebf637e84 100644 --- a/build/devenv/evm/impl.go +++ b/build/devenv/evm/impl.go @@ -305,10 +305,11 @@ func (m *CCIP17EVM) getOrCreateOffRampPoller() (*eventPoller[cciptestinterfaces. messageID: event.MessageId, } events[key] = cciptestinterfaces.ExecutionStateChangedEvent{ - MessageID: event.MessageId, - MessageNumber: event.MessageNumber, - State: cciptestinterfaces.MessageExecutionState(event.State), - ReturnData: event.ReturnData, + SourceChainSelector: protocol.ChainSelector(event.SourceChainSelector), + MessageID: event.MessageId, + MessageNumber: event.MessageNumber, + State: cciptestinterfaces.MessageExecutionState(event.State), + ReturnData: event.ReturnData, } } @@ -485,6 +486,27 @@ func (m *CCIP17EVM) ConfirmExecOnDest(ctx context.Context, from uint64, key ccip } } +// WaitForExecutionState polls the OffRamp's getExecutionState view until the +// message reaches the target state or the context expires. This bypasses the +// event-poller cache which only stores the first event per (chain, seqNo) key +// and is therefore unsuitable for detecting state transitions (e.g. FAILURE->SUCCESS). +func (m *CCIP17EVM) WaitForExecutionState(ctx context.Context, msgID protocol.Bytes32, target cciptestinterfaces.MessageExecutionState, interval time.Duration) error { + ticker := time.NewTicker(interval) + defer ticker.Stop() + for { + state, err := m.offRamp.GetExecutionState(&bind.CallOpts{Context: ctx}, msgID) + if err == nil && cciptestinterfaces.MessageExecutionState(state) == target { + return nil + } + select { + case <-ctx.Done(): + return fmt.Errorf("timed out waiting for execution state %s on message %x (last state=%d, last err=%v)", + target, msgID, state, err) + case <-ticker.C: + } + } +} + func (m *CCIP17EVM) GetEOAReceiverAddress() (protocol.UnknownAddress, error) { // returns the same address for each chain for now - we might need to extend this in the future if we'd ever // need to access any funds on the EOA itself. @@ -1057,6 +1079,7 @@ func (m *CCIP17EVMConfig) GetDeployChainContractsCfg(env *deployment.Environment return ccipChangesets.DeployChainContractsPerChainCfg{ DeployerContract: create2Ref.Address, DeployerKeyOwned: true, + DeployTestRouter: true, RMNRemote: adapters.RMNRemoteDeployParams{ Version: semver.MustParse(rmn_remote.Deploy.Version()), }, @@ -1362,21 +1385,21 @@ func (m *CCIP17EVM) GetMaxDataBytes(ctx context.Context, remoteChainSelector uin return destChainConfig.MaxDataBytes, nil } +func (m *CCIP17EVMConfig) GetChainLaneProfile(_ *deployment.Environment, selector uint64) (cciptestinterfaces.ChainLaneProfile, error) { + return cciptestinterfaces.ChainLaneProfile{ + FeeQuoterDestChainConfig: ccipChangesets.FeeQuoterDestChainConfigOverrides{ + USDPerUnitGas: big.NewInt(1e6), + }, + }, nil +} + func (m *CCIP17EVMConfig) GetConnectionProfile(_ *deployment.Environment, selector uint64) (lanes.ChainDefinition, lanes.CommitteeVerifierRemoteChainInput, error) { + override := evmFeeQuoterDestChainConfigOverride(selector) chainDef := lanes.ChainDefinition{ - Selector: selector, - AddressBytesLength: 20, - BaseExecutionGasCost: 150_000, - FeeQuoterDestChainConfigOverrides: evmFeeQuoterDestChainConfigOverride(selector), + FeeQuoterDestChainConfigOverrides: override, ExecutorDestChainConfig: lanes.ExecutorDestChainConfig{ Enabled: true, }, - DefaultExecutor: datastore.AddressRef{ - Type: datastore.ContractType(sequences.ExecutorProxyType), - Version: semver.MustParse(proxy.Deploy.Version()), - Qualifier: devenvcommon.DefaultExecutorQualifier, - ChainSelector: selector, - }, DefaultInboundCCVs: []datastore.AddressRef{ { Type: datastore.ContractType(versioned_verifier_resolver.CommitteeVerifierResolverType), @@ -1394,11 +1417,9 @@ func (m *CCIP17EVMConfig) GetConnectionProfile(_ *deployment.Environment, select }, }, } - cvConfig := lanes.CommitteeVerifierRemoteChainInput{ GasForVerification: CommitteeVerifierGasForVerification, } - return chainDef, cvConfig, nil } @@ -1423,14 +1444,6 @@ func evmFeeQuoterDestChainConfigOverride(selector uint64) *lanes.FeeQuoterDestCh return &override } -func (m *CCIP17EVMConfig) GetChainLaneProfile(_ *deployment.Environment, selector uint64) (cciptestinterfaces.ChainLaneProfile, error) { - return cciptestinterfaces.ChainLaneProfile{ - FeeQuoterDestChainConfig: ccipChangesets.FeeQuoterDestChainConfigOverrides{ - USDPerUnitGas: big.NewInt(1e6), - }, - }, nil -} - func (m *CCIP17EVMConfig) PostConnect(e *deployment.Environment, selector uint64, remoteSelectors []uint64) error { if err := m.ConfigureUSDCAndLombardForTransfer(e, selector, remoteSelectors); err != nil { return fmt.Errorf("configure USDC/Lombard for transfer: %w", err) @@ -1707,10 +1720,11 @@ func (m *CCIP17EVM) ManuallyExecuteMessage( continue } event = cciptestinterfaces.ExecutionStateChangedEvent{ - MessageID: parsedLog.MessageId, - MessageNumber: parsedLog.MessageNumber, - State: cciptestinterfaces.MessageExecutionState(parsedLog.State), - ReturnData: parsedLog.ReturnData, + SourceChainSelector: protocol.ChainSelector(parsedLog.SourceChainSelector), + MessageID: parsedLog.MessageId, + MessageNumber: parsedLog.MessageNumber, + State: cciptestinterfaces.MessageExecutionState(parsedLog.State), + ReturnData: parsedLog.ReturnData, } break } diff --git a/build/devenv/implcommon.go b/build/devenv/implcommon.go index 17df44b54..a17b06012 100644 --- a/build/devenv/implcommon.go +++ b/build/devenv/implcommon.go @@ -146,10 +146,10 @@ func connectAllChainsCanonical( topology *ccvdeployment.EnvironmentTopology, ) error { if len(blockchains) != len(impls) { - return fmt.Errorf("connectAllChains: mismatched lengths: %d impls and %d blockchains", len(impls), len(blockchains)) + return fmt.Errorf("connectAllChainsCanonical: mismatched lengths: %d impls and %d blockchains", len(impls), len(blockchains)) } if len(selectors) == 0 { - return fmt.Errorf("connectAllChains: selectors must be non-empty") + return fmt.Errorf("connectAllChainsCanonical: selectors must be non-empty") } profiles := make(map[uint64]chainProfile, len(impls)) @@ -383,15 +383,15 @@ func connectAllChainsLegacy( mcmsReaderRegistry := changesetscore.GetRegistry() connectChainsCS := lanes.ConnectChains(laneAdapterRegistry, mcmsReaderRegistry) - cfg := lanes.ConnectChainsConfig{ + connectCfg := lanes.ConnectChainsConfig{ Lanes: laneConfigs, CommitteePopulator: populator, } - if err := connectChainsCS.VerifyPreconditions(*e, cfg); err != nil { - return fmt.Errorf("connect chains precondition check failed: %w", err) + if err := connectChainsCS.VerifyPreconditions(*e, connectCfg); err != nil { + return fmt.Errorf("connectAllChainsLegacy: precondition check failed: %w", err) } - if _, err := connectChainsCS.Apply(*e, cfg); err != nil { - return fmt.Errorf("configure chains for lanes: %w", err) + if _, err := connectChainsCS.Apply(*e, connectCfg); err != nil { + return fmt.Errorf("connectAllChainsLegacy: connect chains: %w", err) } for _, sel := range orderedSelectors { diff --git a/build/devenv/jobs/rotate_key.go b/build/devenv/jobs/rotate_key.go new file mode 100644 index 000000000..bc176997a --- /dev/null +++ b/build/devenv/jobs/rotate_key.go @@ -0,0 +1,262 @@ +package jobs + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "strings" + "time" + + "github.com/sethvargo/go-retry" + + "github.com/smartcontractkit/chainlink-deployments-framework/offchain" + nodev1 "github.com/smartcontractkit/chainlink-protos/job-distributor/v1/node" + "github.com/smartcontractkit/chainlink-testing-framework/framework/clclient" + sdkclient "github.com/smartcontractkit/chainlink/deployment/environment/web/sdk/client" +) + +// RotateOCR2KeyBundle creates a new EVM OCR2 key bundle on the CL node, updates +// the JD chain configs for the provided chain IDs to reference the new bundle, and +// returns the previous and new on-chain signing addresses. +// +// The function: +// 1. Queries JD for the current on-chain signing address (old address). +// 2. Creates a new OCR2 key bundle on the CL node. +// 3. Deletes all existing feeds-manager chain configs for the provided chain IDs. +// 4. Re-creates them with the new bundle ID. +// 5. Waits for JD to reflect the new signing address. +// +// chainIDs must be the decimal string chain IDs (e.g. "1", "100") that already have +// a chain config registered for nodeID in JD. +func RotateOCR2KeyBundle( + ctx context.Context, + clClient *clclient.ChainlinkClient, + jdClient offchain.Client, + nodeID string, + chainIDs []string, +) (oldSigningAddr, newSigningAddr string, err error) { + // 1. Capture current signing address before mutation. + oldSigningAddr, err = fetchSigningAddrFromJD(ctx, jdClient, nodeID) + if err != nil { + return "", "", fmt.Errorf("rotate key: fetch current signing address: %w", err) + } + + // 2. Create new OCR2 key bundle on the CL node. + newKey, _, err := clClient.CreateOCR2Key("evm") + if err != nil { + return "", "", fmt.Errorf("rotate key: create OCR2 key: %w", err) + } + newBundleID := newKey.Data.ID + + // 3. Build SDK client (authenticated) to manage feeds-manager chain configs. + gqlClient, err := NewSDKClient(ctx, clClient) + if err != nil { + return "", "", fmt.Errorf("rotate key: create SDK client: %w", err) + } + + // 4. Get feeds manager ID and existing chain config IDs. + fmID, err := getFeedsManagerID(ctx, gqlClient) + if err != nil { + return "", "", fmt.Errorf("rotate key: get feeds manager: %w", err) + } + + chainConfigIDs, err := listFMChainConfigIDsForChains(ctx, clClient, fmID, chainIDs) + if err != nil { + return "", "", fmt.Errorf("rotate key: list chain config IDs: %w", err) + } + + // 5. Delete existing chain configs. + for _, id := range chainConfigIDs { + if err := gqlClient.DeleteJobDistributorChainConfig(ctx, id); err != nil { + return "", "", fmt.Errorf("rotate key: delete chain config %s: %w", id, err) + } + } + + // 6. Re-create chain configs with new bundle ID. + p2pPeerID, err := gqlClient.FetchP2PPeerID(ctx) + if err != nil { + return "", "", fmt.Errorf("rotate key: fetch P2P peer ID: %w", err) + } + + for _, chainID := range chainIDs { + accountAddr, err := gqlClient.FetchAccountAddress(ctx, chainID) + if err != nil { + return "", "", fmt.Errorf("rotate key: fetch account address for chain %s: %w", chainID, err) + } + input := sdkclient.JobDistributorChainConfigInput{ + JobDistributorID: fmID, + ChainID: chainID, + ChainType: "EVM", + AccountAddr: *accountAddr, + AdminAddr: *accountAddr, + Ocr2Enabled: true, + Ocr2P2PPeerID: *p2pPeerID, + Ocr2KeyBundleID: newBundleID, + Ocr2Plugins: `{"commit":false,"execute":false,"median":false,"mercury":false}`, + } + if err := createAndVerifyChainConfig(ctx, gqlClient, jdClient, nodeID, input); err != nil { + return "", "", fmt.Errorf("rotate key: create chain config for chain %s: %w", chainID, err) + } + } + + // 7. Wait for JD to reflect the new signing address. + newSigningAddr, err = waitForNewSigningAddr(ctx, jdClient, nodeID, oldSigningAddr) + if err != nil { + return "", "", fmt.Errorf("rotate key: wait for new signing address: %w", err) + } + + return oldSigningAddr, newSigningAddr, nil +} + +// fetchSigningAddrFromJD returns the on-chain signing address for the first EVM chain config found in JD. +func fetchSigningAddrFromJD(ctx context.Context, jdClient offchain.Client, nodeID string) (string, error) { + resp, err := jdClient.ListNodeChainConfigs(ctx, &nodev1.ListNodeChainConfigsRequest{ + Filter: &nodev1.ListNodeChainConfigsRequest_Filter{NodeIds: []string{nodeID}}, + }) + if err != nil { + return "", fmt.Errorf("list node chain configs: %w", err) + } + for _, cfg := range resp.ChainConfigs { + if cfg.Ocr2Config != nil && cfg.Ocr2Config.OcrKeyBundle != nil { + if addr := cfg.Ocr2Config.OcrKeyBundle.OnchainSigningAddress; addr != "" { + return addr, nil + } + } + } + return "", errors.New("no EVM chain config with signing address found for node") +} + +// waitForNewSigningAddr polls JD until the node's signing address differs from oldAddr. +func waitForNewSigningAddr(ctx context.Context, jdClient offchain.Client, nodeID, oldAddr string) (string, error) { + backoff := retry.WithMaxDuration(90*time.Second, retry.NewExponential(2*time.Second)) + var newAddr string + err := retry.Do(ctx, backoff, func(ctx context.Context) error { + addr, err := fetchSigningAddrFromJD(ctx, jdClient, nodeID) + if err != nil { + return retry.RetryableError(err) + } + if strings.EqualFold(addr, oldAddr) { + return retry.RetryableError(errors.New("signing address not yet updated in JD")) + } + newAddr = addr + return nil + }) + return newAddr, err +} + +// getFeedsManagerID returns the first feeds manager ID visible to this CL node. +func getFeedsManagerID(ctx context.Context, gqlClient sdkclient.Client) (string, error) { + jds, err := gqlClient.ListJobDistributors(ctx) + if err != nil { + return "", fmt.Errorf("list job distributors: %w", err) + } + if len(jds.FeedsManagers.Results) == 0 { + return "", errors.New("no feeds manager found") + } + return jds.FeedsManagers.Results[0].Id, nil +} + +// fmChainConfigEntry is a minimal representation of a feeds-manager chain config. +type fmChainConfigEntry struct { + ID string `json:"id"` + ChainID string `json:"chainID"` +} + +// listFMChainConfigIDsForChains returns the graphql IDs of existing feeds-manager chain configs +// that match the given chain IDs. It authenticates against the CL node and issues a raw +// GraphQL query because the SDK client's FeedsManagerParts fragment omits chainConfigs. +func listFMChainConfigIDsForChains(ctx context.Context, clClient *clclient.ChainlinkClient, fmID string, chainIDs []string) ([]string, error) { + cookie, err := clSessionCookie(ctx, clClient.URL(), clClient.Config.Email, clClient.Config.Password) + if err != nil { + return nil, fmt.Errorf("auth: %w", err) + } + + entries, err := queryFMChainConfigs(ctx, clClient.URL(), cookie, fmID) + if err != nil { + return nil, err + } + + chainIDSet := make(map[string]struct{}, len(chainIDs)) + for _, c := range chainIDs { + chainIDSet[c] = struct{}{} + } + + ids := make([]string, 0, len(entries)) + for _, e := range entries { + if _, ok := chainIDSet[e.ChainID]; ok { + ids = append(ids, e.ID) + } + } + return ids, nil +} + +// clSessionCookie authenticates with the CL node REST API and returns the session cookie value. +func clSessionCookie(ctx context.Context, baseURL, email, password string) (string, error) { + body, _ := json.Marshal(map[string]string{"email": email, "password": password}) + req, err := http.NewRequestWithContext(ctx, http.MethodPost, baseURL+"/sessions", bytes.NewReader(body)) + if err != nil { + return "", err + } + req.Header.Set("Content-Type", "application/json") + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return "", fmt.Errorf("POST /sessions: %w", err) + } + defer func() { _, _ = io.Copy(io.Discard, resp.Body); _ = resp.Body.Close() }() + + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + return "", fmt.Errorf("POST /sessions returned %d", resp.StatusCode) + } + // Use the raw Set-Cookie header as the SDK client does. + raw := resp.Header.Get("Set-Cookie") + if raw == "" { + return "", errors.New("no Set-Cookie header in session response") + } + return strings.SplitN(raw, ";", 2)[0], nil +} + +// queryFMChainConfigs issues a raw GraphQL query to retrieve chain config IDs for a feeds manager. +func queryFMChainConfigs(ctx context.Context, baseURL, cookie, fmID string) ([]fmChainConfigEntry, error) { + const gql = `{"query":"query GetFMCC($id:ID!){feedsManager(id:$id){...on FeedsManager{chainConfigs{id chainID}}}}","variables":{"id":"%s"}}` + body := strings.NewReader(fmt.Sprintf(gql, fmID)) + + req, err := http.NewRequestWithContext(ctx, http.MethodPost, baseURL+"/query", body) + if err != nil { + return nil, err + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Cookie", cookie) + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return nil, fmt.Errorf("POST /query: %w", err) + } + defer func() { _, _ = io.Copy(io.Discard, resp.Body); _ = resp.Body.Close() }() + + var result struct { + Data struct { + FeedsManager struct { + ChainConfigs []fmChainConfigEntry `json:"chainConfigs"` + } `json:"feedsManager"` + } `json:"data"` + Errors []struct { + Message string `json:"message"` + } `json:"errors"` + } + if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { + return nil, fmt.Errorf("decode graphql response: %w", err) + } + if len(result.Errors) > 0 { + msgs := make([]string, len(result.Errors)) + for i, e := range result.Errors { + msgs[i] = e.Message + } + return nil, fmt.Errorf("graphql errors: %s", strings.Join(msgs, "; ")) + } + return result.Data.FeedsManager.ChainConfigs, nil +} diff --git a/build/devenv/lane_topology.go b/build/devenv/lane_topology.go new file mode 100644 index 000000000..e6a962b4c --- /dev/null +++ b/build/devenv/lane_topology.go @@ -0,0 +1,240 @@ +package ccv + +import ( + "fmt" + "sort" + + chainsel "github.com/smartcontractkit/chain-selectors" + + ccipChangesets "github.com/smartcontractkit/chainlink-ccip/deployment/v2_0_0/changesets" + ccipOffchain "github.com/smartcontractkit/chainlink-ccip/deployment/v2_0_0/offchain" + "github.com/smartcontractkit/chainlink-ccv/build/devenv/cciptestinterfaces" + "github.com/smartcontractkit/chainlink-deployments-framework/deployment" + "github.com/smartcontractkit/chainlink-testing-framework/framework/components/blockchain" +) + +// LanePartialConfigOverrides are per-local-chain deltas layered on GetChainLaneProfile baselines. +type LanePartialConfigOverrides struct { + CommitteeRemotePatches map[uint64]ccipChangesets.CommitteeVerifierRemoteChainConfig + UseTestRouter bool +} + +func mergeCommitteeVerifierRemoteChainConfigForReconcile( + base, patch ccipChangesets.CommitteeVerifierRemoteChainConfig, +) ccipChangesets.CommitteeVerifierRemoteChainConfig { + out := base + if patch.AllowlistEnabled != nil { + out.AllowlistEnabled = patch.AllowlistEnabled + } + if len(patch.AddedAllowlistedSenders) > 0 { + out.AddedAllowlistedSenders = patch.AddedAllowlistedSenders + } + if len(patch.RemovedAllowlistedSenders) > 0 { + out.RemovedAllowlistedSenders = patch.RemovedAllowlistedSenders + } + if patch.GasForVerification != nil { + out.GasForVerification = patch.GasForVerification + } + if patch.FeeUSDCents != nil { + out.FeeUSDCents = patch.FeeUSDCents + } + if patch.PayloadSizeBytes != nil { + out.PayloadSizeBytes = patch.PayloadSizeBytes + } + return out +} + +func committeeVerifiersForTopology( + topology *ccipOffchain.EnvironmentTopology, + remoteSelectors []uint64, +) ([]ccipChangesets.CommitteeVerifierInputConfig, error) { + if topology == nil || topology.NOPTopology == nil { + return nil, fmt.Errorf("topology with NOPTopology is required") + } + + qualifiers := make([]string, 0, len(topology.NOPTopology.Committees)) + for qualifier := range topology.NOPTopology.Committees { + qualifiers = append(qualifiers, qualifier) + } + sort.Strings(qualifiers) + + verifiers := make([]ccipChangesets.CommitteeVerifierInputConfig, 0, len(qualifiers)) + for _, qualifier := range qualifiers { + remoteCV := make(map[uint64]ccipChangesets.CommitteeVerifierRemoteChainConfig, len(remoteSelectors)) + for _, rs := range remoteSelectors { + remoteCV[rs] = ccipChangesets.CommitteeVerifierRemoteChainConfig{} + } + verifiers = append(verifiers, ccipChangesets.CommitteeVerifierInputConfig{ + CommitteeQualifier: qualifier, + RemoteChains: remoteCV, + }) + } + return verifiers, nil +} + +// partialChainConfigFromProfile builds one PartialChainConfig for localSelector +// using ChainLaneProfile data. Empty committee remote configs and adapter-backed +// remote chain fields are resolved by the changeset defaults; profiles only carry +// explicit fee quoter, executor, and CCV overrides. +func partialChainConfigFromProfile( + localSelector uint64, + remoteSelectors []uint64, + topology *ccipOffchain.EnvironmentTopology, + local cciptestinterfaces.ChainLaneProfile, + profiles map[uint64]chainProfile, + o LanePartialConfigOverrides, +) (ccipChangesets.PartialChainConfig, error) { + committeeVerifiers, err := committeeVerifiersForTopology(topology, remoteSelectors) + if err != nil { + return ccipChangesets.PartialChainConfig{}, err + } + if len(o.CommitteeRemotePatches) > 0 { + for i := range committeeVerifiers { + merged := make(map[uint64]ccipChangesets.CommitteeVerifierRemoteChainConfig, len(committeeVerifiers[i].RemoteChains)) + for rs, cfg := range committeeVerifiers[i].RemoteChains { + merged[rs] = cfg + if patch, ok := o.CommitteeRemotePatches[rs]; ok { + merged[rs] = mergeCommitteeVerifierRemoteChainConfigForReconcile(merged[rs], patch) + } + } + committeeVerifiers[i].RemoteChains = merged + } + } + + remoteChains := make(map[uint64]ccipChangesets.PartialRemoteChainConfig, len(remoteSelectors)) + for _, rs := range remoteSelectors { + remoteProfile, ok := profiles[rs] + if !ok { + return ccipChangesets.PartialChainConfig{}, fmt.Errorf("missing profile for remote selector %d", rs) + } + remote := remoteProfile.profile + remoteChains[rs] = ccipChangesets.PartialRemoteChainConfig{ + DefaultInboundCCVs: local.DefaultInboundCCVs, + DefaultOutboundCCVs: local.DefaultOutboundCCVs, + DefaultExecutorQualifier: local.DefaultExecutorQualifier, + FeeQuoterDestChainConfig: remote.FeeQuoterDestChainConfig, + ExecutorDestChainConfig: local.ExecutorDestChainConfig, + } + } + + return ccipChangesets.PartialChainConfig{ + ChainSelector: localSelector, + CommitteeVerifiers: committeeVerifiers, + RemoteChains: remoteChains, + }, nil +} + +// buildConnectionProfilesFromImpls resolves selectors from blockchains and impls, +// builds the peer mesh remote selector lists, and loads GetChainLaneProfile per chain. +func buildConnectionProfilesFromImpls( + impls []cciptestinterfaces.CCIP17Configuration, + blockchains []*blockchain.Input, + selectors []uint64, + e *deployment.Environment, +) ([]uint64, map[uint64]chainProfile, error) { + if len(blockchains) != len(impls) { + return nil, nil, fmt.Errorf("connection profiles: mismatched lengths: %d impls and %d blockchains", len(impls), len(blockchains)) + } + profiles := make(map[uint64]chainProfile, len(impls)) + orderedSelectors := make([]uint64, 0, len(impls)) + for i, impl := range impls { + networkInfo, err := chainsel.GetChainDetailsByChainIDAndFamily(blockchains[i].ChainID, impl.ChainFamily()) + if err != nil { + return nil, nil, fmt.Errorf("chain %d: %w", i, err) + } + sel := networkInfo.ChainSelector + remotes := make([]uint64, 0, len(selectors)) + for _, s := range selectors { + if s != sel { + remotes = append(remotes, s) + } + } + profile, err := impl.GetChainLaneProfile(e, sel) + if err != nil { + return nil, nil, fmt.Errorf("get chain lane profile for chain %d: %w", sel, err) + } + profiles[sel] = chainProfile{ + remotes: remotes, + impl: impl, + profile: profile, + } + orderedSelectors = append(orderedSelectors, sel) + } + return orderedSelectors, profiles, nil +} + +func assertConnectionProfilesCoverSelectors(orderedSelectors, selectors []uint64) error { + want := make(map[uint64]struct{}, len(selectors)) + for _, s := range selectors { + want[s] = struct{}{} + } + got := make(map[uint64]struct{}, len(orderedSelectors)) + for _, s := range orderedSelectors { + got[s] = struct{}{} + } + if len(want) != len(got) { + return fmt.Errorf("reconfigure lanes: selectors set does not match configured chains") + } + for s := range want { + if _, ok := got[s]; !ok { + return fmt.Errorf("reconfigure lanes: selector %d not found in blockchains/impls", s) + } + } + return nil +} + +func lanePartialOverridesFromReconfigureParams(params ReconfigureLanesParams, localSel uint64) LanePartialConfigOverrides { + var o LanePartialConfigOverrides + if params.CommitteePatches != nil { + if inner, ok := params.CommitteePatches[localSel]; ok && len(inner) > 0 { + o.CommitteeRemotePatches = inner + } + } + if params.TestRouterByLane != nil { + if inner, ok := params.TestRouterByLane[localSel]; ok { + for _, v := range inner { + if v { + o.UseTestRouter = true + break + } + } + } + } + return o +} + +// buildPartialChainConfigsFromProfiles builds ConfigureChainsForLanesFromTopology +// partial configs for every chain in orderedSelectors. Zero ReconfigureLanesParams +// matches fresh deploy (production Router lanes). +func buildPartialChainConfigsFromProfiles( + topology *ccipOffchain.EnvironmentTopology, + orderedSelectors []uint64, + profiles map[uint64]chainProfile, + params ReconfigureLanesParams, +) ([]ccipChangesets.PartialChainConfig, bool, error) { + useTestRouter := false + chains := make([]ccipChangesets.PartialChainConfig, 0, len(orderedSelectors)) + for _, localSel := range orderedSelectors { + entry, ok := profiles[localSel] + if !ok { + return nil, false, fmt.Errorf("no profile for local chain %d", localSel) + } + o := lanePartialOverridesFromReconfigureParams(params, localSel) + if o.UseTestRouter { + useTestRouter = true + } + pc, err := partialChainConfigFromProfile( + localSel, + entry.remotes, + topology, + entry.profile, + profiles, + o, + ) + if err != nil { + return nil, false, fmt.Errorf("partial chain config for selector %d: %w", localSel, err) + } + chains = append(chains, pc) + } + return chains, useTestRouter, nil +} diff --git a/build/devenv/reconcile_offchain.go b/build/devenv/reconcile_offchain.go new file mode 100644 index 000000000..b06c5b150 --- /dev/null +++ b/build/devenv/reconcile_offchain.go @@ -0,0 +1,382 @@ +package ccv + +import ( + "context" + "fmt" + + "google.golang.org/grpc/credentials/insecure" + + ccipAdapters "github.com/smartcontractkit/chainlink-ccip/deployment/v2_0_0/adapters" + ccipChangesets "github.com/smartcontractkit/chainlink-ccip/deployment/v2_0_0/changesets" + "github.com/smartcontractkit/chainlink-ccv/bootstrap" + "github.com/smartcontractkit/chainlink-ccv/build/devenv/cciptestinterfaces" + "github.com/smartcontractkit/chainlink-ccv/build/devenv/jobs" + "github.com/smartcontractkit/chainlink-ccv/build/devenv/services" + ccvdeployment "github.com/smartcontractkit/chainlink-ccv/deployment" + ccvshared "github.com/smartcontractkit/chainlink-ccv/deployment/shared" + "github.com/smartcontractkit/chainlink-deployments-framework/datastore" + "github.com/smartcontractkit/chainlink-deployments-framework/deployment" + "github.com/smartcontractkit/chainlink-deployments-framework/offchain" + cldfjd "github.com/smartcontractkit/chainlink-deployments-framework/offchain/jd" + nodev1 "github.com/smartcontractkit/chainlink-protos/job-distributor/v1/node" + "github.com/smartcontractkit/chainlink-testing-framework/framework/components/blockchain" +) + +// ConfigureOffchainOptions controls off-chain configuration after an environment change (on-chain or topology). +// +// By default (RestartTomlConsumers nil), after TOML is regenerated this function updates aggregator +// generated TOML in running containers, writes fresh standalone config files to disk, then restarts +// only TOML-bound Docker services: aggregators, +// indexers, standalone executors, and standalone verifiers. +// Set RestartTomlConsumers to a non-nil false when Docker is unavailable or to avoid restarting consumers +// (e.g. out-of-band reload or diagnostics). +type ConfigureOffchainOptions struct { + FundExecutors bool + + // RestartTomlConsumers when nil means true: docker restart TOML-bound service containers after a successful reconcile. + RestartTomlConsumers *bool +} + +func (o ConfigureOffchainOptions) effectiveRestartTomlConsumers() bool { + if o.RestartTomlConsumers == nil { + return true + } + return *o.RestartTomlConsumers +} + +// ReconfigureOffchainOnly regenerates off-chain configs (aggregator, indexer, verifier job specs) +// and restarts TOML-bound services without applying any on-chain lane changes. Intended for tests +// that need to update the verifier signing key before the on-chain quorum is changed. +func ReconfigureOffchainOnly( + ctx context.Context, + e *deployment.Environment, + in *Cfg, + topology *ccvdeployment.EnvironmentTopology, + impls []cciptestinterfaces.CCIP17Configuration, + opts ConfigureOffchainOptions, +) error { + return configureOffchainAfterOnChainChange(ctx, e, in, impls, topology, nil, opts) +} + +// ConfigureTopologyLanesAndOffchain applies on-chain lane changes from topology and params, then runs off-chain +// regeneration (aggregator, indexer, executor, verifier) and restarts TOML-bound Docker services by default. +func ConfigureTopologyLanesAndOffchain( + ctx context.Context, + e *deployment.Environment, + in *Cfg, + topology *ccvdeployment.EnvironmentTopology, + selectors []uint64, + blockchains []*blockchain.Input, + impls []cciptestinterfaces.CCIP17Configuration, + laneParams ReconfigureLanesParams, + sharedTLSCerts *services.TLSCertPaths, + offchainOpts ConfigureOffchainOptions, +) error { + if err := reconfigureLanesFromTopology(ctx, e, convertTopologyToCCIP(topology), selectors, blockchains, impls, laneParams); err != nil { + return fmt.Errorf("configure topology lanes and offchain: on-chain: %w", err) + } + if err := configureOffchainAfterOnChainChange(ctx, e, in, impls, topology, sharedTLSCerts, offchainOpts); err != nil { + return fmt.Errorf("configure topology lanes and offchain: off-chain: %w", err) + } + return nil +} + +// configureOffchainAfterOnChainChange re-runs GenerateAggregatorConfig, GenerateIndexerConfig, +// executor and verifier job-spec changesets, merges outputs into e.DataStore, and by default restarts TOML-bound Docker services. +func configureOffchainAfterOnChainChange( + ctx context.Context, + e *deployment.Environment, + in *Cfg, + impls []cciptestinterfaces.CCIP17Configuration, + topology *ccvdeployment.EnvironmentTopology, + sharedTLSCerts *services.TLSCertPaths, + opts ConfigureOffchainOptions, +) error { + if e == nil || in == nil || topology == nil { + return fmt.Errorf("reconcile: environment, config, and topology are required") + } + mds := datastore.NewMemoryDataStore() + if err := mds.Merge(e.DataStore); err != nil { + return fmt.Errorf("reconcile: merge initial datastore: %w", err) + } + if _, err := configureOffchainFromTopology(e, in, topology, sharedTLSCerts, mds); err != nil { + return fmt.Errorf("reconcile: configure offchain: %w", err) + } + if opts.FundExecutors { + if err := fundStandaloneExecutorAddresses(ctx, in, impls); err != nil { + return fmt.Errorf("reconcile: fund standalone executors: %w", err) + } + } + if err := acceptPendingJobsAndSync(ctx, e, in); err != nil { + return fmt.Errorf("reconcile: accept pending jobs: %w", err) + } + + if opts.effectiveRestartTomlConsumers() { + if err := configureTomlBoundServiceFiles(ctx, in); err != nil { + return fmt.Errorf("reconcile: configure TOML-bound services: %w", err) + } + if err := restartTomlBoundServices(ctx, in); err != nil { + return fmt.Errorf("reconcile: restart toml-bound services: %w", err) + } + } + return nil +} + +type restartable interface { + Restart(context.Context) error +} + +type refreshable interface { + RefreshConfig(context.Context) error +} + +func restartTomlBoundServices(ctx context.Context, in *Cfg) error { + if in == nil { + return nil + } + restartables := make([]restartable, 0, len(in.Aggregator)+len(in.Indexer)+len(in.Executor)+len(in.Verifier)) + for _, service := range in.Aggregator { + if service != nil { + restartables = append(restartables, service) + } + } + for _, service := range in.Indexer { + if service != nil { + restartables = append(restartables, service) + } + } + for _, service := range in.Executor { + if service != nil { + restartables = append(restartables, service) + } + } + for _, service := range in.Verifier { + if service != nil { + restartables = append(restartables, service) + } + } + for _, service := range restartables { + if err := service.Restart(ctx); err != nil { + return err + } + } + return nil +} + +// OpenDeploymentEnvironmentFromCfg builds selectors and a CLDF operations environment from a loaded env-out Cfg. +func OpenDeploymentEnvironmentFromCfg(cfg *Cfg) ([]uint64, *deployment.Environment, error) { + if cfg == nil { + return nil, nil, fmt.Errorf("open deployment environment: cfg is nil") + } + var ( + offchainClient offchain.Client + nodeIDs []string + ) + jdClient, err := jdClientFromCfg(cfg) + if err != nil { + return nil, nil, fmt.Errorf("open deployment environment: %w", err) + } + if jdClient != nil { + offchainClient = jdClient + nodeIDs, err = jdNodeIDs(jdClient) + if err != nil { + return nil, nil, fmt.Errorf("open deployment environment: list JD nodes: %w", err) + } + } + if cfg.ClientLookup == nil { + clAliases := clModeNOPAliases(cfg.EnvironmentTopology) + if len(clAliases) > 0 { + clientLookup, err := jobs.NewNodeSetClientLookup(cfg.NodeSets, clAliases) + if err != nil { + return nil, nil, fmt.Errorf("open deployment environment: build CL client lookup: %w", err) + } + cfg.ClientLookup = clientLookup + } + } + return NewCLDFOperationsEnvironmentWithOffchain(CLDFEnvironmentConfig{ + Blockchains: cfg.Blockchains, + DataStore: cfg.CLDF.DataStore, + OffchainClient: offchainClient, + NodeIDs: nodeIDs, + }) +} + +// ImplConfigurationsFromCfg returns one CCIP17Configuration per blockchain (same order as connectAllChains / NewEnvironment). +func ImplConfigurationsFromCfg(in *Cfg) ([]cciptestinterfaces.CCIP17Configuration, error) { + if in == nil { + return nil, fmt.Errorf("impl configurations: cfg is nil") + } + impls := make([]cciptestinterfaces.CCIP17Configuration, 0, len(in.Blockchains)) + for _, bc := range in.Blockchains { + impl, err := NewProductConfigurationFromNetwork(bc.Type) + if err != nil { + return nil, err + } + impls = append(impls, impl) + } + return impls, nil +} + +func configureOffchainFromTopology( + e *deployment.Environment, + in *Cfg, + topology *ccvdeployment.EnvironmentTopology, + sharedTLSCerts *services.TLSCertPaths, + ds datastore.MutableDataStore, +) (map[string][]bootstrap.JobSpec, error) { + if e == nil || in == nil || topology == nil { + return nil, fmt.Errorf("environment, config, and topology are required") + } + if ds == nil { + ds = datastore.NewMemoryDataStore() + if err := ds.Merge(e.DataStore); err != nil { + return nil, fmt.Errorf("merge initial datastore: %w", err) + } + } + + ccipTopology := convertTopologyToCCIP(topology) + + ResetMemoryOperationsBundle(e) + + for _, aggregatorInput := range in.Aggregator { + if sharedTLSCerts != nil { + aggregatorInput.SharedTLSCerts = sharedTLSCerts + } + e.DataStore = ds.Seal() + instanceName := aggregatorInput.InstanceName() + output, err := ccipChangesets.GenerateAggregatorConfig(ccipAdapters.GetAggregatorConfigRegistry()).Apply(*e, ccipChangesets.GenerateAggregatorConfigInput{ + ServiceIdentifier: instanceName + "-aggregator", + CommitteeQualifier: aggregatorInput.CommitteeName, + Topology: ccipTopology, + }) + if err != nil { + return nil, fmt.Errorf("generate aggregator config %s: %w", instanceName, err) + } + aggCfg, err := ccvdeployment.GetAggregatorConfig(output.DataStore.Seal(), instanceName+"-aggregator") + if err != nil { + return nil, fmt.Errorf("get aggregator config %s: %w", instanceName, err) + } + aggregatorInput.GeneratedCommittee = aggCfg + if err := ds.Merge(output.DataStore.Seal()); err != nil { + return nil, fmt.Errorf("merge aggregator datastore: %w", err) + } + } + + if len(in.Aggregator) > 0 && len(in.Indexer) > 0 { + e.DataStore = ds.Seal() + firstIdx := in.Indexer[0] + output, err := ccipChangesets.GenerateIndexerConfig(ccipAdapters.GetIndexerConfigRegistry()).Apply(*e, ccipChangesets.GenerateIndexerConfigInput{ + ServiceIdentifier: "indexer", + CommitteeVerifierNameToQualifier: firstIdx.CommitteeVerifierNameToQualifier, + CCTPVerifierNameToQualifier: firstIdx.CCTPVerifierNameToQualifier, + LombardVerifierNameToQualifier: firstIdx.LombardVerifierNameToQualifier, + }) + if err != nil { + return nil, fmt.Errorf("generate indexer config: %w", err) + } + idxCfg, err := ccvdeployment.GetIndexerConfig(output.DataStore.Seal(), "indexer") + if err != nil { + return nil, fmt.Errorf("get indexer config: %w", err) + } + for _, idxIn := range in.Indexer { + idxIn.GeneratedCfg = idxCfg + } + if err := ds.Merge(output.DataStore.Seal()); err != nil { + return nil, fmt.Errorf("merge indexer datastore: %w", err) + } + } + + e.DataStore = ds.Seal() + if err := generateExecutorJobSpecs(e, in, topology, ds); err != nil { + return nil, fmt.Errorf("executor job specs: %w", err) + } + e.DataStore = ds.Seal() + + verifierJobSpecs, err := generateVerifierJobSpecs(e, in, topology, sharedTLSCerts, ds) + if err != nil { + return nil, fmt.Errorf("verifier job specs: %w", err) + } + e.DataStore = ds.Seal() + in.CLDF.DataStore = e.DataStore + + return verifierJobSpecs, nil +} + +func configureTomlBoundServiceFiles(ctx context.Context, in *Cfg) error { + if in == nil { + return nil + } + refreshables := make([]refreshable, 0, len(in.Aggregator)+len(in.Indexer)) + for _, service := range in.Aggregator { + if service != nil { + refreshables = append(refreshables, service) + } + } + for _, service := range in.Indexer { + if service != nil { + refreshables = append(refreshables, service) + } + } + for _, service := range refreshables { + if err := service.RefreshConfig(ctx); err != nil { + return fmt.Errorf("refresh TOML-bound service config: %w", err) + } + } + return nil +} + +func acceptPendingJobsAndSync(ctx context.Context, e *deployment.Environment, in *Cfg) error { + if err := jobs.AcceptPendingJobs(ctx, in.ClientLookup); err != nil { + return err + } + return nil +} + +func clModeNOPAliases(topology *ccvdeployment.EnvironmentTopology) []string { + if topology == nil || topology.NOPTopology == nil { + return nil + } + aliases := make([]string, 0, len(topology.NOPTopology.NOPs)) + for _, nop := range topology.NOPTopology.NOPs { + if nop.GetMode() == ccvshared.NOPModeCL { + alias := nop.Alias + if alias == "" { + alias = nop.Name + } + aliases = append(aliases, alias) + } + } + return aliases +} + +func jdClientFromCfg(cfg *Cfg) (offchain.Client, error) { + if cfg == nil || cfg.JD == nil || cfg.JD.Out == nil { + return nil, nil + } + client, err := cldfjd.NewJDClient(cldfjd.JDConfig{ + GRPC: cfg.JD.Out.ExternalGRPCUrl, + WSRPC: cfg.JD.Out.ExternalWSRPCUrl, + Creds: insecure.NewCredentials(), + }) + if err != nil { + return nil, fmt.Errorf("create JD client from env-out: %w", err) + } + return client, nil +} + +func jdNodeIDs(client offchain.Client) ([]string, error) { + if client == nil { + return nil, nil + } + resp, err := client.ListNodes(context.Background(), &nodev1.ListNodesRequest{}) + if err != nil { + return nil, err + } + nodeIDs := make([]string, 0, len(resp.Nodes)) + for _, node := range resp.Nodes { + if node != nil && node.Id != "" { + nodeIDs = append(nodeIDs, node.Id) + } + } + return nodeIDs, nil +} diff --git a/build/devenv/reconcile_onchain.go b/build/devenv/reconcile_onchain.go new file mode 100644 index 000000000..87573050a --- /dev/null +++ b/build/devenv/reconcile_onchain.go @@ -0,0 +1,165 @@ +package ccv + +import ( + "context" + "fmt" + + "github.com/smartcontractkit/chainlink-ccip/chains/evm/deployment/v2_0_0/operations/committee_verifier" + changesetscore "github.com/smartcontractkit/chainlink-ccip/deployment/utils/changesets" + ccipAdapters "github.com/smartcontractkit/chainlink-ccip/deployment/v2_0_0/adapters" + ccipChangesets "github.com/smartcontractkit/chainlink-ccip/deployment/v2_0_0/changesets" + ccipOffchain "github.com/smartcontractkit/chainlink-ccip/deployment/v2_0_0/offchain" + "github.com/smartcontractkit/chainlink-ccv/build/devenv/cciptestinterfaces" + ccvdeployment "github.com/smartcontractkit/chainlink-ccv/deployment" + "github.com/smartcontractkit/chainlink-deployments-framework/datastore" + "github.com/smartcontractkit/chainlink-deployments-framework/deployment" + "github.com/smartcontractkit/chainlink-deployments-framework/operations" + "github.com/smartcontractkit/chainlink-testing-framework/framework/components/blockchain" +) + +// CommitteeRemotePatches maps local chain selector -> remote chain selector -> committee verifier remote patch. +type CommitteeRemotePatches map[uint64]map[uint64]ccipChangesets.CommitteeVerifierRemoteChainConfig + +// ReconfigureLanesParams configures reconfigureLanesFromTopology. Zero value is valid: production Router lanes +// (UseTestRouter false), no committee patches. To use TestRouter on a chain, set +// TestRouterByLane[localSelector][remoteSelector] = true; the deployment must include a TestRouter contract +// (DeployChainContractsPerChainCfg.DeployTestRouter) or the changeset apply can fail. +type ReconfigureLanesParams struct { + CommitteePatches CommitteeRemotePatches + TestRouterByLane map[uint64]map[uint64]bool +} + +// CommitteeRemotePatchesFromAllowlistArgs builds patches for one local chain from operation-style allowlist args. +func CommitteeRemotePatchesFromAllowlistArgs( + localSelector uint64, + args []committee_verifier.AllowlistConfigArgs, +) CommitteeRemotePatches { + if len(args) == 0 { + return nil + } + inner := make(map[uint64]ccipChangesets.CommitteeVerifierRemoteChainConfig, len(args)) + for _, a := range args { + added := make([]string, len(a.AddedAllowlistedSenders)) + for i, addr := range a.AddedAllowlistedSenders { + added[i] = addr.Hex() + } + removed := make([]string, len(a.RemovedAllowlistedSenders)) + for i, addr := range a.RemovedAllowlistedSenders { + removed[i] = addr.Hex() + } + allow := a.AllowlistEnabled + inner[a.DestChainSelector] = ccipChangesets.CommitteeVerifierRemoteChainConfig{ + AllowlistEnabled: &allow, + AddedAllowlistedSenders: added, + RemovedAllowlistedSenders: removed, + } + } + return CommitteeRemotePatches{localSelector: inner} +} + +// ResetMemoryOperationsBundle replaces the environment operations bundle with a fresh in-memory reporter. +func ResetMemoryOperationsBundle(e *deployment.Environment) { + if e == nil { + return + } + e.OperationsBundle = operations.NewBundle( + e.GetContext, + e.Logger, + operations.NewMemoryReporter(), + ) +} + +// ReconfigureLanesOnchainOnly applies topology-derived lane configuration on-chain +// (committee verifier signature configs, remote chain configs, router lanes) without +// regenerating off-chain configs or restarting any services. Intended for tests that +// need on-chain and aggregator configs to be intentionally out of sync. +func ReconfigureLanesOnchainOnly( + ctx context.Context, + e *deployment.Environment, + topology *ccvdeployment.EnvironmentTopology, + selectors []uint64, + blockchains []*blockchain.Input, + impls []cciptestinterfaces.CCIP17Configuration, + params ReconfigureLanesParams, +) error { + return reconfigureLanesFromTopology(ctx, e, convertTopologyToCCIP(topology), selectors, blockchains, impls, params) +} + +// reconfigureLanesFromTopology runs ConfigureChainsForLanesFromTopology once with a PartialChainConfig per chain +// (full peer mesh). Selector-to-impl mapping matches connectAllChains: impls[i] with blockchains[i], +// chain selector from chain ID + ChainFamily(), GetChainLaneProfile(env, sel) on that impl. +func reconfigureLanesFromTopology( + ctx context.Context, + e *deployment.Environment, + topology *ccipOffchain.EnvironmentTopology, + selectors []uint64, + blockchains []*blockchain.Input, + impls []cciptestinterfaces.CCIP17Configuration, + params ReconfigureLanesParams, +) error { + if e == nil || topology == nil { + return fmt.Errorf("reconfigure lanes: environment and topology are required") + } + if len(selectors) == 0 { + return fmt.Errorf("reconfigure lanes: selectors is required") + } + if len(blockchains) != len(impls) { + return fmt.Errorf("reconfigure lanes: %d blockchains and %d impls", len(blockchains), len(impls)) + } + for _, sel := range selectors { + if !e.BlockChains.Exists(sel) { + return fmt.Errorf("reconfigure lanes: chain selector %d not in environment", sel) + } + } + + orderedSelectors, profiles, err := buildConnectionProfilesFromImpls(impls, blockchains, selectors, e) + if err != nil { + return fmt.Errorf("reconfigure lanes: %w", err) + } + if err := assertConnectionProfilesCoverSelectors(orderedSelectors, selectors); err != nil { + return err + } + + chains, useTestRouter, err := buildPartialChainConfigsFromProfiles(topology, orderedSelectors, profiles, params) + if err != nil { + return fmt.Errorf("reconfigure lanes: %w", err) + } + + ResetMemoryOperationsBundle(e) + e.OperationsBundle = operations.NewBundle( + func() context.Context { return ctx }, + e.Logger, + operations.NewMemoryReporter(), + ) + + out, err := ccipChangesets.ConfigureChainsForLanesFromTopology( + ccipAdapters.GetCommitteeVerifierContractRegistry(), + ccipAdapters.GetChainFamilyRegistry(), + changesetscore.GetRegistry(), + ).Apply(*e, ccipChangesets.ConfigureChainsForLanesFromTopologyConfig{ + Topology: topology, + Chains: chains, + UseTestRouter: useTestRouter, + }) + if err != nil { + return err + } + if out.DataStore != nil { + mds := datastore.NewMemoryDataStore() + if err := mds.Merge(e.DataStore); err != nil { + return fmt.Errorf("reconfigure lanes: merge env datastore: %w", err) + } + if err := mds.Merge(out.DataStore.Seal()); err != nil { + return fmt.Errorf("reconfigure lanes: merge lane changeset datastore: %w", err) + } + e.DataStore = mds.Seal() + } + + for _, sel := range orderedSelectors { + entry := profiles[sel] + if err := entry.impl.PostConnect(e, sel, entry.remotes); err != nil { + return fmt.Errorf("reconfigure lanes: post-connect for chain %d: %w", sel, err) + } + } + return nil +} diff --git a/build/devenv/services/aggregator.go b/build/devenv/services/aggregator.go index a14525e5a..0f07a521d 100644 --- a/build/devenv/services/aggregator.go +++ b/build/devenv/services/aggregator.go @@ -47,6 +47,11 @@ const ( DefaultAggregatorGRPCPort = 50051 ) +// AggregatorGeneratedConfigContainerPath is the committee TOML path inside the aggregator container (matches NewAggregator ContainerFile). +func AggregatorGeneratedConfigContainerPath(instanceName string) string { + return "/etc/aggregator-" + instanceName + "-generated.toml" +} + type AggregatorDBInput struct { Image string `toml:"image"` // HostPort is the port on the host machine that the database will be exposed on. @@ -303,6 +308,54 @@ func (a *AggregatorInput) GenerateConfigs(generatedConfigFileName string) (*Gene }, nil } +// ConfigureGeneratedConfigFile persists only aggregator-*-generated.toml (committee). +// Main config stays the file from NewAggregator. +func (a *AggregatorInput) ConfigureGeneratedConfigFile() error { + if a == nil { + return nil + } + if a.GeneratedCommittee == nil { + return fmt.Errorf("ConfigureGeneratedConfigFile: GeneratedCommittee is required") + } + confDir := util.CCVConfigDir() + instanceName := a.InstanceName() + generatedConfigFileName := fmt.Sprintf("aggregator-%s-generated.toml", instanceName) + configResult, err := a.GenerateConfigs(generatedConfigFileName) + if err != nil { + return fmt.Errorf("write aggregator config files: generate: %w", err) + } + generatedConfigFilePath := filepath.Join(confDir, generatedConfigFileName) + if err := os.WriteFile(generatedConfigFilePath, configResult.GeneratedConfig, 0o644); err != nil { + return fmt.Errorf("write aggregator generated config: %w", err) + } + return nil +} + +// RefreshConfig refreshes the generated committee TOML on disk and syncs it into the running container. +func (a *AggregatorInput) RefreshConfig(ctx context.Context) error { + if a == nil { + return nil + } + if err := a.ConfigureGeneratedConfigFile(); err != nil { + return err + } + instanceName := a.InstanceName() + hostPath := filepath.Join(util.CCVConfigDir(), fmt.Sprintf("aggregator-%s-generated.toml", instanceName)) + containerName := fmt.Sprintf("%s-%s", instanceName, AggregatorContainerNameSuffix) + if err := DockerCopyFileToContainer(ctx, hostPath, containerName, AggregatorGeneratedConfigContainerPath(instanceName)); err != nil { + return fmt.Errorf("refresh aggregator generated config: %w", err) + } + return nil +} + +// Restart restarts the running aggregator container. +func (a *AggregatorInput) Restart(ctx context.Context) error { + if a == nil { + return nil + } + return RestartContainer(ctx, fmt.Sprintf("%s-%s", a.InstanceName(), AggregatorContainerNameSuffix)) +} + func (a *AggregatorInput) GetAPIKeys() ([]AggregatorClientConfig, error) { apiKeyConfigs := make([]AggregatorClientConfig, 0, len(a.APIClients)) for _, client := range a.APIClients { diff --git a/build/devenv/services/committeeverifier/base.go b/build/devenv/services/committeeverifier/base.go index 4c5592523..5f7637a49 100644 --- a/build/devenv/services/committeeverifier/base.go +++ b/build/devenv/services/committeeverifier/base.go @@ -153,6 +153,27 @@ type Output struct { JDNodeID string `toml:"jd_node_id"` } +func (in *Input) DockerContainerName() string { + if in == nil { + return "" + } + if in.Out != nil && in.Out.ContainerName != "" { + return services.NormalizeDockerContainerName(in.Out.ContainerName) + } + if in.ChainFamily == chainsel.FamilyEVM { + return fmt.Sprintf("evm-%s", in.ContainerName) + } + return services.NormalizeDockerContainerName(in.ContainerName) +} + +// Restart restarts the running verifier container. +func (in *Input) Restart(ctx context.Context) error { + if in == nil || in.Mode != services.Standalone { + return nil + } + return services.RestartContainer(ctx, in.DockerContainerName()) +} + func ApplyDefaults(in Input) Input { if in.Image == "" { in.Image = DefaultVerifierImage diff --git a/build/devenv/services/docker.go b/build/devenv/services/docker.go new file mode 100644 index 000000000..433a99e3a --- /dev/null +++ b/build/devenv/services/docker.go @@ -0,0 +1,42 @@ +package services + +import ( + "context" + "fmt" + "os/exec" + "strings" +) + +// NormalizeDockerContainerName strips leading slash from Docker inspect names so CLI calls match the container. +func NormalizeDockerContainerName(name string) string { + return strings.TrimPrefix(strings.TrimSpace(name), "/") +} + +// DockerCopyFileToContainer copies a host file into a running container path. +func DockerCopyFileToContainer(ctx context.Context, hostPath, containerName, containerPath string) error { + containerName = NormalizeDockerContainerName(containerName) + if hostPath == "" || containerName == "" || containerPath == "" { + return nil + } + dest := containerName + ":" + containerPath + cmd := exec.CommandContext(ctx, "docker", "cp", hostPath, dest) + out, err := cmd.CombinedOutput() + if err != nil { + return fmt.Errorf("docker cp %s -> %s: %w: %s", hostPath, dest, err, out) + } + return nil +} + +// RestartContainer restarts a running Docker container by name. +func RestartContainer(ctx context.Context, name string) error { + name = NormalizeDockerContainerName(name) + if name == "" { + return nil + } + cmd := exec.CommandContext(ctx, "docker", "restart", name) + out, err := cmd.CombinedOutput() + if err != nil { + return fmt.Errorf("docker restart %s: %w: %s", name, err, out) + } + return nil +} diff --git a/build/devenv/services/executor.go b/build/devenv/services/executor.go index bb4366d0e..fb7d32af8 100644 --- a/build/devenv/services/executor.go +++ b/build/devenv/services/executor.go @@ -85,6 +85,39 @@ func (v *ExecutorInput) GenerateConfigWithBlockchainInfos(blockchainInfos chaina return cfg, nil } +// ConfigureConfigFile persists the current standalone executor config and returns the host path. +func (v *ExecutorInput) ConfigureConfigFile(blockchainOutputs []*ctfblockchain.Output) (string, error) { + if v == nil { + return "", nil + } + ApplyExecutorDefaults(v) + + blockchainInfos, err := ConvertBlockchainOutputsToInfo(blockchainOutputs) + if err != nil { + return "", fmt.Errorf("failed to generate blockchain infos from blockchain outputs: %w", err) + } + blockchainInfos = filterOutUnsupportedChains(blockchainInfos) + + config, err := v.GenerateConfigWithBlockchainInfos(blockchainInfos) + if err != nil { + return "", fmt.Errorf("failed to generate config for executor: %w", err) + } + confDir := util.CCVConfigDir() + configFilePath := filepath.Join(confDir, fmt.Sprintf("executor-%s-config.toml", v.ContainerName)) + if err := os.WriteFile(configFilePath, config, 0o644); err != nil { + return "", fmt.Errorf("failed to write executor config to file: %w", err) + } + return configFilePath, nil +} + +// Restart restarts the running executor container. +func (v *ExecutorInput) Restart(ctx context.Context) error { + if v == nil || v.Mode != Standalone { + return nil + } + return RestartContainer(ctx, v.ContainerName) +} + // TransmitterAddressResolver derives an on-chain address from a hex-encoded private key. type TransmitterAddressResolver func(privateKeyHex string) (protocol.UnknownAddress, error) @@ -140,24 +173,9 @@ func NewExecutor(in *ExecutorInput, blockchainOutputs []*ctfblockchain.Output) ( if err != nil { return in.Out, err } - - // Generate blockchain infos for standalone mode - blockchainInfos, err := ConvertBlockchainOutputsToInfo(blockchainOutputs) - if err != nil { - return nil, fmt.Errorf("failed to generate blockchain infos from blockchain outputs: %w", err) - } - - blockchainInfos = filterOutUnsupportedChains(blockchainInfos) - - // Generate and store config file with blockchain infos for standalone mode - config, err := in.GenerateConfigWithBlockchainInfos(blockchainInfos) + configFilePath, err := in.ConfigureConfigFile(blockchainOutputs) if err != nil { - return nil, fmt.Errorf("failed to generate config for executor: %w", err) - } - confDir := util.CCVConfigDir() - configFilePath := filepath.Join(confDir, fmt.Sprintf("executor-%s-config.toml", in.ContainerName)) - if err := os.WriteFile(configFilePath, config, 0o644); err != nil { - return nil, fmt.Errorf("failed to write executor config to file: %w", err) + return nil, err } /* Service */ @@ -224,6 +242,9 @@ type TransmitterKeyGenerator func() (string, error) // using the given key generator. func SetTransmitterPrivateKey(execs []*ExecutorInput, keyGen TransmitterKeyGenerator) ([]*ExecutorInput, error) { for _, exec := range execs { + if exec.TransmitterPrivateKey != "" { + continue + } pk, err := keyGen() if err != nil { return nil, fmt.Errorf("failed to generate transmitter private key: %w", err) diff --git a/build/devenv/services/executor/base.go b/build/devenv/services/executor/base.go index c39ed46f3..55fba5f9d 100644 --- a/build/devenv/services/executor/base.go +++ b/build/devenv/services/executor/base.go @@ -67,7 +67,7 @@ type Input struct { GeneratedConfig string `toml:"-"` // GeneratedJobSpecs contains all job specs for this executor. - GeneratedJobSpecs []string `toml:"-"` + GeneratedJobSpecs []bootstrap.JobSpec `toml:"-"` // Bootstrap is the bootstrap configuration for bootstrapped mode. Bootstrap *services.BootstrapInput `toml:"bootstrap"` @@ -97,6 +97,13 @@ type Output struct { JDNodeID string `toml:"jd_node_id"` } +func (v *Input) Restart(ctx context.Context) error { + if v == nil || v.Mode != services.Standalone { + return nil + } + return services.RestartContainer(ctx, v.ContainerName) +} + // RebuildExecutorJobSpecWithBlockchainInfos takes a job spec and rebuilds it with blockchain infos // added to the inner config. This is needed for standalone executors which require blockchain // connection information (CL nodes get this from their own chain config). @@ -571,6 +578,9 @@ type TransmitterAddressResolver func(privateKeyHex string) (protocol.UnknownAddr // using the given key generator. Pass a family-specific generator from ImplFactory. func SetTransmitterPrivateKey(execs []*Input, keyGen TransmitterKeyGenerator) ([]*Input, error) { for _, exec := range execs { + if exec.TransmitterPrivateKey != "" { + continue + } pk, err := keyGen() if err != nil { return nil, fmt.Errorf("failed to generate transmitter private key: %w", err) diff --git a/build/devenv/services/indexer.go b/build/devenv/services/indexer.go index 3795b8a91..cf9f5d0d2 100644 --- a/build/devenv/services/indexer.go +++ b/build/devenv/services/indexer.go @@ -118,32 +118,21 @@ func injectPostgresURI(cfg *config.Config, uri string) { cfg.Storage.Single.Postgres.URI = uri } -// NewIndexer creates and starts a new Service container using testcontainers. -// Will be called once per indexer instance. -func NewIndexer(in *IndexerInput) (*IndexerOutput, error) { +// ConfigureConfigFiles persists the current indexer config, generated config, and secrets +// using the same paths NewIndexer mounts into the running container. +func (in *IndexerInput) ConfigureConfigFiles() (string, string, string, error) { if in == nil { - return nil, nil - } - if in.Out != nil && in.Out.UseCache { - return in.Out, nil + return "", "", "", nil } - ctx := context.Background() defaults(in) if in.GeneratedCfg == nil { - return nil, fmt.Errorf("GeneratedCfg is required for indexer") + return "", "", "", fmt.Errorf("GeneratedCfg is required for indexer") } - if in.IndexerConfig == nil { - return nil, fmt.Errorf("IndexerConfig is required for indexer") + return "", "", "", fmt.Errorf("IndexerConfig is required for indexer") } - p, err := CwdSourcePath(in.SourceCodePath) - if err != nil { - return in.Out, err - } - - // Per-instance config dir and filenames (supports multiple indexers, like NewAggregator per committee). confDir := util.CCVConfigDir() configFileName := fmt.Sprintf("indexer-%s-config.toml", in.ContainerName) generatedConfigFileName := "generated.toml" @@ -155,7 +144,6 @@ func NewIndexer(in *IndexerInput) (*IndexerOutput, error) { in.IndexerConfig.GeneratedConfigPath = generatedConfigFileName - // Per-instance DB credentials (from config or derived from container name for multi-instance isolation). dbName := in.DB.Database if dbName == "" { dbName = in.ContainerName @@ -169,7 +157,6 @@ func NewIndexer(in *IndexerInput) (*IndexerOutput, error) { dbPass = in.ContainerName } - // DB connection string: from config (StorageConnectionURL) when set, else build from DB/container (aligned with aggregator). dbContainerName := in.ContainerName + IndexerDBContainerSuffix var dbConnectionString string if in.StorageConnectionURL != "" { @@ -183,22 +170,21 @@ func NewIndexer(in *IndexerInput) (*IndexerOutput, error) { buff := new(bytes.Buffer) encoder := toml.NewEncoder(buff) encoder.Indent = "" - err = encoder.Encode(in.IndexerConfig) - if err != nil { - return nil, fmt.Errorf("failed to encode config: %w", err) + if err := encoder.Encode(in.IndexerConfig); err != nil { + return "", "", "", fmt.Errorf("failed to encode config: %w", err) } if err := os.WriteFile(configPath, buff.Bytes(), 0o644); err != nil { - return nil, fmt.Errorf("failed to write config: %w", err) + return "", "", "", fmt.Errorf("failed to write config: %w", err) } genBuff := new(bytes.Buffer) genEncoder := toml.NewEncoder(genBuff) genEncoder.Indent = "" if err := genEncoder.Encode(in.GeneratedCfg); err != nil { - return nil, fmt.Errorf("failed to encode generated config: %w", err) + return "", "", "", fmt.Errorf("failed to encode generated config: %w", err) } if err := os.WriteFile(generatedConfigPath, genBuff.Bytes(), 0o644); err != nil { - return nil, fmt.Errorf("failed to write generated config: %w", err) + return "", "", "", fmt.Errorf("failed to write generated config: %w", err) } secretsToEncode := in.Secrets @@ -209,14 +195,87 @@ func NewIndexer(in *IndexerInput) (*IndexerOutput, error) { secEncoder := toml.NewEncoder(secretsBuffer) secEncoder.Indent = "" if err := secEncoder.Encode(secretsToEncode); err != nil { - return nil, fmt.Errorf("failed to encode secrets: %w", err) + return "", "", "", fmt.Errorf("failed to encode secrets: %w", err) } if err := os.WriteFile(secretsPath, secretsBuffer.Bytes(), 0o644); err != nil { - return nil, fmt.Errorf("failed to write secrets file: %w", err) + return "", "", "", fmt.Errorf("failed to write secrets file: %w", err) + } + + return configPath, generatedConfigPath, secretsPath, nil +} + +// RefreshConfig rewrites the mounted indexer config files from the current in-memory config. +func (in *IndexerInput) RefreshConfig(context.Context) error { + if in == nil { + return nil + } + _, _, _, err := in.ConfigureConfigFiles() + return err +} + +// Restart restarts the running indexer container. +func (in *IndexerInput) Restart(ctx context.Context) error { + if in == nil { + return nil + } + name := in.ContainerName + if in.Out != nil && in.Out.ContainerName != "" { + name = in.Out.ContainerName + } + return RestartContainer(ctx, name) +} + +// NewIndexer creates and starts a new Service container using testcontainers. +// Will be called once per indexer instance. +func NewIndexer(in *IndexerInput) (*IndexerOutput, error) { + if in == nil { + return nil, nil + } + if in.Out != nil && in.Out.UseCache { + return in.Out, nil + } + ctx := context.Background() + defaults(in) + + if in.GeneratedCfg == nil { + return nil, fmt.Errorf("GeneratedCfg is required for indexer") + } + + if in.IndexerConfig == nil { + return nil, fmt.Errorf("IndexerConfig is required for indexer") + } + + p, err := CwdSourcePath(in.SourceCodePath) + if err != nil { + return in.Out, err + } + configPath, generatedConfigPath, secretsPath, err := in.ConfigureConfigFiles() + if err != nil { + return nil, err } // Database: unique name and host port per instance (like aggregator DB per committee). // one db instance per indexer. + dbName := in.DB.Database + if dbName == "" { + dbName = in.ContainerName + } + dbUser := in.DB.Username + if dbUser == "" { + dbUser = in.ContainerName + } + dbPass := in.DB.Password + if dbPass == "" { + dbPass = in.ContainerName + } + dbContainerName := in.ContainerName + IndexerDBContainerSuffix + var dbConnectionString string + if in.StorageConnectionURL != "" { + dbConnectionString = in.StorageConnectionURL + } else { + dbConnectionString = fmt.Sprintf("postgresql://%s:%s@%s:5432/%s?sslmode=disable", + dbUser, dbPass, dbContainerName, dbName) + } _, err = postgres.Run(ctx, in.DB.Image, testcontainers.WithName(dbContainerName), @@ -252,6 +311,7 @@ func NewIndexer(in *IndexerInput) (*IndexerOutput, error) { internalPortStr := strconv.Itoa(internalPort) // Container paths for mounted config (same path in every container; each has its own files). + generatedConfigFileName := "generated.toml" containerConfigPath := filepath.Join(IndexerConfigDirContainer, "config.toml") containerGeneratedPath := filepath.Join(IndexerConfigDirContainer, generatedConfigFileName) containerSecretsPath := filepath.Join(IndexerConfigDirContainer, "secrets.toml") diff --git a/build/devenv/tests/e2e/environment_change_reconcile_test.go b/build/devenv/tests/e2e/environment_change_reconcile_test.go new file mode 100644 index 000000000..4f8a58f1c --- /dev/null +++ b/build/devenv/tests/e2e/environment_change_reconcile_test.go @@ -0,0 +1,664 @@ +package e2e + +import ( + "context" + "errors" + "fmt" + "os" + "slices" + "strings" + "testing" + "time" + + "github.com/BurntSushi/toml" + "github.com/Masterminds/semver/v3" + "github.com/ethereum/go-ethereum/common" + "github.com/stretchr/testify/require" + + chain_selectors "github.com/smartcontractkit/chain-selectors" + routeroperations "github.com/smartcontractkit/chainlink-ccip/chains/evm/deployment/v1_2_0/operations/router" + "github.com/smartcontractkit/chainlink-ccip/chains/evm/deployment/v2_0_0/operations/committee_verifier" + "github.com/smartcontractkit/chainlink-ccip/chains/evm/deployment/v2_0_0/operations/proxy" + "github.com/smartcontractkit/chainlink-ccip/chains/evm/deployment/v2_0_0/sequences" + "github.com/smartcontractkit/chainlink-ccip/chains/evm/deployment/v2_0_0/versioned_verifier_resolver" + "github.com/smartcontractkit/chainlink-ccip/deployment/v2_0_0/offchain/operations/fetch_signing_keys" + ccv "github.com/smartcontractkit/chainlink-ccv/build/devenv" + "github.com/smartcontractkit/chainlink-ccv/build/devenv/cciptestinterfaces" + devenvcommon "github.com/smartcontractkit/chainlink-ccv/build/devenv/common" + ccvevm "github.com/smartcontractkit/chainlink-ccv/build/devenv/evm" + "github.com/smartcontractkit/chainlink-ccv/build/devenv/tests/e2e/tcapi" + ccvcomm "github.com/smartcontractkit/chainlink-ccv/common" + "github.com/smartcontractkit/chainlink-ccv/common/committee" + ccvdeployment "github.com/smartcontractkit/chainlink-ccv/deployment" + "github.com/smartcontractkit/chainlink-ccv/protocol" + "github.com/smartcontractkit/chainlink-deployments-framework/datastore" + "github.com/smartcontractkit/chainlink-deployments-framework/deployment" + "github.com/smartcontractkit/chainlink-deployments-framework/operations" +) + +func environmentChangeSmokeConfigPath() string { + if p := os.Getenv("ENVIRONMENT_CHANGE_SMOKE_TEST_CONFIG"); p != "" { + return p + } + return GetSmokeTestConfig() +} + +func requireEnvironmentChangeEnvFile(t *testing.T) string { + t.Helper() + path := environmentChangeSmokeConfigPath() + if _, err := os.Stat(path); err != nil { + t.Skipf("environment change E2E requires env-out (missing %q: %v); run ccv up and set ENVIRONMENT_CHANGE_SMOKE_TEST_CONFIG or use SMOKE_TEST_CONFIG via GetSmokeTestConfig", path, err) + } + return path +} + +type environmentChangeReconcileHarness struct { + Cfg *ccv.Cfg + Selectors []uint64 + Env *deployment.Environment + Topology *ccvdeployment.EnvironmentTopology + Impls []cciptestinterfaces.CCIP17Configuration +} + +const ( + environmentChangeAssertMessageTimeout = 4 * time.Minute + environmentChangePostMessageExecTimeout = 2 * time.Minute +) + +var errEnvironmentChangeEOADefaultVerifierPrerequisites = errors.New("EOA default verifier message prerequisites not met") + +func newEnvironmentChangeReconcileHarness(t *testing.T) *environmentChangeReconcileHarness { + t.Helper() + path := requireEnvironmentChangeEnvFile(t) + cfg, err := ccv.LoadOutput[ccv.Cfg](path) + require.NoError(t, err) + if err := ccv.RequireFullCLModeForEnvironmentChangeReconcile(cfg); err != nil { + t.Skipf("environment change reconcile E2E requires full CL mode (topology NOPs cl, verifier/executor mode cl, nodesets): %v; use a CL env-out or set ENVIRONMENT_CHANGE_SMOKE_TEST_CONFIG", err) + } + selectors, env, err := ccv.OpenDeploymentEnvironmentFromCfg(cfg) + require.NoError(t, err) + require.NotEmpty(t, selectors) + topology := ccv.BuildEnvironmentTopology(cfg, env) + require.NotNil(t, topology) + impls, err := ccv.ImplConfigurationsFromCfg(cfg) + require.NoError(t, err) + require.Len(t, impls, len(cfg.Blockchains)) + return &environmentChangeReconcileHarness{ + Cfg: cfg, + Selectors: selectors, + Env: env, + Topology: topology, + Impls: impls, + } +} + +func testRouterDeployedOnSelector(t *testing.T, env *deployment.Environment, selector uint64) bool { + t.Helper() + _, err := env.DataStore.Addresses().Get(datastore.NewAddressRefKey( + selector, + datastore.ContractType(routeroperations.TestRouterContractType), + semver.MustParse(routeroperations.DeployTestRouter.Version()), + "", + )) + return err == nil +} + +func environmentChangeLinkedLongContext(t *testing.T) context.Context { + t.Helper() + base := t.Context() + longCtx, cancel := context.WithTimeout(context.Background(), 10*time.Minute) + go func() { + <-base.Done() + cancel() + }() + t.Cleanup(cancel) + return longCtx +} + +func environmentChangeReconcileOpts() ccv.ConfigureOffchainOptions { + return ccv.ConfigureOffchainOptions{FundExecutors: false} +} + +func requireEnvironmentChangeReconcile(t *testing.T, ctx context.Context, h *environmentChangeReconcileHarness, lane ccv.ReconfigureLanesParams) { + t.Helper() + require.NoError(t, ccv.ConfigureTopologyLanesAndOffchain( + ctx, h.Env, h.Cfg, h.Topology, h.Selectors, h.Cfg.Blockchains, h.Impls, lane, nil, environmentChangeReconcileOpts(), + )) +} + +func requireNOPSigningKeyFromJD(t *testing.T, h *environmentChangeReconcileHarness, nopAlias string) string { + t.Helper() + require.NotNil(t, h.Env) + require.NotNil(t, h.Env.Offchain) + + report, err := operations.ExecuteOperation( + h.Env.OperationsBundle, + fetch_signing_keys.FetchNOPSigningKeys, + fetch_signing_keys.FetchSigningKeysDeps{ + JDClient: h.Env.Offchain, + Logger: h.Env.Logger, + NodeIDs: h.Env.NodeIDs, + }, + fetch_signing_keys.FetchSigningKeysInput{NOPAliases: []string{nopAlias}}, + ) + require.NoError(t, err) + + signersByFamily, ok := report.Output.SigningKeysByNOP[nopAlias] + require.True(t, ok, "NOP %q needs signing keys from JD", nopAlias) + + evmSigner := signersByFamily[chain_selectors.FamilyEVM] + require.NotEmpty(t, evmSigner, "NOP %q needs EVM signer from JD", nopAlias) + return evmSigner +} + +type environmentChangeEOADefaultVerifierInputs struct { + receiver protocol.UnknownAddress + ccvs []protocol.CCV + executor protocol.UnknownAddress +} + +func environmentChangeEOADefaultVerifierConfig(cfg *ccv.Cfg, src, dest cciptestinterfaces.CCIP17) (*environmentChangeEOADefaultVerifierInputs, error) { + receiver, err := dest.GetEOAReceiverAddress() + if err != nil { + return nil, err + } + ccvAddr, err := tcapi.GetContractAddress( + cfg, + src.ChainSelector(), + datastore.ContractType(versioned_verifier_resolver.CommitteeVerifierResolverType), + versioned_verifier_resolver.Version.String(), + devenvcommon.DefaultCommitteeVerifierQualifier, + "committee verifier proxy", + ) + if err != nil { + return nil, err + } + executorAddr, err := tcapi.GetContractAddress( + cfg, + src.ChainSelector(), + datastore.ContractType(sequences.ExecutorProxyType), + proxy.Deploy.Version(), + devenvcommon.DefaultExecutorQualifier, + "executor", + ) + if err != nil { + return nil, err + } + return &environmentChangeEOADefaultVerifierInputs{ + receiver: receiver, + ccvs: []protocol.CCV{{CCVAddress: ccvAddr, Args: []byte{}, ArgsLen: 0}}, + executor: executorAddr, + }, nil +} + +func runEnvironmentChangeEOADefaultVerifierWithIndexedResult( + ctx context.Context, + harness tcapi.TestHarness, + cfg *ccv.Cfg, + src, dest cciptestinterfaces.CCIP17, + useTestRouter bool, +) (tcapi.AssertionResult, error) { + inputs, err := environmentChangeEOADefaultVerifierConfig(cfg, src, dest) + if err != nil { + return tcapi.AssertionResult{}, fmt.Errorf("%w: %w", errEnvironmentChangeEOADefaultVerifierPrerequisites, err) + } + + seqNo, err := src.GetExpectedNextSequenceNumber(ctx, dest.ChainSelector()) + if err != nil { + return tcapi.AssertionResult{}, fmt.Errorf("failed to get expected next sequence number: %w", err) + } + sendMessageResult, err := src.SendMessage( + ctx, dest.ChainSelector(), cciptestinterfaces.MessageFields{ + Receiver: inputs.receiver, + Data: []byte("multi-verifier test"), + }, cciptestinterfaces.MessageOptions{ + Version: 3, + ExecutionGasLimit: 200_000, + FinalityConfig: 1, + Executor: inputs.executor, + CCVs: inputs.ccvs, + UseTestRouter: useTestRouter, + }, + ) + if err != nil { + return tcapi.AssertionResult{}, fmt.Errorf("failed to send message: %w", err) + } + if len(sendMessageResult.ReceiptIssuers) != 3 { + return tcapi.AssertionResult{}, fmt.Errorf("expected 3 receipt issuers, got %d", len(sendMessageResult.ReceiptIssuers)) + } + sentEvent, err := src.ConfirmSendOnSource(ctx, dest.ChainSelector(), cciptestinterfaces.MessageEventKey{SeqNum: seqNo}, tcapi.DefaultSentTimeout) + if err != nil { + return tcapi.AssertionResult{}, fmt.Errorf("failed to confirm send on source: %w", err) + } + aggregatorClient := harness.AggregatorClients[devenvcommon.DefaultCommitteeVerifierQualifier] + chainMap, err := harness.Lib.ChainsMap(ctx) + if err != nil { + return tcapi.AssertionResult{}, fmt.Errorf("failed to get chains map: %w", err) + } + testCtx, cleanup := tcapi.NewTestingContext(ctx, chainMap, aggregatorClient, harness.IndexerMonitor) + defer cleanup() + + result, err := testCtx.AssertMessage(sentEvent.MessageID, tcapi.AssertMessageOptions{ + TickInterval: time.Second, + ExpectedVerifierResults: 1, + Timeout: environmentChangeAssertMessageTimeout, + AssertVerifierLogs: false, + AssertExecutorLogs: false, + }) + if err != nil { + return result, fmt.Errorf("failed to assert message: %w", err) + } + if result.AggregatedResult == nil { + return result, fmt.Errorf("aggregated result is nil") + } + if len(result.IndexedVerifications.Results) != 1 { + return result, fmt.Errorf("expected 1 indexed verification, got %d", len(result.IndexedVerifications.Results)) + } + e, err := chainMap[dest.ChainSelector()].ConfirmExecOnDest(ctx, src.ChainSelector(), cciptestinterfaces.MessageEventKey{SeqNum: seqNo}, environmentChangePostMessageExecTimeout) + if err != nil { + return result, fmt.Errorf("failed to confirm exec on dest: %w", err) + } + if e.State != cciptestinterfaces.ExecutionStateSuccess { + return result, fmt.Errorf("expected execution state success, got %s", e.State) + } + return result, nil +} + +// runEnvironmentChangeEOADefaultVerifierIndexedResultUntilExecStops returns the message ID and +// sequence number when the indexer and aggregator pipeline succeeds but destination execution +// does not reach SUCCESS (typically FAILURE after a committee threshold mismatch). +func runEnvironmentChangeEOADefaultVerifierIndexedResultUntilExecStops( + ctx context.Context, + harness tcapi.TestHarness, + cfg *ccv.Cfg, + src, dest cciptestinterfaces.CCIP17, + useTestRouter bool, +) ([32]byte, uint64, tcapi.AssertionResult, cciptestinterfaces.MessageExecutionState, error) { + var zeroMsg [32]byte + inputs, err := environmentChangeEOADefaultVerifierConfig(cfg, src, dest) + if err != nil { + return zeroMsg, 0, tcapi.AssertionResult{}, 0, fmt.Errorf("%w: %w", errEnvironmentChangeEOADefaultVerifierPrerequisites, err) + } + + seqNo, err := src.GetExpectedNextSequenceNumber(ctx, dest.ChainSelector()) + if err != nil { + return zeroMsg, 0, tcapi.AssertionResult{}, 0, fmt.Errorf("failed to get expected next sequence number: %w", err) + } + sendMessageResult, err := src.SendMessage( + ctx, dest.ChainSelector(), cciptestinterfaces.MessageFields{ + Receiver: inputs.receiver, + Data: []byte("multi-verifier test"), + }, cciptestinterfaces.MessageOptions{ + Version: 3, + ExecutionGasLimit: 200_000, + FinalityConfig: 1, + Executor: inputs.executor, + CCVs: inputs.ccvs, + UseTestRouter: useTestRouter, + }, + ) + if err != nil { + return zeroMsg, 0, tcapi.AssertionResult{}, 0, fmt.Errorf("failed to send message: %w", err) + } + if len(sendMessageResult.ReceiptIssuers) != 3 { + return zeroMsg, 0, tcapi.AssertionResult{}, 0, fmt.Errorf("expected 3 receipt issuers, got %d", len(sendMessageResult.ReceiptIssuers)) + } + sentEvent, err := src.ConfirmSendOnSource(ctx, dest.ChainSelector(), cciptestinterfaces.MessageEventKey{SeqNum: seqNo}, tcapi.DefaultSentTimeout) + if err != nil { + return zeroMsg, 0, tcapi.AssertionResult{}, 0, fmt.Errorf("failed to confirm send on source: %w", err) + } + aggregatorClient := harness.AggregatorClients[devenvcommon.DefaultCommitteeVerifierQualifier] + chainMap, err := harness.Lib.ChainsMap(ctx) + if err != nil { + return zeroMsg, 0, tcapi.AssertionResult{}, 0, fmt.Errorf("failed to get chains map: %w", err) + } + testCtx, cleanup := tcapi.NewTestingContext(ctx, chainMap, aggregatorClient, harness.IndexerMonitor) + defer cleanup() + + result, err := testCtx.AssertMessage(sentEvent.MessageID, tcapi.AssertMessageOptions{ + TickInterval: time.Second, + ExpectedVerifierResults: 1, + Timeout: environmentChangeAssertMessageTimeout, + AssertVerifierLogs: false, + AssertExecutorLogs: false, + }) + if err != nil { + return sentEvent.MessageID, seqNo, result, 0, fmt.Errorf("failed to assert message: %w", err) + } + if result.AggregatedResult == nil { + return sentEvent.MessageID, seqNo, result, 0, fmt.Errorf("aggregated result is nil") + } + if len(result.IndexedVerifications.Results) != 1 { + return sentEvent.MessageID, seqNo, result, 0, fmt.Errorf("expected 1 indexed verification, got %d", len(result.IndexedVerifications.Results)) + } + e, err := chainMap[dest.ChainSelector()].ConfirmExecOnDest(ctx, src.ChainSelector(), cciptestinterfaces.MessageEventKey{SeqNum: seqNo}, environmentChangePostMessageExecTimeout) + if err != nil { + return sentEvent.MessageID, seqNo, result, 0, fmt.Errorf("failed to confirm exec on dest: %w", err) + } + if e.State == cciptestinterfaces.ExecutionStateSuccess { + return sentEvent.MessageID, seqNo, result, e.State, fmt.Errorf("expected execution to stop before success, got success") + } + return sentEvent.MessageID, seqNo, result, e.State, nil +} + +// requireEnvironmentChangeMessageReachesExecutionSuccess polls the OffRamp +// getExecutionState view directly until the message reaches SUCCESS. This +// bypasses the event-poller cache (which write-once caches the first event per +// seqNo and would immediately return the stale FAILURE for a retried message). +func requireEnvironmentChangeMessageReachesExecutionSuccess( + t *testing.T, + ctx context.Context, + dest cciptestinterfaces.CCIP17, + msgID [32]byte, +) { + t.Helper() + destEVM, ok := dest.(*ccvevm.CCIP17EVM) + require.True(t, ok, "dest chain must be *CCIP17EVM for direct execution-state polling") + pollCtx, cancel := context.WithTimeout(ctx, environmentChangeAssertMessageTimeout+environmentChangePostMessageExecTimeout) + defer cancel() + require.NoError(t, destEVM.WaitForExecutionState( + pollCtx, msgID, cciptestinterfaces.ExecutionStateSuccess, 2*time.Second, + ), "expected original message to reach SUCCESS after recovery") +} + +func twoDistinctEVMSelectorsFromHarness(t *testing.T, h *environmentChangeReconcileHarness) (srcSel, destSel uint64) { + t.Helper() + require.GreaterOrEqual(t, len(h.Cfg.Blockchains), 2, "need at least two chains") + for _, bc := range h.Cfg.Blockchains { + if bc.Out == nil || bc.Out.Family != chain_selectors.FamilyEVM { + continue + } + d, err := chain_selectors.GetChainDetailsByChainIDAndFamily(bc.ChainID, bc.Out.Family) + require.NoError(t, err) + if srcSel == 0 { + srcSel = d.ChainSelector + continue + } + if d.ChainSelector != srcSel { + destSel = d.ChainSelector + break + } + } + require.NotZero(t, destSel, "need two distinct EVM chain selectors") + return srcSel, destSel +} + +func reconfigureEnvironmentChangeCommitteeAllowlist(t *testing.T, h *environmentChangeReconcileHarness, srcSel uint64, args []committee_verifier.AllowlistConfigArgs) { + t.Helper() + ctx := ccv.Plog.WithContext(t.Context()) + patches := ccv.CommitteeRemotePatchesFromAllowlistArgs(srcSel, args) + requireEnvironmentChangeReconcile(t, ctx, h, ccv.ReconfigureLanesParams{CommitteePatches: patches}) +} + +func reconfigureEnvironmentChangeAllowlistEnabled(t *testing.T, h *environmentChangeReconcileHarness, srcSel, destSel uint64, sender common.Address) { + t.Helper() + reconfigureEnvironmentChangeCommitteeAllowlist(t, h, srcSel, []committee_verifier.AllowlistConfigArgs{ + { + DestChainSelector: destSel, + AllowlistEnabled: true, + AddedAllowlistedSenders: []common.Address{sender}, + }, + }) +} + +func reconfigureEnvironmentChangeAllowlistDisabled(t *testing.T, h *environmentChangeReconcileHarness, srcSel, destSel uint64, sender common.Address) { + t.Helper() + reconfigureEnvironmentChangeCommitteeAllowlist(t, h, srcSel, []committee_verifier.AllowlistConfigArgs{ + { + DestChainSelector: destSel, + AllowlistEnabled: false, + RemovedAllowlistedSenders: []common.Address{sender}, + }, + }) +} + +func ccip17PairForSelectors(t *testing.T, ctx context.Context, lib *ccv.Lib, srcSel, destSel uint64) (src, dest cciptestinterfaces.CCIP17) { + t.Helper() + chains, err := lib.Chains(ctx) + require.NoError(t, err) + for _, c := range chains { + if c.Details.ChainSelector == srcSel { + src = c.CCIP17 + } + if c.Details.ChainSelector == destSel { + dest = c.CCIP17 + } + } + require.NotNil(t, src, "no CCIP17 impl for source selector %d", srcSel) + require.NotNil(t, dest, "no CCIP17 impl for dest selector %d", destSel) + return src, dest +} + +func requireEnvironmentChangeEOADefaultVerifierMessage(t *testing.T, ctx context.Context, th tcapi.TestHarness, cfg *ccv.Cfg, src, dest cciptestinterfaces.CCIP17) { + t.Helper() + _, err := runEnvironmentChangeEOADefaultVerifierWithIndexedResult(ctx, th, cfg, src, dest, false) + if errors.Is(err, errEnvironmentChangeEOADefaultVerifierPrerequisites) { + t.Skip("EOA default verifier message prerequisites not met for this env") + } + require.NoError(t, err) +} + +func requireEnvironmentChangeEOADefaultVerifierMessageExpectError(t *testing.T, ctx context.Context, th tcapi.TestHarness, cfg *ccv.Cfg, src, dest cciptestinterfaces.CCIP17) { + t.Helper() + _, err := runEnvironmentChangeEOADefaultVerifierWithIndexedResult(ctx, th, cfg, src, dest, false) + if errors.Is(err, errEnvironmentChangeEOADefaultVerifierPrerequisites) { + t.Skip("EOA default verifier message prerequisites not met for this env") + } + require.Error(t, err, "EOA message must not complete end-to-end when committee allowlist excludes deployer") +} + +func requireEnvironmentChangeEOADefaultVerifierMessageWithTestRouter(t *testing.T, ctx context.Context, th tcapi.TestHarness, cfg *ccv.Cfg, src, dest cciptestinterfaces.CCIP17, useTestRouter bool) { + t.Helper() + _, err := runEnvironmentChangeEOADefaultVerifierWithIndexedResult(ctx, th, cfg, src, dest, useTestRouter) + if errors.Is(err, errEnvironmentChangeEOADefaultVerifierPrerequisites) { + t.Skip("EOA default verifier message prerequisites not met for this env") + } + require.NoError(t, err) +} + +func TestEnvironmentChangeReconcile_CommitteeVerifierAllowlistDecoyExpectErrorThenDeployerHappyPath(t *testing.T) { + h := newEnvironmentChangeReconcileHarness(t) + ctx := ccv.Plog.WithContext(t.Context()) + path := environmentChangeSmokeConfigPath() + th, err := tcapi.NewTestHarness(ctx, path, h.Cfg, chain_selectors.FamilyEVM) + if err != nil { + t.Skipf("message verification needs tcapi harness (aggregators/indexer): %v", err) + } + srcSel, destSel := twoDistinctEVMSelectorsFromHarness(t, h) + src, dest := ccip17PairForSelectors(t, ctx, th.Lib, srcSel, destSel) + deployer := h.Env.BlockChains.EVMChains()[srcSel].DeployerKey.From + decoy := common.HexToAddress("0x0000000000000000000000000000000000000001") + require.NotEqual(t, deployer, decoy, "decoy allowlist entry must not be the chain deployer used to send messages") + + requireEnvironmentChangeEOADefaultVerifierMessage(t, ctx, th, h.Cfg, src, dest) + + reconfigureEnvironmentChangeAllowlistEnabled(t, h, srcSel, destSel, decoy) + requireEnvironmentChangeEOADefaultVerifierMessageExpectError(t, ctx, th, h.Cfg, src, dest) + + reconfigureEnvironmentChangeAllowlistDisabled(t, h, srcSel, destSel, decoy) + requireEnvironmentChangeEOADefaultVerifierMessage(t, ctx, th, h.Cfg, src, dest) + + reconfigureEnvironmentChangeAllowlistEnabled(t, h, srcSel, destSel, deployer) + requireEnvironmentChangeEOADefaultVerifierMessage(t, ctx, th, h.Cfg, src, dest) + + reconfigureEnvironmentChangeAllowlistDisabled(t, h, srcSel, destSel, deployer) + requireEnvironmentChangeEOADefaultVerifierMessage(t, ctx, th, h.Cfg, src, dest) +} + +func testRouterSourceAndDestSelectors(t *testing.T, h *environmentChangeReconcileHarness) (srcSel, destSel uint64) { + t.Helper() + require.GreaterOrEqual(t, len(h.Selectors), 2, "need at least two chains for a directed lane") + for _, sel := range h.Selectors { + if !testRouterDeployedOnSelector(t, h.Env, sel) { + continue + } + for _, other := range h.Selectors { + if other != sel { + return sel, other + } + } + } + return 0, 0 +} + +func TestEnvironmentChangeReconcile_TestRouterLaneThenProductionRouterExpectMessagesSucceedEachStage(t *testing.T) { + h := newEnvironmentChangeReconcileHarness(t) + ctx := ccv.Plog.WithContext(t.Context()) + srcSel, destSel := testRouterSourceAndDestSelectors(t, h) + if srcSel == 0 { + t.Skip("no chain in topology has TestRouter in datastore; redeploy with current devenv (TestRouter is deployed by default)") + } + path := environmentChangeSmokeConfigPath() + th, err := tcapi.NewTestHarness(ctx, path, h.Cfg, chain_selectors.FamilyEVM) + if err != nil { + t.Skipf("message verification needs tcapi harness (aggregators/indexer): %v", err) + } + src, dest := ccip17PairForSelectors(t, ctx, th.Lib, srcSel, destSel) + testRouterLanes := map[uint64]map[uint64]bool{ + srcSel: {destSel: true}, + } + + requireEnvironmentChangeEOADefaultVerifierMessageWithTestRouter(t, ctx, th, h.Cfg, src, dest, false) + + requireEnvironmentChangeReconcile(t, ctx, h, ccv.ReconfigureLanesParams{TestRouterByLane: testRouterLanes}) + requireEnvironmentChangeEOADefaultVerifierMessageWithTestRouter(t, ctx, th, h.Cfg, src, dest, true) + + requireEnvironmentChangeReconcile(t, ctx, h, ccv.ReconfigureLanesParams{}) + requireEnvironmentChangeEOADefaultVerifierMessageWithTestRouter(t, ctx, th, h.Cfg, src, dest, false) +} + +func pickRemovableNOPAliasFromDefaultCommittee(t *testing.T, topo *ccvdeployment.EnvironmentTopology) string { + t.Helper() + require.NotNil(t, topo.NOPTopology) + comm, ok := topo.NOPTopology.Committees[devenvcommon.DefaultCommitteeVerifierQualifier] + require.True(t, ok, "default committee not in topology") + var refAliases []string + for sel, cc := range comm.ChainConfigs { + require.GreaterOrEqual(t, len(cc.NOPAliases), 2, "chain %s needs at least 2 NOPs in default committee", sel) + if refAliases == nil { + refAliases = append([]string(nil), cc.NOPAliases...) + continue + } + require.ElementsMatch(t, refAliases, cc.NOPAliases, "default committee NOP aliases must match across chains for this test") + } + return refAliases[len(refAliases)-1] +} + +func removeNOPAliasFromEveryCommitteeChainConfigs(t *testing.T, topo *ccvdeployment.EnvironmentTopology, removeAlias string) { + t.Helper() + require.NotNil(t, topo.NOPTopology) + for qual, comm := range topo.NOPTopology.Committees { + for sel, cc := range comm.ChainConfigs { + if !slices.Contains(cc.NOPAliases, removeAlias) { + continue + } + next := slices.Clone(cc.NOPAliases) + next = slices.DeleteFunc(next, func(a string) bool { return a == removeAlias }) + require.NotEmpty(t, next, "committee %q chain %s would have no NOPs", qual, sel) + cc.NOPAliases = next + th := cc.Threshold + if int(th) > len(cc.NOPAliases) { + th = uint8(len(cc.NOPAliases)) + } + if th < 1 { + th = 1 + } + cc.Threshold = th + comm.ChainConfigs[sel] = cc + } + topo.NOPTopology.Committees[qual] = comm + } + require.NoError(t, topo.Validate()) +} + +func requireVerifierResultsQuorumExcludesRecoveredSigner(t *testing.T, ar tcapi.AssertionResult, excludedSignerHex string) { + t.Helper() + excluded := common.HexToAddress(strings.TrimSpace(excludedSignerHex)) + require.NotEqual(t, common.Address{}, excluded, "excluded NOP signer must parse as an address") + + for i, row := range ar.IndexedVerifications.Results { + vr := row.VerifierResult + ccvData := vr.CCVData + require.Greater(t, len(ccvData), committee.VerifierVersionLength, "indexed verification %d: ccv data too short", i) + + hash, err := committee.NewSignableHash(vr.MessageID, ccvData) + require.NoError(t, err, "indexed verification %d: signable hash", i) + + rs, ss, err := protocol.DecodeSignatures(ccvData[committee.VerifierVersionLength:]) + require.NoError(t, err, "indexed verification %d: decode quorum signatures from ccv data", i) + + signers, err := protocol.RecoverECDSASigners(hash, rs, ss) + require.NoError(t, err, "indexed verification %d: recover signers from quorum signatures", i) + + for _, sgn := range signers { + require.NotEqualf(t, excluded, sgn, + "indexed verification %d: recovered quorum signer must not be removed NOP %s (got %s)", + i, excludedSignerHex, sgn.Hex()) + } + } + + if ar.AggregatedResult != nil && len(ar.AggregatedResult.CcvData) > committee.VerifierVersionLength && ar.AggregatedResult.Message != nil { + pm, err := ccvcomm.MapProtoMessageToProtocolMessage(ar.AggregatedResult.Message) + require.NoError(t, err) + mid, err := pm.MessageID() + require.NoError(t, err) + ccvData := ar.AggregatedResult.CcvData + hash, err := committee.NewSignableHash(mid, ccvData) + require.NoError(t, err) + rs, ss, err := protocol.DecodeSignatures(ccvData[committee.VerifierVersionLength:]) + require.NoError(t, err) + signers, err := protocol.RecoverECDSASigners(hash, rs, ss) + require.NoError(t, err) + for _, sgn := range signers { + require.NotEqualf(t, excluded, sgn, + "aggregated verifier result: recovered quorum signer must not be removed NOP %s (got %s)", + excludedSignerHex, sgn.Hex()) + } + } +} + +func TestEnvironmentChangeReconcile_RemoveDefaultCommitteeNOPAndLowerThresholdExpectMessageSuccessWithoutRemovedNOPVerification(t *testing.T) { + h := newEnvironmentChangeReconcileHarness(t) + topoSnap, err := toml.Marshal(*h.Cfg.EnvironmentTopology) + require.NoError(t, err) + t.Cleanup(func() { + var restored ccvdeployment.EnvironmentTopology + if err := toml.Unmarshal(topoSnap, &restored); err != nil { + t.Logf("reconcile test cleanup: restore topology: %v", err) + return + } + *h.Cfg.EnvironmentTopology = restored + h.Topology = ccv.BuildEnvironmentTopology(h.Cfg, h.Env) + cleanupCtx := ccv.Plog.WithContext(context.Background()) + if err := ccv.ConfigureTopologyLanesAndOffchain( + cleanupCtx, h.Env, h.Cfg, h.Topology, h.Selectors, h.Cfg.Blockchains, h.Impls, ccv.ReconfigureLanesParams{}, nil, environmentChangeReconcileOpts(), + ); err != nil { + t.Logf("reconcile test cleanup: reconcile: %v", err) + } + }) + + ctx := ccv.Plog.WithContext(environmentChangeLinkedLongContext(t)) + path := environmentChangeSmokeConfigPath() + th, err := tcapi.NewTestHarness(ctx, path, h.Cfg, chain_selectors.FamilyEVM) + if err != nil { + t.Skipf("message verification needs tcapi harness (aggregators/indexer): %v", err) + } + srcSel, destSel := twoDistinctEVMSelectorsFromHarness(t, h) + src, dest := ccip17PairForSelectors(t, ctx, th.Lib, srcSel, destSel) + + built := ccv.BuildEnvironmentTopology(h.Cfg, h.Env) + removeAlias := pickRemovableNOPAliasFromDefaultCommittee(t, built) + evmSigner := requireNOPSigningKeyFromJD(t, h, removeAlias) + + removeNOPAliasFromEveryCommitteeChainConfigs(t, h.Cfg.EnvironmentTopology, removeAlias) + h.Topology = ccv.BuildEnvironmentTopology(h.Cfg, h.Env) + + requireEnvironmentChangeReconcile(t, ctx, h, ccv.ReconfigureLanesParams{}) + + ar, err := runEnvironmentChangeEOADefaultVerifierWithIndexedResult(ctx, th, h.Cfg, src, dest, false) + if errors.Is(err, errEnvironmentChangeEOADefaultVerifierPrerequisites) { + t.Skip("EOA default verifier prerequisites not met for this env") + } + require.NoError(t, err) + requireVerifierResultsQuorumExcludesRecoveredSigner(t, ar, evmSigner) +} diff --git a/build/devenv/tests/e2e/environment_change_signer_rotation_test.go b/build/devenv/tests/e2e/environment_change_signer_rotation_test.go new file mode 100644 index 000000000..4b8becec1 --- /dev/null +++ b/build/devenv/tests/e2e/environment_change_signer_rotation_test.go @@ -0,0 +1,541 @@ +package e2e + +import ( + "context" + "encoding/hex" + "errors" + "fmt" + "slices" + "strconv" + "strings" + "testing" + "time" + + "github.com/BurntSushi/toml" + "github.com/stretchr/testify/require" + + chain_selectors "github.com/smartcontractkit/chain-selectors" + offchainshared "github.com/smartcontractkit/chainlink-ccip/deployment/v2_0_0/offchain/shared" + ccv "github.com/smartcontractkit/chainlink-ccv/build/devenv" + "github.com/smartcontractkit/chainlink-ccv/build/devenv/cciptestinterfaces" + devenvcommon "github.com/smartcontractkit/chainlink-ccv/build/devenv/common" + "github.com/smartcontractkit/chainlink-ccv/build/devenv/jobs" + "github.com/smartcontractkit/chainlink-ccv/build/devenv/services" + "github.com/smartcontractkit/chainlink-ccv/build/devenv/tests/e2e/indexercli" + "github.com/smartcontractkit/chainlink-ccv/build/devenv/tests/e2e/tcapi" + "github.com/smartcontractkit/chainlink-ccv/build/devenv/tests/e2e/verifiercli" + ccvdeployment "github.com/smartcontractkit/chainlink-ccv/deployment" + ccvshared "github.com/smartcontractkit/chainlink-ccv/deployment/shared" + "github.com/smartcontractkit/chainlink-testing-framework/framework" + nodesetpkg "github.com/smartcontractkit/chainlink-testing-framework/framework/components/simple_node_set" +) + +const rotationFakeNOPAlias = "nop-rotation-old-key" + +func snapshotEnvironmentTopologyCleanup(t *testing.T, h *environmentChangeReconcileHarness) { + t.Helper() + topoSnap, err := toml.Marshal(*h.Cfg.EnvironmentTopology) + require.NoError(t, err) + t.Cleanup(func() { + var restored ccvdeployment.EnvironmentTopology + if err := toml.Unmarshal(topoSnap, &restored); err != nil { + t.Logf("signer rotation test cleanup: restore topology: %v", err) + return + } + *h.Cfg.EnvironmentTopology = restored + h.Topology = ccv.BuildEnvironmentTopology(h.Cfg, h.Env) + cleanupCtx := ccv.Plog.WithContext(context.Background()) + if err := ccv.ConfigureTopologyLanesAndOffchain( + cleanupCtx, h.Env, h.Cfg, h.Topology, h.Selectors, h.Cfg.Blockchains, h.Impls, ccv.ReconfigureLanesParams{}, nil, environmentChangeReconcileOpts(), + ); err != nil { + t.Logf("signer rotation test cleanup: reconcile: %v", err) + } + }) +} + +func evmDecimalChainIDsFromHarness(h *environmentChangeReconcileHarness) []string { + var out []string + for _, bc := range h.Cfg.Blockchains { + if bc.Out == nil || bc.Out.Family != chain_selectors.FamilyEVM { + continue + } + if bc.ChainID == "" { + continue + } + out = append(out, bc.ChainID) + } + return out +} + +func requireRotateNOPSigningKey(t *testing.T, ctx context.Context, h *environmentChangeReconcileHarness, nopAlias string) (addrOld, addrNew string) { + t.Helper() + require.NotNil(t, h.Cfg.ClientLookup) + require.NotNil(t, h.Env.Offchain) + require.NotEmpty(t, h.Env.NodeIDs) + clClient, ok := h.Cfg.ClientLookup.GetClient(nopAlias) + require.True(t, ok, "CL client for NOP %q", nopAlias) + lookup, err := offchainshared.FetchNodeLookup(ctx, h.Env.Offchain, h.Env.NodeIDs) + require.NoError(t, err) + node, ok := lookup.FindByName(nopAlias) + require.True(t, ok, "JD node for NOP %q", nopAlias) + chainIDs := evmDecimalChainIDsFromHarness(h) + require.NotEmpty(t, chainIDs) + oldAddr, newAddr, err := jobs.RotateOCR2KeyBundle(ctx, clClient, h.Env.Offchain, node.Id, chainIDs) + require.NoError(t, err) + return oldAddr, newAddr +} + +func updateNOPSignerAddressEVM(t *testing.T, topo *ccvdeployment.EnvironmentTopology, nopAlias, evmHex string) { + t.Helper() + require.NotNil(t, topo.NOPTopology) + ok := topo.NOPTopology.SetNOPSignerAddress(nopAlias, chain_selectors.FamilyEVM, evmHex) + require.True(t, ok, "NOP %q must exist in topology", nopAlias) + require.NoError(t, topo.Validate()) +} + +func addRotationOverlapNOPToTopology(t *testing.T, topo *ccvdeployment.EnvironmentTopology, fakeAlias, evmAddrOld string) { + t.Helper() + require.NotNil(t, topo.NOPTopology) + comm, ok := topo.NOPTopology.Committees[devenvcommon.DefaultCommitteeVerifierQualifier] + require.True(t, ok, "default committee") + fake := ccvdeployment.NOPConfig{ + Alias: fakeAlias, + Name: fakeAlias, + SignerAddressByFamily: map[string]string{ + chain_selectors.FamilyEVM: evmAddrOld, + }, + Mode: ccvshared.NOPModeStandalone, + } + topo.NOPTopology.NOPs = append(topo.NOPTopology.NOPs, fake) + for sel, cc := range comm.ChainConfigs { + next := slices.Clone(cc.NOPAliases) + next = append(next, fakeAlias) + cc.NOPAliases = next + comm.ChainConfigs[sel] = cc + } + topo.NOPTopology.Committees[devenvcommon.DefaultCommitteeVerifierQualifier] = comm + refreshNOPTopologyIndex(t, topo) + require.NoError(t, topo.Validate()) +} + +func removeNOPFromTopology(t *testing.T, topo *ccvdeployment.EnvironmentTopology, alias string) { + t.Helper() + require.NotNil(t, topo.NOPTopology) + topo.NOPTopology.NOPs = slices.DeleteFunc(slices.Clone(topo.NOPTopology.NOPs), func(n ccvdeployment.NOPConfig) bool { + a := n.Alias + if a == "" { + a = n.Name + } + return a == alias + }) + for qual, comm := range topo.NOPTopology.Committees { + for sel, cc := range comm.ChainConfigs { + if !slices.Contains(cc.NOPAliases, alias) { + continue + } + next := slices.Clone(cc.NOPAliases) + next = slices.DeleteFunc(next, func(a string) bool { return a == alias }) + require.NotEmpty(t, next, "committee %q chain %s would have no NOPs", qual, sel) + cc.NOPAliases = next + th := cc.Threshold + if int(th) > len(cc.NOPAliases) { + th = uint8(len(cc.NOPAliases)) + } + if th < 1 { + th = 1 + } + cc.Threshold = th + comm.ChainConfigs[sel] = cc + } + topo.NOPTopology.Committees[qual] = comm + } + refreshNOPTopologyIndex(t, topo) + require.NoError(t, topo.Validate()) +} + +func changeThresholdInDefaultCommitteeChainConfigs(t *testing.T, topo *ccvdeployment.EnvironmentTopology, delta int) { + t.Helper() + require.NotNil(t, topo.NOPTopology) + comm, ok := topo.NOPTopology.Committees[devenvcommon.DefaultCommitteeVerifierQualifier] + require.True(t, ok, "default committee") + for sel, cc := range comm.ChainConfigs { + n := int(cc.Threshold) + delta + require.GreaterOrEqual(t, n, 1, "threshold would drop below 1 on chain %s", sel) + require.LessOrEqual(t, n, len(cc.NOPAliases), "threshold would exceed NOP count on chain %s", sel) + cc.Threshold = uint8(n) + comm.ChainConfigs[sel] = cc + } + topo.NOPTopology.Committees[devenvcommon.DefaultCommitteeVerifierQualifier] = comm + require.NoError(t, topo.Validate()) +} + +func defaultCommitteeFirstChainThresholdAndSize(t *testing.T, topo *ccvdeployment.EnvironmentTopology) (threshold uint8, nopCount int) { + t.Helper() + require.NotNil(t, topo.NOPTopology) + comm, ok := topo.NOPTopology.Committees[devenvcommon.DefaultCommitteeVerifierQualifier] + require.True(t, ok, "default committee") + for _, cc := range comm.ChainConfigs { + return cc.Threshold, len(cc.NOPAliases) + } + require.Fail(t, "default committee has no chain configs") + return 0, 0 +} + +func defaultCommitteeChainThresholdAndSize(t *testing.T, topo *ccvdeployment.EnvironmentTopology, chainSelector uint64) (threshold uint8, nopCount int) { + t.Helper() + require.NotNil(t, topo.NOPTopology) + comm, ok := topo.NOPTopology.Committees[devenvcommon.DefaultCommitteeVerifierQualifier] + require.True(t, ok, "default committee") + cc, ok := comm.ChainConfigs[strconv.FormatUint(chainSelector, 10)] + require.True(t, ok, "default committee chain config for selector %d", chainSelector) + return cc.Threshold, len(cc.NOPAliases) +} + +func requireEnvironmentChangeOnchainOnlyReconcile(t *testing.T, ctx context.Context, h *environmentChangeReconcileHarness, lane ccv.ReconfigureLanesParams) { + t.Helper() + require.NoError(t, ccv.ReconfigureLanesOnchainOnly( + ctx, h.Env, h.Topology, h.Selectors, h.Cfg.Blockchains, h.Impls, lane, + )) +} + +func requireEnvironmentChangeOffchainOnlyReconcile(t *testing.T, ctx context.Context, h *environmentChangeReconcileHarness) { + t.Helper() + require.NoError(t, ccv.ReconfigureOffchainOnly( + ctx, h.Env, h.Cfg, h.Topology, h.Impls, environmentChangeReconcileOpts(), + )) +} + +func indexerContainerName(t *testing.T, cfg *ccv.Cfg) string { + t.Helper() + require.GreaterOrEqual(t, len(cfg.Indexer), 1) + require.NotNil(t, cfg.Indexer[0].Out) + name := cfg.Indexer[0].Out.ContainerName + if len(name) > 0 && name[0] == '/' { + return name[1:] + } + return name +} + +func refreshNOPTopologyIndex(t *testing.T, topo *ccvdeployment.EnvironmentTopology) { + t.Helper() + require.NotNil(t, topo.NOPTopology) + raw, err := toml.Marshal(*topo.NOPTopology) + require.NoError(t, err) + var nt ccvdeployment.NOPTopology + require.NoError(t, toml.Unmarshal(raw, &nt)) + topo.NOPTopology = &nt +} + +func requireIndexerDiscoveryReplaySinceForce(t *testing.T, ctx context.Context, cfg *ccv.Cfg, since uint64) { + t.Helper() + indexer := indexercli.NewClient(indexerContainerName(t, cfg)) + out, err := indexer.DiscoverySince(ctx, strconv.FormatUint(since, 10), true) + require.NoError(t, err, "indexer discovery replay: %s", out) +} + +func requireIndexerReplayMessagesByIDsForce(t *testing.T, ctx context.Context, cfg *ccv.Cfg, msgIDHex string) { + t.Helper() + indexer := indexercli.NewClient(indexerContainerName(t, cfg)) + out, err := indexer.MessagesByID(ctx, msgIDHex, true) + require.NoError(t, err, "indexer messages replay: %s", out) +} + +func requireIndexerMessagesDiscoveryMessagesReplayByIDsForce(t *testing.T, ctx context.Context, cfg *ccv.Cfg, discoverySince uint64, msgIDHex string) { + t.Helper() + requireIndexerReplayMessagesByIDsForce(t, ctx, cfg, msgIDHex) + requireIndexerDiscoveryReplaySinceForce(t, ctx, cfg, discoverySince) + requireIndexerReplayMessagesByIDsForce(t, ctx, cfg, msgIDHex) +} + +func clNodeContainerAtGlobalSpecIndex(t *testing.T, cfg *ccv.Cfg, globalIdx int) string { + t.Helper() + g := 0 + for _, nsi := range cfg.NodeSets { + require.NotNil(t, nsi, "nodeset input") + require.NotNil(t, nsi.Out, "nodeset output required to resolve CL container") + httpStart := nsi.HTTPPortRangeStart + if httpStart == 0 { + httpStart = nodesetpkg.DefaultHTTPPortStaticRangeStart + } + for li := range nsi.NodeSpecs { + if g == globalIdx { + wantPort := httpStart + li + needle := ":" + strconv.Itoa(wantPort) + for _, cl := range nsi.Out.CLNodes { + require.NotNil(t, cl) + require.NotNil(t, cl.Node) + if strings.Contains(cl.Node.ExternalURL, needle) { + return services.NormalizeDockerContainerName(cl.Node.ContainerName) + } + } + require.Fail(t, fmt.Sprintf("no CL node found for expected HTTP port (port=%d nodeset=%q)", wantPort, nsi.Name)) + } + g++ + } + } + require.Fail(t, fmt.Sprintf("global node spec index out of range: %d", globalIdx)) + return "" +} + +func clNodeContainerForVerifier(t *testing.T, cfg *ccv.Cfg, nopAlias string) string { + t.Helper() + topo := cfg.EnvironmentTopology + require.NotNil(t, topo) + require.NotNil(t, topo.NOPTopology) + idx, ok := topo.NOPTopology.GetNOPIndex(nopAlias) + require.True(t, ok, "NOP alias %q not in topology", nopAlias) + return clNodeContainerAtGlobalSpecIndex(t, cfg, idx) +} + +func requireRestartDefaultExecutorContainers(t *testing.T, ctx context.Context, cfg *ccv.Cfg) { + t.Helper() + seen := make(map[string]struct{}) + var containerNames []string + for _, exec := range cfg.Executor { + if exec == nil || exec.ExecutorQualifier != devenvcommon.DefaultExecutorQualifier { + continue + } + var containerName string + if exec.Out != nil && exec.Out.ContainerName != "" { + containerName = services.NormalizeDockerContainerName(exec.Out.ContainerName) + } else if exec.NOPAlias != "" { + containerName = clNodeContainerForVerifier(t, cfg, exec.NOPAlias) + } + require.NotEmpty(t, containerName, "default executor container name") + if _, ok := seen[containerName]; ok { + continue + } + seen[containerName] = struct{}{} + containerNames = append(containerNames, containerName) + } + slices.Sort(containerNames) + require.NotEmpty(t, containerNames, "default executor container names") + for _, containerName := range containerNames { + require.NoError(t, verifiercli.NewCLNodeClient(containerName).RestartAndWaitReady(ctx)) + } +} + +func rewindCLVerifierSourceHeights(t *testing.T, ctx context.Context, containerName string, verifierIDs []string, srcSel uint64) { + t.Helper() + require.NotEmpty(t, containerName) + seen := make(map[string]struct{}) + var ids []string + for _, id := range verifierIDs { + if id == "" { + continue + } + if _, ok := seen[id]; ok { + continue + } + seen[id] = struct{}{} + ids = append(ids, id) + } + require.NotEmpty(t, ids, "verifier IDs for CL rewind") + vc := verifiercli.NewCLNodeClient(containerName) + require.NoError(t, vc.Pause(ctx)) + t.Cleanup(func() { vc.ResumeBestEffort(ctx) }) + for _, verifierID := range ids { + _, err := vc.ChainStatuses().SetFinalizedHeight(ctx, + verifiercli.FormatChainSelector(srcSel), + verifierID, + verifiercli.FormatBlockHeight(0), + ) + require.NoError(t, err) + } + require.NoError(t, vc.RestartAndWaitReady(ctx)) +} + +func requireRewindAllDefaultVerifierSourceHeights(t *testing.T, ctx context.Context, cfg *ccv.Cfg, srcSel uint64) { + t.Helper() + type clRewind struct { + containerName string + verifierIDs []string + } + clByNOP := make(map[string]*clRewind) + rewound := 0 + for _, v := range cfg.Verifier { + if v == nil || v.CommitteeName != devenvcommon.DefaultCommitteeVerifierQualifier { + continue + } + if v.Out == nil || v.Out.VerifierID == "" { + continue + } + switch v.Mode { + case services.Standalone: + if v.Out.ContainerName == "" { + continue + } + containerName := services.NormalizeDockerContainerName(v.Out.ContainerName) + vc := verifiercli.NewClient(containerName) + require.NoError(t, vc.Pause(ctx)) + t.Cleanup(func() { vc.ResumeBestEffort(ctx) }) + _, err := vc.ChainStatuses().SetFinalizedHeight(ctx, verifiercli.FormatChainSelector(srcSel), v.Out.VerifierID, verifiercli.FormatBlockHeight(0)) + require.NoError(t, err) + require.NoError(t, vc.RestartAndWaitReady(ctx)) + rewound++ + case services.CL: + cr, ok := clByNOP[v.NOPAlias] + if !ok { + cr = &clRewind{containerName: clNodeContainerForVerifier(t, cfg, v.NOPAlias)} + clByNOP[v.NOPAlias] = cr + } + cr.verifierIDs = append(cr.verifierIDs, v.Out.VerifierID) + rewound++ + default: + continue + } + } + nopAliases := make([]string, 0, len(clByNOP)) + for a := range clByNOP { + nopAliases = append(nopAliases, a) + } + slices.Sort(nopAliases) + for _, a := range nopAliases { + cr := clByNOP[a] + rewindCLVerifierSourceHeights(t, ctx, cr.containerName, cr.verifierIDs, srcSel) + } + require.Greater(t, rewound, 0, "no default committee verifier with verifier_id and rewind target (standalone container or CL topology) found") +} + +func TestEnvironmentChangeReconcile_SignerKeyRotationExpectMessagesSucceedEachPhase(t *testing.T) { + h := newEnvironmentChangeReconcileHarness(t) + snapshotEnvironmentTopologyCleanup(t, h) + ctx := ccv.Plog.WithContext(environmentChangeLinkedLongContext(t)) + path := environmentChangeSmokeConfigPath() + th, err := tcapi.NewTestHarness(ctx, path, h.Cfg, chain_selectors.FamilyEVM) + if err != nil { + t.Skipf("message verification needs tcapi harness: %v", err) + } + srcSel, destSel := twoDistinctEVMSelectorsFromHarness(t, h) + src, dest := ccip17PairForSelectors(t, ctx, th.Lib, srcSel, destSel) + + oldNOP := pickRemovableNOPAliasFromDefaultCommittee(t, h.Topology) + addrOld, addrNew := requireRotateNOPSigningKey(t, ctx, h, oldNOP) + updateNOPSignerAddressEVM(t, h.Cfg.EnvironmentTopology, oldNOP, addrNew) + require.NotEqualf(t, strings.ToLower(addrOld), strings.ToLower(addrNew), "rotation must change address") + + addRotationOverlapNOPToTopology(t, h.Cfg.EnvironmentTopology, rotationFakeNOPAlias, addrOld) + h.Topology = ccv.BuildEnvironmentTopology(h.Cfg, h.Env) + requireEnvironmentChangeReconcile(t, ctx, h, ccv.ReconfigureLanesParams{}) + requireEnvironmentChangeEOADefaultVerifierMessage(t, ctx, th, h.Cfg, src, dest) + + removeNOPFromTopology(t, h.Cfg.EnvironmentTopology, rotationFakeNOPAlias) + h.Topology = ccv.BuildEnvironmentTopology(h.Cfg, h.Env) + requireEnvironmentChangeReconcile(t, ctx, h, ccv.ReconfigureLanesParams{}) + + ar, err := runEnvironmentChangeEOADefaultVerifierWithIndexedResult(ctx, th, h.Cfg, src, dest, false) + if errors.Is(err, errEnvironmentChangeEOADefaultVerifierPrerequisites) { + t.Skip("EOA default verifier prerequisites not met for this env") + } + require.NoError(t, err) + requireVerifierResultsQuorumExcludesRecoveredSigner(t, ar, addrOld) +} + +func TestEnvironmentChangeReconcile_SignerKeyRotationOffchainFirstExpectMessageSuccessAfterFullReconcile(t *testing.T) { + h := newEnvironmentChangeReconcileHarness(t) + snapshotEnvironmentTopologyCleanup(t, h) + ctx := ccv.Plog.WithContext(environmentChangeLinkedLongContext(t)) + path := environmentChangeSmokeConfigPath() + th, err := tcapi.NewTestHarness(ctx, path, h.Cfg, chain_selectors.FamilyEVM) + if err != nil { + t.Skipf("message verification needs tcapi harness: %v", err) + } + srcSel, destSel := twoDistinctEVMSelectorsFromHarness(t, h) + src, dest := ccip17PairForSelectors(t, ctx, th.Lib, srcSel, destSel) + + T, N := defaultCommitteeChainThresholdAndSize(t, h.Topology, srcSel) + if T == uint8(N) { + t.Skipf("off-chain-first rotation needs T < N on source chain %d so remaining NOPs can meet threshold while one NOP is ignored", srcSel) + } + + requireEnvironmentChangeReconcile(t, ctx, h, ccv.ReconfigureLanesParams{}) + + oldNOP := pickRemovableNOPAliasFromDefaultCommittee(t, h.Topology) + _ = requireNOPSigningKeyFromJD(t, h, oldNOP) + addrOld, addrNew := requireRotateNOPSigningKey(t, ctx, h, oldNOP) + updateNOPSignerAddressEVM(t, h.Cfg.EnvironmentTopology, oldNOP, addrNew) + h.Topology = ccv.BuildEnvironmentTopology(h.Cfg, h.Env) + + requireEnvironmentChangeOffchainOnlyReconcile(t, ctx, h) + requireEnvironmentChangeEOADefaultVerifierMessage(t, ctx, th, h.Cfg, src, dest) + + requireEnvironmentChangeReconcile(t, ctx, h, ccv.ReconfigureLanesParams{}) + ar, err := runEnvironmentChangeEOADefaultVerifierWithIndexedResult(ctx, th, h.Cfg, src, dest, false) + if errors.Is(err, errEnvironmentChangeEOADefaultVerifierPrerequisites) { + t.Skip("EOA default verifier prerequisites not met for this env") + } + require.NoError(t, err) + requireVerifierResultsQuorumExcludesRecoveredSigner(t, ar, addrOld) +} + +func TestEnvironmentChangeReconcile_ThresholdDecreaseExpectMessageSuccess(t *testing.T) { + h := newEnvironmentChangeReconcileHarness(t) + snapshotEnvironmentTopologyCleanup(t, h) + ctx := ccv.Plog.WithContext(environmentChangeLinkedLongContext(t)) + path := environmentChangeSmokeConfigPath() + th, err := tcapi.NewTestHarness(ctx, path, h.Cfg, chain_selectors.FamilyEVM) + if err != nil { + t.Skipf("message verification needs tcapi harness: %v", err) + } + srcSel, destSel := twoDistinctEVMSelectorsFromHarness(t, h) + src, dest := ccip17PairForSelectors(t, ctx, th.Lib, srcSel, destSel) + + T, _ := defaultCommitteeFirstChainThresholdAndSize(t, h.Topology) + if T < 2 { + t.Skip("threshold decrease test needs initial threshold >= 2") + } + requireEnvironmentChangeReconcile(t, ctx, h, ccv.ReconfigureLanesParams{}) + requireEnvironmentChangeEOADefaultVerifierMessage(t, ctx, th, h.Cfg, src, dest) + + changeThresholdInDefaultCommitteeChainConfigs(t, h.Cfg.EnvironmentTopology, -1) + h.Topology = ccv.BuildEnvironmentTopology(h.Cfg, h.Env) + requireEnvironmentChangeReconcile(t, ctx, h, ccv.ReconfigureLanesParams{}) + requireEnvironmentChangeEOADefaultVerifierMessage(t, ctx, th, h.Cfg, src, dest) +} + +func TestEnvironmentChangeReconcile_ThresholdIncreaseRecoveryExpectMessageSuccessAfterFullReconcile(t *testing.T) { + h := newEnvironmentChangeReconcileHarness(t) + snapshotEnvironmentTopologyCleanup(t, h) + ctx := ccv.Plog.WithContext(environmentChangeLinkedLongContext(t)) + path := environmentChangeSmokeConfigPath() + th, err := tcapi.NewTestHarness(ctx, path, h.Cfg, chain_selectors.FamilyEVM) + if err != nil { + t.Skipf("message verification needs tcapi harness: %v", err) + } + srcSel, destSel := twoDistinctEVMSelectorsFromHarness(t, h) + src, dest := ccip17PairForSelectors(t, ctx, th.Lib, srcSel, destSel) + + T, N := defaultCommitteeFirstChainThresholdAndSize(t, h.Topology) + if int(T) >= N { + t.Skip("threshold increase test needs T < N") + } + + requireEnvironmentChangeReconcile(t, ctx, h, ccv.ReconfigureLanesParams{}) + requireEnvironmentChangeEOADefaultVerifierMessage(t, ctx, th, h.Cfg, src, dest) + + changeThresholdInDefaultCommitteeChainConfigs(t, h.Cfg.EnvironmentTopology, +1) + h.Topology = ccv.BuildEnvironmentTopology(h.Cfg, h.Env) + requireEnvironmentChangeOnchainOnlyReconcile(t, ctx, h, ccv.ReconfigureLanesParams{}) + + msgID, _, _, execState, err := runEnvironmentChangeEOADefaultVerifierIndexedResultUntilExecStops(ctx, th, h.Cfg, src, dest, false) + if errors.Is(err, errEnvironmentChangeEOADefaultVerifierPrerequisites) { + t.Skip("EOA default verifier prerequisites not met for this env") + } + require.NoError(t, err) + require.Equal(t, cciptestinterfaces.ExecutionStateFailure, execState, "expected execution failure while on-chain threshold exceeds off-chain aggregation") + + requireEnvironmentChangeReconcile(t, ctx, h, ccv.ReconfigureLanesParams{}) + + requireRewindAllDefaultVerifierSourceHeights(t, ctx, h.Cfg, srcSel) + // Wait for the re-aggregation to complete + time.Sleep(10 * time.Second) + + msgHex := "0x" + hex.EncodeToString(msgID[:]) + requireIndexerMessagesDiscoveryMessagesReplayByIDsForce(t, ctx, h.Cfg, 3, msgHex) + requireRestartDefaultExecutorContainers(t, ctx, h.Cfg) + t.Cleanup(func() { + _, _ = framework.SaveContainerLogs(fmt.Sprintf("%s-%s", framework.DefaultCTFLogsDir, t.Name()+"-replay")) + }) + + requireEnvironmentChangeMessageReachesExecutionSuccess(t, ctx, dest, msgID) +} diff --git a/build/devenv/tests/e2e/indexercli/client.go b/build/devenv/tests/e2e/indexercli/client.go new file mode 100644 index 000000000..982410221 --- /dev/null +++ b/build/devenv/tests/e2e/indexercli/client.go @@ -0,0 +1,89 @@ +// Package indexercli is a test-only client for the indexer replay CLI +// inside a running indexer container. +package indexercli + +import ( + "context" + "fmt" + "os/exec" + "strings" +) + +const DefaultReplayBinary = "/bin/indexer-replay" + +// Client talks to the indexer replay CLI inside one container. +type Client struct { + containerName string + replayBinary string +} + +// Option configures a Client. +type Option func(*Client) + +// WithReplayBinary overrides the in-container replay binary path. +func WithReplayBinary(path string) Option { + return func(c *Client) { c.replayBinary = path } +} + +// NewClient returns a Client bound to containerName. A leading slash from +// Docker inspect output is stripped so callers can pass container names through. +func NewClient(containerName string, opts ...Option) *Client { + c := &Client{ + containerName: strings.TrimPrefix(containerName, "/"), + replayBinary: DefaultReplayBinary, + } + for _, opt := range opts { + opt(c) + } + return c +} + +// Container returns the normalized container name. +func (c *Client) Container() string { + return c.containerName +} + +// Exec runs docker exec against the bound indexer container. +func (c *Client) Exec(ctx context.Context, args ...string) (string, error) { + full := append([]string{"exec", c.containerName}, args...) + cmd := exec.CommandContext(ctx, "docker", full...) + out, err := cmd.CombinedOutput() + if err != nil { + return string(out), fmt.Errorf("docker exec %s %v: %w (output: %s)", c.containerName, args, err, string(out)) + } + return string(out), nil +} + +// Replay runs an indexer-replay subcommand. +func (c *Client) Replay(ctx context.Context, subcommand string, args ...string) (string, error) { + full := append([]string{c.replayBinary, subcommand}, args...) + return c.Exec(ctx, full...) +} + +// List runs indexer-replay list. +func (c *Client) List(ctx context.Context) (string, error) { + return c.Replay(ctx, "list") +} + +// Status runs indexer-replay status. +func (c *Client) Status(ctx context.Context, id string) (string, error) { + return c.Replay(ctx, "status", "--id", id) +} + +// DiscoverySince runs indexer-replay discovery --since, optionally with --force. +func (c *Client) DiscoverySince(ctx context.Context, since string, force bool) (string, error) { + args := []string{"--since", since} + if force { + args = append(args, "--force") + } + return c.Replay(ctx, "discovery", args...) +} + +// MessagesByID runs indexer-replay messages --ids for one message, optionally with --force. +func (c *Client) MessagesByID(ctx context.Context, msgIDHex string, force bool) (string, error) { + args := []string{"--ids", msgIDHex} + if force { + args = append(args, "--force") + } + return c.Replay(ctx, "messages", args...) +} diff --git a/build/devenv/tests/e2e/smoke_replay_cli_test.go b/build/devenv/tests/e2e/smoke_replay_cli_test.go index 0dfbf2ab0..a4b1d96e4 100644 --- a/build/devenv/tests/e2e/smoke_replay_cli_test.go +++ b/build/devenv/tests/e2e/smoke_replay_cli_test.go @@ -5,7 +5,6 @@ import ( "database/sql" "encoding/hex" "fmt" - "os/exec" "strings" "testing" "time" @@ -20,20 +19,13 @@ import ( ccv "github.com/smartcontractkit/chainlink-ccv/build/devenv" "github.com/smartcontractkit/chainlink-ccv/build/devenv/cciptestinterfaces" devenvcommon "github.com/smartcontractkit/chainlink-ccv/build/devenv/common" + "github.com/smartcontractkit/chainlink-ccv/build/devenv/tests/e2e/indexercli" "github.com/smartcontractkit/chainlink-ccv/build/devenv/tests/e2e/tcapi" "github.com/smartcontractkit/chainlink-ccv/protocol" "github.com/smartcontractkit/chainlink-deployments-framework/datastore" "github.com/smartcontractkit/chainlink-testing-framework/framework" ) -const replayBinary = "/bin/indexer-replay" - -func execInContainer(ctx context.Context, containerName string, args ...string) (string, error) { - cmd := exec.CommandContext(ctx, "docker", append([]string{"exec", containerName}, args...)...) - out, err := cmd.CombinedOutput() - return string(out), err -} - func openIndexerDB(t *testing.T, in *ccv.Cfg) (*sql.DB, string) { t.Helper() require.GreaterOrEqual(t, len(in.Indexer), 1, "expected at least one indexer") @@ -56,10 +48,6 @@ func openIndexerDB(t *testing.T, in *ccv.Cfg) (*sql.DB, string) { return db, containerName } -func replayCLIArgs(subcommand string, extra ...string) []string { - return append([]string{replayBinary, subcommand}, extra...) -} - // TestE2ESmoke_ReplayCLI verifies the replay CLI subcommands work end-to-end: // migration check, list, status, and a discovery dry-run. func TestE2ESmoke_ReplayCLI(t *testing.T) { @@ -68,6 +56,7 @@ func TestE2ESmoke_ReplayCLI(t *testing.T) { require.NoError(t, err) db, containerName := openIndexerDB(t, in) + indexer := indexercli.NewClient(containerName) const fakeJobID = "a1b2c3d4-e5f6-7890-abcd-ef1234567890" @@ -81,7 +70,7 @@ func TestE2ESmoke_ReplayCLI(t *testing.T) { }) t.Run("list empty", func(t *testing.T) { - out, err := execInContainer(t.Context(), containerName, replayCLIArgs("list")...) + out, err := indexer.List(t.Context()) require.NoError(t, err, "list should succeed; output: %s", out) require.Contains(t, out, "No replay jobs found", "empty list should report no jobs; output: %s", out) }) @@ -95,20 +84,20 @@ func TestE2ESmoke_ReplayCLI(t *testing.T) { ) require.NoError(t, err, "seeding fake replay job") - out, err := execInContainer(ctx, containerName, replayCLIArgs("list")...) + out, err := indexer.List(ctx) require.NoError(t, err, "list should succeed; output: %s", out) require.Contains(t, out, fakeJobID, "list output must contain the seeded job ID; output: %s", out) }) t.Run("status", func(t *testing.T) { - out, err := execInContainer(t.Context(), containerName, replayCLIArgs("status", "--id", fakeJobID)...) + out, err := indexer.Status(t.Context(), fakeJobID) require.NoError(t, err, "status should succeed; output: %s", out) require.Contains(t, out, fakeJobID, "status output must contain the job ID; output: %s", out) require.Contains(t, out, "completed", "status output must show completed status; output: %s", out) }) t.Run("discovery", func(t *testing.T) { - out, err := execInContainer(t.Context(), containerName, replayCLIArgs("discovery", "--since", "1")...) + out, err := indexer.DiscoverySince(t.Context(), "1", false) require.NoError(t, err, "discovery replay should succeed with sequence 1; output: %s", out) }) } @@ -241,8 +230,8 @@ func TestE2ESmoke_ReplayForceOverwrite(t *testing.T) { // ── Step 2: replay msg1 only with --force via --ids ───────────────────── t.Log("Step 2: replaying msg1 with messages --ids --force...") - out, err := execInContainer(ctx, containerName, - replayCLIArgs("messages", "--ids", msgHex1, "--force")...) + indexer := indexercli.NewClient(containerName) + out, err := indexer.MessagesByID(ctx, msgHex1, true) require.NoError(t, err, "messages replay failed; output: %s", out) ts1AfterIDs, err := getIngestionTimestamp(ctx, db, msgHex1) @@ -261,8 +250,7 @@ func TestE2ESmoke_ReplayForceOverwrite(t *testing.T) { // ── Step 3: replay both with --force via discovery --since ─────────────── t.Logf("Step 3: replaying both with discovery --since %s --force...", discoverySince) - out, err = execInContainer(ctx, containerName, - replayCLIArgs("discovery", "--since", discoverySince, "--force")...) + out, err = indexer.DiscoverySince(ctx, discoverySince, true) require.NoError(t, err, "discovery force replay failed; output: %s", out) ts1AfterDisc, err := getIngestionTimestamp(ctx, db, msgHex1) @@ -281,8 +269,7 @@ func TestE2ESmoke_ReplayForceOverwrite(t *testing.T) { // ── Step 4: replay without --force (backfill-only, nothing to fill) ───── t.Logf("Step 4: replaying with discovery --since %s (no --force)...", discoverySince) - out, err = execInContainer(ctx, containerName, - replayCLIArgs("discovery", "--since", discoverySince)...) + out, err = indexer.DiscoverySince(ctx, discoverySince, false) require.NoError(t, err, "discovery backfill replay failed; output: %s", out) ts1AfterBackfill, err := getIngestionTimestamp(ctx, db, msgHex1) diff --git a/build/devenv/tests/e2e/tcapi/basic/v3.go b/build/devenv/tests/e2e/tcapi/basic/v3.go index eb98e31b6..d5b72b6b9 100644 --- a/build/devenv/tests/e2e/tcapi/basic/v3.go +++ b/build/devenv/tests/e2e/tcapi/basic/v3.go @@ -37,10 +37,11 @@ type v3TestCaseBase struct { // v3TestCase is for tests that use ExtraArgsV3. type v3TestCase struct { v3TestCaseBase - receiver protocol.UnknownAddress - ccvs []protocol.CCV - executor protocol.UnknownAddress - hydrate func(ctx context.Context, tc *v3TestCase, cfg *ccv.Cfg) bool + receiver protocol.UnknownAddress + ccvs []protocol.CCV + useTestRouter bool + executor protocol.UnknownAddress + hydrate func(ctx context.Context, tc *v3TestCase, cfg *ccv.Cfg) bool } func (tc *v3TestCase) Name() string { @@ -64,6 +65,7 @@ func (tc *v3TestCase) Run(ctx context.Context, harness tcapi.TestHarness, cfg *c FinalityConfig: tc.finality, Executor: tc.executor, CCVs: tc.ccvs, + UseTestRouter: tc.useTestRouter, }) if err != nil { return fmt.Errorf("failed to send message: %w", err) diff --git a/build/devenv/tests/e2e/verifiercli/chainlink_node.go b/build/devenv/tests/e2e/verifiercli/chainlink_node.go new file mode 100644 index 000000000..1e1c61df6 --- /dev/null +++ b/build/devenv/tests/e2e/verifiercli/chainlink_node.go @@ -0,0 +1,171 @@ +package verifiercli + +import ( + "context" + "fmt" + "os/exec" + "strings" + "time" +) + +const ( + DefaultCLNodeBinaryPath = "chainlink" + DefaultCLNodeProcessName = "chainlink" + DefaultCLNodePassword = "/config/node_password" + DefaultCLNodeHealthURL = "http://localhost:6688/health" +) + +var DefaultCLNodeConfigArgs = []string{ + "-c", "/config/config", + "-c", "/config/overrides", + "-c", "/config/user-overrides", + "-s", "/config/secrets", + "-s", "/config/secrets-overrides", + "-s", "/config/user-secrets-overrides", +} + +var CLNodeChainStatusesSubcommand = []string{"local", "ccv", "chain-statuses"} + +// CLNodeClient talks to the verifier CLI exposed through a full Chainlink node. +type CLNodeClient struct { + containerName string + binaryPath string + processMatch string + configArgs []string + passwordFile string + healthURL string +} + +// CLNodeOption configures a CLNodeClient. +type CLNodeOption func(*CLNodeClient) + +// WithCLNodeBinaryPath overrides the in-container chainlink binary path. +func WithCLNodeBinaryPath(path string) CLNodeOption { + return func(c *CLNodeClient) { c.binaryPath = path } +} + +// WithCLNodeProcessMatch overrides the process match used for Pause and Resume. +func WithCLNodeProcessMatch(match string) CLNodeOption { + return func(c *CLNodeClient) { c.processMatch = match } +} + +// WithCLNodeConfigArgs overrides the chainlink -c/-s config flags. +func WithCLNodeConfigArgs(args ...string) CLNodeOption { + return func(c *CLNodeClient) { c.configArgs = append([]string(nil), args...) } +} + +// WithCLNodePasswordFile overrides the --password file path. Empty disables the flag. +func WithCLNodePasswordFile(path string) CLNodeOption { + return func(c *CLNodeClient) { c.passwordFile = path } +} + +// WithCLNodeHealthURL overrides the local health endpoint probed after restart. +func WithCLNodeHealthURL(url string) CLNodeOption { + return func(c *CLNodeClient) { c.healthURL = url } +} + +// NewCLNodeClient returns a Client bound to a Chainlink node container. +func NewCLNodeClient(containerName string, opts ...CLNodeOption) *CLNodeClient { + c := &CLNodeClient{ + containerName: strings.TrimPrefix(containerName, "/"), + binaryPath: DefaultCLNodeBinaryPath, + processMatch: DefaultCLNodeProcessName, + configArgs: append([]string(nil), DefaultCLNodeConfigArgs...), + passwordFile: DefaultCLNodePassword, + healthURL: DefaultCLNodeHealthURL, + } + for _, opt := range opts { + opt(c) + } + return c +} + +// Container returns the normalized container name. +func (c *CLNodeClient) Container() string { + return c.containerName +} + +// Exec runs docker exec against the bound Chainlink node container. +func (c *CLNodeClient) Exec(ctx context.Context, args ...string) (string, error) { + full := append([]string{"exec", c.containerName}, args...) + cmd := exec.CommandContext(ctx, "docker", full...) + out, err := cmd.CombinedOutput() + if err != nil { + return string(out), fmt.Errorf("docker exec %s %v: %w (output: %s)", c.containerName, args, err, string(out)) + } + return string(out), nil +} + +// CLI runs the chainlink local ccv subcommand tree. +func (c *CLNodeClient) CLI(ctx context.Context, subcommand []string, args ...string) (string, error) { + full := append([]string{c.binaryPath}, c.configArgs...) + full = append(full, subcommand...) + if c.passwordFile != "" { + full = append(full, "--password", c.passwordFile) + } + full = append(full, args...) + return c.Exec(ctx, full...) +} + +// Pause stops the Chainlink process while tests mutate local CLI state. +func (c *CLNodeClient) Pause(ctx context.Context) error { + _, err := c.Exec(ctx, "pkill", "-STOP", "-f", c.processMatch) + return err +} + +// Resume continues a process stopped by Pause. +func (c *CLNodeClient) Resume(ctx context.Context) error { + _, err := c.Exec(ctx, "pkill", "-CONT", "-f", c.processMatch) + return err +} + +// ResumeBestEffort resumes without propagating errors for cleanup paths. +func (c *CLNodeClient) ResumeBestEffort(ctx context.Context) { + _, _ = c.Exec(ctx, "pkill", "-CONT", "-f", c.processMatch) +} + +// RestartAndWaitReady restarts the node container and polls its local health endpoint. +func (c *CLNodeClient) RestartAndWaitReady(ctx context.Context) error { + restartCmd := exec.CommandContext(ctx, "docker", "restart", c.containerName) + if out, err := restartCmd.CombinedOutput(); err != nil { + return fmt.Errorf("docker restart %s: %w (output: %s)", c.containerName, err, string(out)) + } + + deadline := time.Now().Add(defaultRestartReadyTimeout) + var lastErr error + for time.Now().Before(deadline) { + if _, err := c.Exec(ctx, "curl", "-sf", c.healthURL); err == nil { + return nil + } else { + lastErr = err + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(defaultRestartReadyInterval): + } + } + if lastErr == nil { + lastErr = fmt.Errorf("no health probe error recorded") + } + return fmt.Errorf("CL node %s not healthy within %s: %w", c.containerName, defaultRestartReadyTimeout, lastErr) +} + +// CLNodeChainStatusesClient wraps the chainlink local ccv chain-statuses commands. +type CLNodeChainStatusesClient struct { + client *CLNodeClient +} + +// ChainStatuses returns a sub-client for chain-statuses operations. +func (c *CLNodeClient) ChainStatuses() CLNodeChainStatusesClient { + return CLNodeChainStatusesClient{client: c} +} + +// SetFinalizedHeight rewinds or advances a verifier's finalized source height. +func (s CLNodeChainStatusesClient) SetFinalizedHeight(ctx context.Context, sel ChainSelector, verifierID string, height BlockHeight) (string, error) { + return s.client.CLI(ctx, CLNodeChainStatusesSubcommand, + "set-finalized-height", + "--chain-selector", string(sel), + "--verifier-id", verifierID, + "--block-height", string(height)) +}