diff --git a/docs/docs/resources/pipeline-components/dependencies/kpops_structure.yaml b/docs/docs/resources/pipeline-components/dependencies/kpops_structure.yaml index f5e8f205a..20157432f 100644 --- a/docs/docs/resources/pipeline-components/dependencies/kpops_structure.yaml +++ b/docs/docs/resources/pipeline-components/dependencies/kpops_structure.yaml @@ -1,6 +1,7 @@ kpops_components_fields: helm-app: - name + - enabled - prefix - from_ - to @@ -11,11 +12,13 @@ kpops_components_fields: - version kafka-app: - name + - enabled - prefix - from_ - to kafka-connector: - name + - enabled - prefix - from_ - to @@ -25,6 +28,7 @@ kpops_components_fields: - resetter_values kafka-sink-connector: - name + - enabled - prefix - from_ - to @@ -34,6 +38,7 @@ kpops_components_fields: - resetter_values kafka-source-connector: - name + - enabled - prefix - from_ - to @@ -44,6 +49,7 @@ kpops_components_fields: - offset_topic kubernetes-app: - name + - enabled - prefix - from_ - to @@ -51,11 +57,13 @@ kpops_components_fields: - values pipeline-component: - name + - enabled - prefix - from_ - to producer-app: - name + - enabled - prefix - from_ - to @@ -66,6 +74,7 @@ kpops_components_fields: - version producer-app-v2: - name + - enabled - prefix - from_ - to @@ -76,6 +85,7 @@ kpops_components_fields: - version streams-app: - name + - enabled - prefix - from_ - to @@ -86,6 +96,7 @@ kpops_components_fields: - version streams-app-v2: - name + - enabled - prefix - from_ - to @@ -96,6 +107,7 @@ kpops_components_fields: - version streams-bootstrap: - name + - enabled - prefix - from_ - to @@ -106,6 +118,7 @@ kpops_components_fields: - version streams-bootstrap-v2: - name + - enabled - prefix - from_ - to diff --git a/docs/docs/schema/defaults.json b/docs/docs/schema/defaults.json index 7ed023a1d..c1dc4dbea 100644 --- a/docs/docs/schema/defaults.json +++ b/docs/docs/schema/defaults.json @@ -130,6 +130,12 @@ }, "description": "Helm diff config" }, + "enabled": { + "default": true, + "description": "Whether the component is enabled and should be included in the pipeline", + "title": "Enabled", + "type": "boolean" + }, "from": { "anyOf": [ { @@ -563,6 +569,12 @@ "additionalProperties": true, "description": "Base component for Kafka-based components.", "properties": { + "enabled": { + "default": true, + "description": "Whether the component is enabled and should be included in the pipeline", + "title": "Enabled", + "type": "boolean" + }, "from": { "anyOf": [ { @@ -669,6 +681,12 @@ "$ref": "#/$defs/KafkaConnectorConfig", "description": "Connector config" }, + "enabled": { + "default": true, + "description": "Whether the component is enabled and should be included in the pipeline", + "title": "Enabled", + "type": "boolean" + }, "from": { "anyOf": [ { @@ -814,6 +832,12 @@ "$ref": "#/$defs/KafkaConnectorConfig", "description": "Connector config" }, + "enabled": { + "default": true, + "description": "Whether the component is enabled and should be included in the pipeline", + "title": "Enabled", + "type": "boolean" + }, "from": { "anyOf": [ { @@ -901,6 +925,12 @@ "$ref": "#/$defs/KafkaConnectorConfig", "description": "Connector config" }, + "enabled": { + "default": true, + "description": "Whether the component is enabled and should be included in the pipeline", + "title": "Enabled", + "type": "boolean" + }, "from": { "anyOf": [ { @@ -1047,6 +1077,12 @@ "additionalProperties": true, "description": "Base class for all Kubernetes apps.\nAll built-in components are Kubernetes apps, except for the Kafka connectors.", "properties": { + "enabled": { + "default": true, + "description": "Whether the component is enabled and should be included in the pipeline", + "title": "Enabled", + "type": "boolean" + }, "from": { "anyOf": [ { @@ -1379,6 +1415,12 @@ "additionalProperties": true, "description": "Base class for all components.", "properties": { + "enabled": { + "default": true, + "description": "Whether the component is enabled and should be included in the pipeline", + "title": "Enabled", + "type": "boolean" + }, "from": { "anyOf": [ { @@ -1626,6 +1668,12 @@ }, "description": "Helm diff config" }, + "enabled": { + "default": true, + "description": "Whether the component is enabled and should be included in the pipeline", + "title": "Enabled", + "type": "boolean" + }, "from": { "default": null, "description": "Producer doesn't support FromSection", @@ -1713,6 +1761,12 @@ }, "description": "Helm diff config" }, + "enabled": { + "default": true, + "description": "Whether the component is enabled and should be included in the pipeline", + "title": "Enabled", + "type": "boolean" + }, "from": { "default": null, "description": "Producer doesn't support FromSection", @@ -2782,6 +2836,12 @@ }, "description": "Helm diff config" }, + "enabled": { + "default": true, + "description": "Whether the component is enabled and should be included in the pipeline", + "title": "Enabled", + "type": "boolean" + }, "from": { "anyOf": [ { @@ -2876,6 +2936,12 @@ }, "description": "Helm diff config" }, + "enabled": { + "default": true, + "description": "Whether the component is enabled and should be included in the pipeline", + "title": "Enabled", + "type": "boolean" + }, "from": { "anyOf": [ { @@ -3503,6 +3569,12 @@ }, "description": "Helm diff config" }, + "enabled": { + "default": true, + "description": "Whether the component is enabled and should be included in the pipeline", + "title": "Enabled", + "type": "boolean" + }, "from": { "anyOf": [ { @@ -3591,6 +3663,12 @@ }, "description": "Helm diff config" }, + "enabled": { + "default": true, + "description": "Whether the component is enabled and should be included in the pipeline", + "title": "Enabled", + "type": "boolean" + }, "from": { "anyOf": [ { diff --git a/docs/docs/schema/pipeline.json b/docs/docs/schema/pipeline.json index 858163dbb..77fd51ce2 100644 --- a/docs/docs/schema/pipeline.json +++ b/docs/docs/schema/pipeline.json @@ -130,6 +130,12 @@ }, "description": "Helm diff config" }, + "enabled": { + "default": true, + "description": "Whether the component is enabled and should be included in the pipeline", + "title": "Enabled", + "type": "boolean" + }, "from": { "anyOf": [ { @@ -631,6 +637,12 @@ "$ref": "#/$defs/KafkaConnectorConfig", "description": "Connector config" }, + "enabled": { + "default": true, + "description": "Whether the component is enabled and should be included in the pipeline", + "title": "Enabled", + "type": "boolean" + }, "from": { "anyOf": [ { @@ -718,6 +730,12 @@ "$ref": "#/$defs/KafkaConnectorConfig", "description": "Connector config" }, + "enabled": { + "default": true, + "description": "Whether the component is enabled and should be included in the pipeline", + "title": "Enabled", + "type": "boolean" + }, "from": { "anyOf": [ { @@ -1281,6 +1299,12 @@ }, "description": "Helm diff config" }, + "enabled": { + "default": true, + "description": "Whether the component is enabled and should be included in the pipeline", + "title": "Enabled", + "type": "boolean" + }, "from": { "default": null, "description": "Producer doesn't support FromSection", @@ -1368,6 +1392,12 @@ }, "description": "Helm diff config" }, + "enabled": { + "default": true, + "description": "Whether the component is enabled and should be included in the pipeline", + "title": "Enabled", + "type": "boolean" + }, "from": { "default": null, "description": "Producer doesn't support FromSection", @@ -2437,6 +2467,12 @@ }, "description": "Helm diff config" }, + "enabled": { + "default": true, + "description": "Whether the component is enabled and should be included in the pipeline", + "title": "Enabled", + "type": "boolean" + }, "from": { "anyOf": [ { @@ -2531,6 +2567,12 @@ }, "description": "Helm diff config" }, + "enabled": { + "default": true, + "description": "Whether the component is enabled and should be included in the pipeline", + "title": "Enabled", + "type": "boolean" + }, "from": { "anyOf": [ { diff --git a/kpops/components/base_components/pipeline_component.py b/kpops/components/base_components/pipeline_component.py index d08ab73b9..6604a2661 100644 --- a/kpops/components/base_components/pipeline_component.py +++ b/kpops/components/base_components/pipeline_component.py @@ -47,6 +47,11 @@ class PipelineComponent(BaseDefaultsComponent, ABC): """ name: str + enabled: bool = Field( + default=True, + title="Enabled", + description="Whether the component is enabled and should be included in the pipeline", + ) prefix: str = "${pipeline.name}-" from_: FromSection | None = Field( default=None, diff --git a/kpops/pipeline/__init__.py b/kpops/pipeline/__init__.py index 7fe06086a..e8f4bf642 100644 --- a/kpops/pipeline/__init__.py +++ b/kpops/pipeline/__init__.py @@ -309,6 +309,9 @@ def apply_component( """ component = component_class(**component_data) component = self.enrich_component_with_env(component) + # if component is disabled then we skip it + if not component.enabled: + return # inflate & enrich components for inflated_component in component.inflate(): # TODO: recursively if inflated_component.from_: diff --git a/tests/components/streams_bootstrap/test_streams_app.py b/tests/components/streams_bootstrap/test_streams_app.py index b45b2963e..8e92b4eb1 100644 --- a/tests/components/streams_bootstrap/test_streams_app.py +++ b/tests/components/streams_bootstrap/test_streams_app.py @@ -817,6 +817,7 @@ def test_generate(self, stateful_streams_app: StreamsApp): "helm_name_override": STREAMS_APP_HELM_NAME_OVERRIDE, "helm_release_name": STREAMS_APP_RELEASE_NAME, "name": STREAMS_APP_NAME, + "enabled": True, "namespace": NAMESPACE, "prefix": PREFIX, "to": { diff --git a/tests/pipeline/resources/pipelines-with-graphs/simple-pipeline/pipeline_production.yaml b/tests/pipeline/resources/pipelines-with-graphs/simple-pipeline/pipeline_production.yaml new file mode 100644 index 000000000..8ad46e95d --- /dev/null +++ b/tests/pipeline/resources/pipelines-with-graphs/simple-pipeline/pipeline_production.yaml @@ -0,0 +1,6 @@ +- type: producer-app + name: app1 + enabled: false + +- type: streams-app + name: app2 diff --git a/tests/pipeline/snapshots/test_example/test_generate/atm-fraud/pipeline.yaml b/tests/pipeline/snapshots/test_example/test_generate/atm-fraud/pipeline.yaml index e867b1e0d..7acef86c9 100644 --- a/tests/pipeline/snapshots/test_example/test_generate/atm-fraud/pipeline.yaml +++ b/tests/pipeline/snapshots/test_example/test_generate/atm-fraud/pipeline.yaml @@ -1,5 +1,6 @@ - type: producer-app-v2 name: account-producer + enabled: true prefix: atm-fraud- to: topics: @@ -29,6 +30,7 @@ helm_name_override: atm-fraud-account-producer - type: producer-app-v2 name: transaction-avro-producer + enabled: true prefix: atm-fraud- to: topics: @@ -61,6 +63,7 @@ helm_name_override: atm-fraud-transaction-avro-producer - type: streams-app-v2 name: transaction-joiner + enabled: true prefix: atm-fraud- to: topics: @@ -102,6 +105,7 @@ helm_name_override: atm-fraud-transaction-joiner - type: streams-app-v2 name: fraud-detector + enabled: true prefix: atm-fraud- to: topics: @@ -143,6 +147,7 @@ helm_name_override: atm-fraud-fraud-detector - type: streams-app-v2 name: account-linker + enabled: true prefix: atm-fraud- from: topics: {} @@ -194,6 +199,7 @@ helm_name_override: atm-fraud-account-linker - type: kafka-sink-connector name: postgresql-connector + enabled: true prefix: atm-fraud- config: connector.class: io.confluent.connect.jdbc.JdbcSinkConnector diff --git a/tests/pipeline/snapshots/test_example/test_generate/word-count/pipeline.yaml b/tests/pipeline/snapshots/test_example/test_generate/word-count/pipeline.yaml index b82d9ac2f..056264a29 100644 --- a/tests/pipeline/snapshots/test_example/test_generate/word-count/pipeline.yaml +++ b/tests/pipeline/snapshots/test_example/test_generate/word-count/pipeline.yaml @@ -1,5 +1,6 @@ - type: producer-app name: data-producer + enabled: true prefix: word-count- to: topics: @@ -25,6 +26,7 @@ helm_name_override: word-count-data-producer - type: streams-app name: word-count-app + enabled: true prefix: word-count- to: topics: @@ -60,6 +62,7 @@ helm_name_override: word-count-word-count-app - type: kafka-sink-connector name: redis-sink-connector + enabled: true prefix: word-count- config: connector.class: com.github.jcustenborder.kafka.connect.redis.RedisSinkConnector diff --git a/tests/pipeline/snapshots/test_generate/test_default_config/pipeline.yaml b/tests/pipeline/snapshots/test_generate/test_default_config/pipeline.yaml index c1814b3f2..74a91aeb5 100644 --- a/tests/pipeline/snapshots/test_generate/test_default_config/pipeline.yaml +++ b/tests/pipeline/snapshots/test_generate/test_default_config/pipeline.yaml @@ -1,5 +1,6 @@ - type: producer-app name: app1 + enabled: true prefix: resources-custom-config- to: topics: @@ -25,6 +26,7 @@ helm_name_override: resources-custom-config-app1 - type: streams-app name: app2 + enabled: true prefix: resources-custom-config- to: topics: diff --git a/tests/pipeline/snapshots/test_generate/test_inflate_pipeline/pipeline.yaml b/tests/pipeline/snapshots/test_generate/test_inflate_pipeline/pipeline.yaml index 53620a337..1a9f20984 100644 --- a/tests/pipeline/snapshots/test_generate/test_inflate_pipeline/pipeline.yaml +++ b/tests/pipeline/snapshots/test_generate/test_inflate_pipeline/pipeline.yaml @@ -1,5 +1,6 @@ - type: scheduled-producer name: scheduled-producer + enabled: true prefix: resources-pipeline-with-inflate- to: topics: @@ -27,6 +28,7 @@ helm_name_override: resources-pipeline-with-inflate-scheduled-producer - type: converter name: converter + enabled: true prefix: resources-pipeline-with-inflate- to: topics: @@ -73,6 +75,7 @@ helm_name_override: resources-pipeline-with-inflate-converter - type: should-inflate name: should-inflate + enabled: true prefix: resources-pipeline-with-inflate- to: topics: @@ -121,6 +124,7 @@ helm_name_override: resources-pipeline-with-inflate-should-inflate - type: kafka-sink-connector name: should-inflate-inflated-sink-connector + enabled: true prefix: resources-pipeline-with-inflate- to: topics: @@ -148,6 +152,7 @@ resetter_values: {} - type: streams-app name: should-inflate-inflated-streams-app + enabled: true prefix: resources-pipeline-with-inflate- to: topics: diff --git a/tests/pipeline/snapshots/test_generate/test_kafka_connect_sink_weave_from_topics/pipeline.yaml b/tests/pipeline/snapshots/test_generate/test_kafka_connect_sink_weave_from_topics/pipeline.yaml index 06e731199..2f055caee 100644 --- a/tests/pipeline/snapshots/test_generate/test_kafka_connect_sink_weave_from_topics/pipeline.yaml +++ b/tests/pipeline/snapshots/test_generate/test_kafka_connect_sink_weave_from_topics/pipeline.yaml @@ -1,5 +1,6 @@ - type: streams-app name: streams-app + enabled: true prefix: resources-kafka-connect-sink- from: topics: @@ -36,6 +37,7 @@ helm_name_override: resources-kafka-connect-sink-streams-app - type: kafka-sink-connector name: es-sink-connector + enabled: true prefix: resources-kafka-connect-sink- config: connector.class: io.confluent.connect.elasticsearch.ElasticsearchSinkConnector diff --git a/tests/pipeline/snapshots/test_generate/test_load_pipeline/pipeline.yaml b/tests/pipeline/snapshots/test_generate/test_load_pipeline/pipeline.yaml index 13039597d..4cf30b3ac 100644 --- a/tests/pipeline/snapshots/test_generate/test_load_pipeline/pipeline.yaml +++ b/tests/pipeline/snapshots/test_generate/test_load_pipeline/pipeline.yaml @@ -1,5 +1,6 @@ - type: scheduled-producer name: scheduled-producer + enabled: true prefix: resources-first-pipeline- to: topics: @@ -27,6 +28,7 @@ helm_name_override: resources-first-pipeline-scheduled-producer - type: converter name: converter + enabled: true prefix: resources-first-pipeline- to: topics: @@ -73,6 +75,7 @@ helm_name_override: resources-first-pipeline-converter - type: filter name: a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name + enabled: true prefix: resources-first-pipeline- to: topics: diff --git a/tests/pipeline/snapshots/test_generate/test_load_pipeline_with_folder_path/pipeline.yaml b/tests/pipeline/snapshots/test_generate/test_load_pipeline_with_folder_path/pipeline.yaml index d04d9c094..a3f353813 100644 --- a/tests/pipeline/snapshots/test_generate/test_load_pipeline_with_folder_path/pipeline.yaml +++ b/tests/pipeline/snapshots/test_generate/test_load_pipeline_with_folder_path/pipeline.yaml @@ -1,5 +1,6 @@ - type: scheduled-producer name: scheduled-producer + enabled: true prefix: resources-pipeline-folders-pipeline-1- to: topics: @@ -28,6 +29,7 @@ - type: converter name: converter + enabled: true prefix: resources-pipeline-folders-pipeline-2- to: topics: @@ -73,6 +75,7 @@ - type: filter name: a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name + enabled: true prefix: resources-pipeline-folders-pipeline-3- to: topics: diff --git a/tests/pipeline/snapshots/test_generate/test_load_pipeline_with_multiple_pipeline_paths/pipeline.yaml b/tests/pipeline/snapshots/test_generate/test_load_pipeline_with_multiple_pipeline_paths/pipeline.yaml index d04d9c094..a3f353813 100644 --- a/tests/pipeline/snapshots/test_generate/test_load_pipeline_with_multiple_pipeline_paths/pipeline.yaml +++ b/tests/pipeline/snapshots/test_generate/test_load_pipeline_with_multiple_pipeline_paths/pipeline.yaml @@ -1,5 +1,6 @@ - type: scheduled-producer name: scheduled-producer + enabled: true prefix: resources-pipeline-folders-pipeline-1- to: topics: @@ -28,6 +29,7 @@ - type: converter name: converter + enabled: true prefix: resources-pipeline-folders-pipeline-2- to: topics: @@ -73,6 +75,7 @@ - type: filter name: a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name + enabled: true prefix: resources-pipeline-folders-pipeline-3- to: topics: diff --git a/tests/pipeline/snapshots/test_generate/test_model_serialization/pipeline.yaml b/tests/pipeline/snapshots/test_generate/test_model_serialization/pipeline.yaml index e3b6da8ae..e95e26dac 100644 --- a/tests/pipeline/snapshots/test_generate/test_model_serialization/pipeline.yaml +++ b/tests/pipeline/snapshots/test_generate/test_model_serialization/pipeline.yaml @@ -1,5 +1,6 @@ - type: producer-app name: account-producer + enabled: true prefix: resources-pipeline-with-paths- namespace: test values: diff --git a/tests/pipeline/snapshots/test_generate/test_no_input_topic/pipeline.yaml b/tests/pipeline/snapshots/test_generate/test_no_input_topic/pipeline.yaml index 7e49d3695..c3fc46f86 100644 --- a/tests/pipeline/snapshots/test_generate/test_no_input_topic/pipeline.yaml +++ b/tests/pipeline/snapshots/test_generate/test_no_input_topic/pipeline.yaml @@ -1,5 +1,6 @@ - type: streams-app name: app1 + enabled: true prefix: resources-no-input-topic-pipeline- from: topics: @@ -42,6 +43,7 @@ helm_name_override: resources-no-input-topic-pipeline-app1 - type: streams-app name: app2 + enabled: true prefix: resources-no-input-topic-pipeline- to: topics: diff --git a/tests/pipeline/snapshots/test_generate/test_no_user_defined_components/pipeline.yaml b/tests/pipeline/snapshots/test_generate/test_no_user_defined_components/pipeline.yaml index 997416bd7..1fd3fc56e 100644 --- a/tests/pipeline/snapshots/test_generate/test_no_user_defined_components/pipeline.yaml +++ b/tests/pipeline/snapshots/test_generate/test_no_user_defined_components/pipeline.yaml @@ -1,5 +1,6 @@ - type: streams-app name: streams-app + enabled: true prefix: resources-no-user-defined-components- from: topics: diff --git a/tests/pipeline/snapshots/test_generate/test_pipelines_with_envs/pipeline.yaml b/tests/pipeline/snapshots/test_generate/test_pipelines_with_envs/pipeline.yaml index 12735972e..c7958b8c8 100644 --- a/tests/pipeline/snapshots/test_generate/test_pipelines_with_envs/pipeline.yaml +++ b/tests/pipeline/snapshots/test_generate/test_pipelines_with_envs/pipeline.yaml @@ -1,5 +1,6 @@ - type: scheduled-producer name: input-producer + enabled: true prefix: resources-pipeline-with-envs- to: topics: @@ -27,6 +28,7 @@ helm_name_override: resources-pipeline-with-envs-input-producer - type: converter name: converter + enabled: true prefix: resources-pipeline-with-envs- to: topics: @@ -73,6 +75,7 @@ helm_name_override: resources-pipeline-with-envs-converter - type: filter name: filter + enabled: true prefix: resources-pipeline-with-envs- to: topics: diff --git a/tests/pipeline/snapshots/test_generate/test_prefix_pipeline_component/pipeline.yaml b/tests/pipeline/snapshots/test_generate/test_prefix_pipeline_component/pipeline.yaml index 88bdd4569..88c9327b8 100644 --- a/tests/pipeline/snapshots/test_generate/test_prefix_pipeline_component/pipeline.yaml +++ b/tests/pipeline/snapshots/test_generate/test_prefix_pipeline_component/pipeline.yaml @@ -1,5 +1,6 @@ - type: producer-app name: account-producer + enabled: true prefix: from-pipeline-component- namespace: ${NAMESPACE} values: diff --git a/tests/pipeline/snapshots/test_generate/test_read_from_component/pipeline.yaml b/tests/pipeline/snapshots/test_generate/test_read_from_component/pipeline.yaml index e71bfa809..71474a107 100644 --- a/tests/pipeline/snapshots/test_generate/test_read_from_component/pipeline.yaml +++ b/tests/pipeline/snapshots/test_generate/test_read_from_component/pipeline.yaml @@ -1,5 +1,6 @@ - type: producer-app name: producer1 + enabled: true prefix: resources-read-from-component- to: topics: @@ -19,6 +20,7 @@ helm_name_override: resources-read-from-component-producer1 - type: producer-app name: producer2 + enabled: true prefix: '' to: topics: @@ -38,6 +40,7 @@ helm_name_override: producer2 - type: should-inflate name: inflate-step + enabled: true prefix: resources-read-from-component- to: topics: @@ -79,6 +82,7 @@ helm_name_override: resources-read-from-component-inflate-step - type: kafka-sink-connector name: inflate-step-inflated-sink-connector + enabled: true prefix: resources-read-from-component- to: topics: @@ -106,6 +110,7 @@ resetter_values: {} - type: streams-app name: inflate-step-inflated-streams-app + enabled: true prefix: resources-read-from-component- to: topics: @@ -137,6 +142,7 @@ helm_name_override: resources-read-from-component-inflate-step-inflated-streams-app - type: should-inflate name: inflate-step-without-prefix + enabled: true prefix: '' to: topics: @@ -178,6 +184,7 @@ helm_name_override: inflate-step-without-prefix - type: kafka-sink-connector name: inflate-step-without-prefix-inflated-sink-connector + enabled: true prefix: resources-read-from-component- to: topics: @@ -205,6 +212,7 @@ resetter_values: {} - type: streams-app name: inflate-step-without-prefix-inflated-streams-app + enabled: true prefix: resources-read-from-component- to: topics: @@ -236,6 +244,7 @@ helm_name_override: resources-read-from-component-inflate-step-without-prefix-000dc - type: streams-app name: consumer1 + enabled: true prefix: resources-read-from-component- from: topics: {} @@ -272,6 +281,7 @@ helm_name_override: resources-read-from-component-consumer1 - type: streams-app name: consumer2 + enabled: true prefix: resources-read-from-component- from: topics: {} @@ -307,6 +317,7 @@ helm_name_override: resources-read-from-component-consumer2 - type: streams-app name: consumer3 + enabled: true prefix: resources-read-from-component- from: topics: @@ -342,6 +353,7 @@ helm_name_override: resources-read-from-component-consumer3 - type: streams-app name: consumer4 + enabled: true prefix: resources-read-from-component- from: topics: {} @@ -374,6 +386,7 @@ helm_name_override: resources-read-from-component-consumer4 - type: streams-app name: consumer5 + enabled: true prefix: resources-read-from-component- from: topics: {} diff --git a/tests/pipeline/snapshots/test_generate/test_streams_bootstrap/pipeline.yaml b/tests/pipeline/snapshots/test_generate/test_streams_bootstrap/pipeline.yaml index 2511c5760..e26b91f1a 100644 --- a/tests/pipeline/snapshots/test_generate/test_streams_bootstrap/pipeline.yaml +++ b/tests/pipeline/snapshots/test_generate/test_streams_bootstrap/pipeline.yaml @@ -1,5 +1,6 @@ - type: my-producer-app name: my-producer-app + enabled: true prefix: resources-streams-bootstrap- to: topics: @@ -49,6 +50,7 @@ helm_name_override: resources-streams-bootstrap-my-producer-app - type: my-streams-app name: my-streams-app + enabled: true prefix: resources-streams-bootstrap- from: topics: diff --git a/tests/pipeline/snapshots/test_generate/test_substitute_in_component/pipeline.yaml b/tests/pipeline/snapshots/test_generate/test_substitute_in_component/pipeline.yaml index 1d56d1682..41343b612 100644 --- a/tests/pipeline/snapshots/test_generate/test_substitute_in_component/pipeline.yaml +++ b/tests/pipeline/snapshots/test_generate/test_substitute_in_component/pipeline.yaml @@ -1,5 +1,6 @@ - type: scheduled-producer name: scheduled-producer + enabled: true prefix: resources-component-type-substitution- to: topics: @@ -31,6 +32,7 @@ helm_name_override: resources-component-type-substitution-schedule-b0010 - type: converter name: converter + enabled: true prefix: resources-component-type-substitution- to: topics: @@ -77,6 +79,7 @@ helm_name_override: resources-component-type-substitution-converter - type: filter name: filter-app + enabled: true prefix: resources-component-type-substitution- to: topics: diff --git a/tests/pipeline/snapshots/test_generate/test_with_custom_config_with_absolute_defaults_path/pipeline.yaml b/tests/pipeline/snapshots/test_generate/test_with_custom_config_with_absolute_defaults_path/pipeline.yaml index 10a84b7f7..38af3c628 100644 --- a/tests/pipeline/snapshots/test_generate/test_with_custom_config_with_absolute_defaults_path/pipeline.yaml +++ b/tests/pipeline/snapshots/test_generate/test_with_custom_config_with_absolute_defaults_path/pipeline.yaml @@ -1,5 +1,6 @@ - type: producer-app name: app1 + enabled: true prefix: resources-custom-config- to: topics: @@ -25,6 +26,7 @@ helm_name_override: resources-custom-config-app1 - type: streams-app name: app2 + enabled: true prefix: resources-custom-config- to: topics: diff --git a/tests/pipeline/snapshots/test_generate/test_with_custom_config_with_relative_defaults_path/pipeline.yaml b/tests/pipeline/snapshots/test_generate/test_with_custom_config_with_relative_defaults_path/pipeline.yaml index 10a84b7f7..38af3c628 100644 --- a/tests/pipeline/snapshots/test_generate/test_with_custom_config_with_relative_defaults_path/pipeline.yaml +++ b/tests/pipeline/snapshots/test_generate/test_with_custom_config_with_relative_defaults_path/pipeline.yaml @@ -1,5 +1,6 @@ - type: producer-app name: app1 + enabled: true prefix: resources-custom-config- to: topics: @@ -25,6 +26,7 @@ helm_name_override: resources-custom-config-app1 - type: streams-app name: app2 + enabled: true prefix: resources-custom-config- to: topics: diff --git a/tests/pipeline/snapshots/test_generate/test_with_env_defaults/pipeline.yaml b/tests/pipeline/snapshots/test_generate/test_with_env_defaults/pipeline.yaml index a88c8e28e..cf59070dd 100644 --- a/tests/pipeline/snapshots/test_generate/test_with_env_defaults/pipeline.yaml +++ b/tests/pipeline/snapshots/test_generate/test_with_env_defaults/pipeline.yaml @@ -1,5 +1,6 @@ - type: streams-app name: streams-app-development + enabled: true prefix: resources-pipeline-with-env-defaults- from: topics: @@ -36,6 +37,7 @@ helm_name_override: resources-pipeline-with-env-defaults-streams-app-development - type: kafka-sink-connector name: es-sink-connector + enabled: true prefix: resources-pipeline-with-env-defaults- config: connector.class: io.confluent.connect.elasticsearch.ElasticsearchSinkConnector diff --git a/tests/pipeline/test_generate.py b/tests/pipeline/test_generate.py index 9ec6d90bd..57d3ddf7a 100644 --- a/tests/pipeline/test_generate.py +++ b/tests/pipeline/test_generate.py @@ -700,6 +700,48 @@ def test_validate_simple_graph(self): assert len(topic_nodes) == 1 assert len(pipeline.components) == len(pipeline._graph.nodes) - len(topic_nodes) + def test_validate_components_are_disabled_in_production_but_enabled_on_development( + self, + ): + pipeline_production = kpops.generate( + pipeline_path=RESOURCE_PATH + / "pipelines-with-graphs" + / "simple-pipeline" + / PIPELINE_YAML, + environment="production", + ) + pipeline_development = kpops.generate( + pipeline_path=RESOURCE_PATH + / "pipelines-with-graphs" + / "simple-pipeline" + / PIPELINE_YAML, + environment="development", + ) + + assert len(pipeline_production.components) == 1 + assert len(pipeline_production._graph.edges) == 0 + topic_nodes_production = [ + node + for node in pipeline_production._graph.nodes + if node.startswith("topic-") + ] + assert len(topic_nodes_production) == 0 + assert len(pipeline_production.components) == len( + pipeline_production._graph.nodes + ) - len(topic_nodes_production) + + assert len(pipeline_development._graph.nodes) == 3 + assert len(pipeline_development._graph.edges) == 2 + topic_nodes_development = [ + node + for node in pipeline_development._graph.nodes + if node.startswith("topic-") + ] + assert len(topic_nodes_development) == 1 + assert len(pipeline_development.components) == len( + pipeline_development._graph.nodes + ) - len(topic_nodes_development) + def test_validate_topic_and_component_same_name(self): pipeline = kpops.generate( RESOURCE_PATH