diff --git a/api/v4/ingestorcluster_types.go b/api/v4/ingestorcluster_types.go index 4e206e76a..a3e2d66c7 100644 --- a/api/v4/ingestorcluster_types.go +++ b/api/v4/ingestorcluster_types.go @@ -76,11 +76,6 @@ type IngestorClusterStatus struct { // Auxillary message describing CR status Message string `json:"message"` - // Credential secret version to track changes to the secret and trigger rolling restart of indexer cluster peers when the secret is updated - CredentialSecretVersion string `json:"credentialSecretVersion,omitempty"` - - // Service account to track changes to the service account and trigger rolling restart of indexer cluster peers when the service account is updated - ServiceAccount string `json:"serviceAccount,omitempty"` } // +kubebuilder:object:root=true diff --git a/config/crd/bases/enterprise.splunk.com_ingestorclusters.yaml b/config/crd/bases/enterprise.splunk.com_ingestorclusters.yaml index 380109fb8..b27799d39 100644 --- a/config/crd/bases/enterprise.splunk.com_ingestorclusters.yaml +++ b/config/crd/bases/enterprise.splunk.com_ingestorclusters.yaml @@ -4612,11 +4612,6 @@ spec: description: App Framework version info for future use type: integer type: object - credentialSecretVersion: - description: Credential secret version to track changes to the secret - and trigger rolling restart of indexer cluster peers when the secret - is updated - type: string message: description: Auxillary message describing CR status type: string @@ -4647,11 +4642,6 @@ spec: selector: description: Selector for pods used by HorizontalPodAutoscaler type: string - serviceAccount: - description: Service account to track changes to the service account - and trigger rolling restart of indexer cluster peers when the service - account is updated - type: string telAppInstalled: description: Telemetry App installation flag type: boolean diff --git a/docs/IndexIngestionSeparation.md b/docs/IndexIngestionSeparation.md index 1a06606fe..522b1918d 100644 --- a/docs/IndexIngestionSeparation.md +++ b/docs/IndexIngestionSeparation.md @@ -4,1156 +4,546 @@ parent: Deploy & Configure nav_order: 6 --- -# Background +# Index and Ingestion Separation -Separation between ingestion and indexing services within Splunk Operator for Kubernetes enables the operator to independently manage the ingestion service while maintaining seamless integration with the indexing service. +## What Are Ingestors? -This separation enables: -- Independent scaling: Match resource allocation to ingestion or indexing workload. -- Data durability: Off‑load buffer management and retry logic to a durable message queue. -- Operational clarity: Separate monitoring dashboards for ingestion throughput vs indexing latency. +In a traditional Splunk deployment, every indexer both receives data from forwarders and writes that data to disk. This tight coupling means you cannot independently scale the pods that handle incoming traffic from the pods that perform the compute-intensive work of writing and searching buckets. -## Splunk Support +**Ingestors** are a dedicated tier of Splunk pods whose only job is to accept data from forwarders and publish it to a durable message queue (Amazon SQS). They hold no index data. A separate **Indexer** tier pulls messages off that queue and writes them to buckets. The two tiers are decoupled by the queue and can be scaled, upgraded, and monitored completely independently. -These features are supported for Splunk 10.2 and above versions. +![Index and Ingestion Separation — SOK Architecture](images/index_ingestion_separation.png) -# Important Note +### Why use Ingestors on Kubernetes? -> [!WARNING] -> **For customers deploying SmartBus on CMP, the Splunk Operator for Kubernetes (SOK) manages the configuration and lifecycle of the ingestor tier. The following SOK guide provides implementation details for setting up ingestion separation and integrating with existing indexers. This reference is primarily intended for CMP users leveraging SOK-managed ingestors.** +| Benefit | Detail | +|---------|--------| +| **Independent scaling** | Add ingestor pods during ingestion spikes without touching indexers, and vice-versa. | +| **Data durability** | SQS provides at-least-once delivery and a dead-letter queue. Data is never dropped if indexers are down. | +| **Zero-restart first-boot** | SOK delivers queue configuration to each ingestor pod via a Kubernetes ConfigMap and an init container. Splunk starts with the right `outputs.conf` already in place — no in-place REST restart is needed. | +| **Credential-free IRSA** | When running on EKS with IRSA, no static AWS credentials are stored. The pod identity is used directly. | +| **Operational clarity** | Separate dashboards for ingestion throughput (ingestor pods) vs. indexing latency (indexer pods). | -# Document Variables +> **Splunk version requirement:** Splunk Enterprise 10.2 or later. -- SPLUNK_IMAGE_VERSION: Splunk Enterprise Docker Image version +> [!WARNING] +> **For customers deploying SmartBus on CMP, SOK manages the configuration and lifecycle of the ingestor tier. This guide is primarily intended for CMP users leveraging SOK-managed ingestors.** -# Queue +--- -Queue is introduced to store message queue information to be shared among IngestorCluster and IndexerCluster. +## Custom Resources -## Spec +Index and Ingestion Separation introduces three new Custom Resources and enhances one existing resource. All are namespaced. -Queue inputs can be found in the table below. As of now, only SQS provider of message queue is supported. +| Resource | Short name | Purpose | +|----------|-----------|---------| +| `Queue` | `queue` | Describes the SQS queue and dead-letter queue | +| `ObjectStorage` | `os` | Describes the S3 bucket used for oversized messages | +| `IngestorCluster` | `ing` | Manages the ingestor StatefulSet (receives data, publishes to queue) | +| `IndexerCluster` | `idc` / `idxc` | Enhanced to pull from the queue and write to indexes | -| Key | Type | Description | -| ---------- | ------- | ------------------------------------------------- | -| provider | string | [Required] Provider of message queue (Allowed values: sqs, sqs_cp) | -| sqs | SQS | [Required if provider=sqs or provider=sqs_cp] SQS message queue inputs | +All four resources live in the `enterprise.splunk.com/v4` API group. -SQS message queue inputs can be found in the table below. +### Queue -| Key | Type | Description | -| ---------- | ------- | ------------------------------------------------- | -| name | string | [Required] Name of the queue | -| authRegion | string | [Required] Region where the queue is located | -| endpoint | string | [Optional, if not provided formed based on authRegion] AWS SQS Service endpoint -| dlq | string | [Required] Name of the dead letter queue | -| volumes | []VolumeSpec | [Optional] List of remote storage volumes used to mount the credentials for queue and bucket access (must contain s3_access_key and s3_secret_key) | +Describes the message queue where ingestors publish events. -**SOK doesn't support update of any of the Queue inputs except from the volumes which allow the change of secrets.** +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `provider` | string | Yes | Queue technology. Allowed: `sqs`, `sqs_cp` | +| `sqs.name` | string | Yes | SQS queue name | +| `sqs.authRegion` | string | Yes | AWS region of the queue | +| `sqs.dlq` | string | Yes | Dead-letter queue name | +| `sqs.endpoint` | string | No | Override SQS endpoint (defaults to regional endpoint) | +| `sqs.volumes` | []VolumeSpec | No | Kubernetes Secrets that provide static `s3_access_key` / `s3_secret_key` credentials. Omit when using IRSA. | -## Example -``` +> **Immutable fields:** `provider`, `sqs.name`, `sqs.authRegion`, `sqs.dlq`, `sqs.endpoint`. Only `sqs.volumes` (credentials) can be updated after creation. + +```yaml apiVersion: enterprise.splunk.com/v4 kind: Queue metadata: name: queue + namespace: splunk spec: provider: sqs sqs: - name: sqs-test + name: sqs-smartbus authRegion: us-west-2 - endpoint: https://sqs.us-west-2.amazonaws.com - dlq: sqs-dlq-test + dlq: sqs-smartbus-dlq + # volumes only needed when NOT using IRSA: volumes: - - name: s3-sqs-volume + - name: s3-sqs-creds secretRef: s3-secret ``` -# ObjectStorage - -ObjectStorage is introduced to store large messages (messages that exceed the size of messages that can be stored in SQS) to be shared among IngestorCluster and IndexerCluster. - -## Spec - -ObjectStorage inputs can be found in the table below. As of now, only S3 provider of object storage is supported. - -| Key | Type | Description | -| ---------- | ------- | ------------------------------------------------- | -| provider | string | [Required] Provider of object storage (Allowed values: s3) | -| s3 | S3 | [Required if provider=s3] S3 object storage inputs | +### ObjectStorage -S3 object storage inputs can be found in the table below. +Describes the S3 bucket used to relay messages that exceed SQS size limits. -| Key | Type | Description | -| ---------- | ------- | ------------------------------------------------- | -| path | string | [Required] Remote storage location for messages that are larger than the underlying maximum message size | -| endpoint | string | [Optional, if not provided formed based on authRegion] S3-compatible service endpoint +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `provider` | string | Yes | Object storage technology. Allowed: `s3` | +| `s3.path` | string | Yes | S3 bucket path (e.g. `s3://my-bucket/prefix` or `my-bucket/prefix`) | +| `s3.endpoint` | string | No | Override S3 endpoint (defaults to regional endpoint) | -**SOK doesn't support update of any of the ObjectStorage inputs.** +> **Immutable fields:** All ObjectStorage fields are immutable after creation. -## Example -``` +```yaml apiVersion: enterprise.splunk.com/v4 kind: ObjectStorage metadata: name: os + namespace: splunk spec: provider: s3 s3: - path: ingestion/smartbus-test + path: s3://my-smartbus-bucket/ingestion endpoint: https://s3.us-west-2.amazonaws.com ``` -# IngestorCluster - -IngestorCluster is introduced for high‑throughput data ingestion into a durable message queue. Its Splunk pods are configured to receive events (outputs.conf) and publish them to a message queue. - -## Spec - -In addition to common spec inputs, the IngestorCluster resource provides the following Spec configuration parameters. +### IngestorCluster -| Key | Type | Description | -| ---------- | ------- | ------------------------------------------------- | -| replicas | integer | The number of replicas (defaults to 3) | -| queueRef | corev1.ObjectReference | Message queue reference | -| objectStorageRef | corev1.ObjectReference | Object storage reference | +Manages the Splunk ingestor StatefulSet. Each pod receives data from forwarders and publishes it to the queue. -**SOK doesn't support update of queueRef and objectStorageRef.** +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `replicas` | integer | Yes | Number of ingestor pods (min 1, default 1) | +| `queueRef.name` | string | Yes | Name of the `Queue` CR in the same namespace | +| `objectStorageRef.name` | string | Yes | Name of the `ObjectStorage` CR in the same namespace | +| `serviceAccount` | string | No | Kubernetes ServiceAccount with SQS + S3 IAM permissions (required for IRSA) | +| `image` | string | No | Splunk Enterprise container image | +| `appRepo` | AppFrameworkSpec | No | App Framework configuration for deploying Splunk apps to ingestors | -**First provisioning or scaling up the number of replicas requires Ingestor Cluster Splunkd restart, but this restart is implemented automatically and done by SOK.** +> **Immutable fields:** `queueRef`, `objectStorageRef`. These cannot be changed after the IngestorCluster is created. -## Example +**How SOK configures ingestor pods:** +SOK builds a Kubernetes ConfigMap named `splunk--ingestor-queue-config` containing the Splunk app `100-sok-ingestorcluster` with `outputs.conf`, `default-mode.conf`, `app.conf`, and `local.meta`. An init container (`init-ingestor-queue-config`) runs before Splunk starts and symlinks these files from the ConfigMap mount into the Splunk app directory. Splunk boots with the correct queue configuration already in place — no restart is required. -The example presented below configures IngestorCluster named ingestor with Splunk ${SPLUNK_IMAGE_VERSION} image that resides in a default namespace and is scaled to 3 replicas that serve the ingestion traffic. This IngestorCluster custom resource is set up with the s3-secret credentials allowing it to perform SQS and S3 operations. Queue and ObjectStorage references allow the user to specify queue and bucket settings for the ingestion process. - -In this case, the setup uses the SQS and S3 based configuration where the messages are stored in sqs-test queue in us-west-2 region with dead letter queue set to sqs-dlq-test queue. The object storage is set to ingestion bucket in smartbus-test directory. Based on these inputs, default-mode.conf and outputs.conf files are configured accordingly. - -``` +```yaml apiVersion: enterprise.splunk.com/v4 kind: IngestorCluster metadata: name: ingestor + namespace: splunk finalizers: - enterprise.splunk.com/delete-pvc spec: - serviceAccount: ingestor-sa replicas: 3 image: splunk/splunk:${SPLUNK_IMAGE_VERSION} + serviceAccount: ingestor-sa # omit if using static credentials via Queue volumes queueRef: name: queue objectStorageRef: name: os ``` -# IndexerCluster - -IndexerCluster is enhanced to support index‑only mode enabling independent scaling, loss‑safe buffering, and simplified day‑0/day‑n management via Kubernetes CRDs. Its Splunk pods are configured to pull events from the queue (inputs.conf) and index them. +### IndexerCluster -## Spec +An existing SOK resource, enhanced to pull events from the queue. When `queueRef` and `objectStorageRef` are set, the indexer pods receive their `inputs.conf`, `outputs.conf`, and `default-mode.conf` configuration to drain the queue and write events to indexes. -In addition to common spec inputs, the IndexerCluster resource provides the following Spec configuration parameters. +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `replicas` | integer | Yes | Number of indexer pods | +| `clusterManagerRef.name` | string | Yes | Name of the `ClusterManager` CR | +| `queueRef.name` | string | No | Name of the `Queue` CR (must set both or neither) | +| `objectStorageRef.name` | string | No | Name of the `ObjectStorage` CR (must set both or neither) | -| Key | Type | Description | -| ---------- | ------- | ------------------------------------------------- | -| replicas | integer | The number of replicas (defaults to 3) | -| queueRef | corev1.ObjectReference | Message queue reference | -| objectStorageRef | corev1.ObjectReference | Object storage reference | +> **Immutable fields:** `queueRef`, `objectStorageRef` are immutable once set. -**SOK doesn't support update of queueRef and objectStorageRef.** - -**First provisioning or scaling up the number of replicas requires Indexer Cluster Splunkd restart, but this restart is implemented automatically and done by SOK.** - -## Example - -The example presented below configures IndexerCluster named indexer with Splunk ${SPLUNK_IMAGE_VERSION} image that resides in a default namespace and is scaled to 3 replicas that serve the indexing traffic. This IndexerCluster custom resource is set up with the s3-secret credentials allowing it to perform SQS and S3 operations. Queue and ObjectStorage references allow the user to specify queue and bucket settings for the indexing process. - -In this case, the setup uses the SQS and S3 based configuration where the messages are stored in and retrieved from sqs-test queue in us-west-2 region with dead letter queue set to sqs-dlq-test queue. The object storage is set to ingestion bucket in smartbus-test directory. Based on these inputs, default-mode.conf, inputs.conf and outputs.conf files are configured accordingly. - -``` +```yaml apiVersion: enterprise.splunk.com/v4 kind: ClusterManager metadata: name: cm + namespace: splunk finalizers: - enterprise.splunk.com/delete-pvc spec: - serviceAccount: ingestor-sa image: splunk/splunk:${SPLUNK_IMAGE_VERSION} + serviceAccount: ingestor-sa --- apiVersion: enterprise.splunk.com/v4 kind: IndexerCluster metadata: name: indexer + namespace: splunk finalizers: - enterprise.splunk.com/delete-pvc spec: clusterManagerRef: name: cm - serviceAccount: ingestor-sa - replicas: 3 + replicas: 3 image: splunk/splunk:${SPLUNK_IMAGE_VERSION} + serviceAccount: ingestor-sa queueRef: name: queue objectStorageRef: name: os ``` -# Common Spec - -Common spec values for all SOK Custom Resources can be found in [CustomResources doc](CustomResources.md). - -# Helm Charts - -Queue, ObjectStorage and IngestorCluster have been added to the splunk/splunk-enterprise Helm chart. IndexerCluster has also been enhanced to support new inputs. - -## Example - -Below examples describe how to define values for Queue, ObjectStorage, IngestorCluster and IndexerCluster similarly to the above yaml files specifications. - -``` -queue: - enabled: true - name: queue - provider: sqs - sqs: - name: sqs-test - authRegion: us-west-2 - endpoint: https://sqs.us-west-2.amazonaws.com - dlq: sqs-dlq-test - volumes: - - name: s3-sqs-volume - secretRef: s3-secret -``` - -``` -objectStorage: - enabled: true - name: os - provider: s3 - s3: - endpoint: https://s3.us-west-2.amazonaws.com - path: ingestion/smartbus-test -``` +--- -``` -ingestorCluster: - enabled: true - name: ingestor - replicaCount: 3 - serviceAccount: ingestor-sa - queueRef: - name: queue - objectStorageRef: - name: os -``` +## Critical User Journey -``` -clusterManager: - enabled: true - name: cm - replicaCount: 1 - serviceAccount: ingestor-sa +This section walks through the complete lifecycle from zero to a running index-ingestion-separated deployment, from the user's point of view. -indexerCluster: - enabled: true - name: indexer - replicaCount: 3 - serviceAccount: ingestor-sa - clusterManagerRef: - name: cm - queueRef: - name: queue - objectStorageRef: - name: os -``` +### Prerequisites -# Service Account +- SOK installed and running in your cluster (see [Install](Install.md)) +- An EKS cluster (or equivalent) with IAM IRSA configured, **or** static AWS credentials available in a Kubernetes Secret +- An Amazon SQS queue, dead-letter queue, and S3 bucket provisioned in AWS -To be able to configure ingestion and indexing resources correctly in a secure manner, it is required to provide these resources with the service account that is configured with a minimum set of permissions to complete required operations. With this provided, the right credentials are used by Splunk to peform its tasks. +### Step 1 — Set Up IAM Permissions -## Example +Ingestors and indexers need permission to put and receive SQS messages and to read/write the S3 bucket for oversized messages. -The example presented below configures the ingestor-sa service account by using eksctl utility. It sets up the service account for cluster-name cluster in region us-west-2 with AmazonS3FullAccess and AmazonSQSFullAccess access policies. +**Option A — IRSA (recommended for EKS):** Create a Kubernetes ServiceAccount annotated with an IAM role ARN. The role needs `AmazonSQSFullAccess` and `AmazonS3FullAccess` (or equivalent least-privilege policies). -``` -eksctl create iamserviceaccount \ +```bash +eksctl create iamserviceaccount \ --name ingestor-sa \ - --cluster ind-ing-sep-demo \ + --namespace splunk \ + --cluster my-cluster \ --region us-west-2 \ - --attach-policy-arn arn:aws:iam::aws:policy/AmazonS3FullAccess \ --attach-policy-arn arn:aws:iam::aws:policy/AmazonSQSFullAccess \ + --attach-policy-arn arn:aws:iam::aws:policy/AmazonS3FullAccess \ --approve \ --override-existing-serviceaccounts ``` -``` -$ kubectl describe sa ingestor-sa -Name: ingestor-sa -Namespace: default -Labels: app.kubernetes.io/managed-by=eksctl -Annotations: eks.amazonaws.com/role-arn: arn:aws:iam::111111111111:role/eksctl-ind-ing-sep-demo-addon-iamserviceac-Role1-123456789123 -Image pull secrets: -Mountable secrets: -Tokens: -Events: -``` - -``` -$ aws iam get-role --role-name eksctl-ind-ing-sep-demo-addon-iamserviceac-Role1-123456789123 -{ - "Role": { - "Path": "/", - "RoleName": "eksctl-ind-ing-sep-demo-addon-iamserviceac-Role1-123456789123", - "RoleId": "123456789012345678901", - "Arn": "arn:aws:iam::111111111111:role/eksctl-ind-ing-sep-demo-addon-iamserviceac-Role1-123456789123", - "CreateDate": "2025-08-07T12:03:31+00:00", - "AssumeRolePolicyDocument": { - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Principal": { - "Federated": "arn:aws:iam::111111111111:oidc-provider/oidc.eks.us-west-2.amazonaws.com/id/1234567890123456789012345678901" - }, - "Action": "sts:AssumeRoleWithWebIdentity", - "Condition": { - "StringEquals": { - "oidc.eks.us-west-2.amazonaws.com/id/1234567890123456789012345678901:aud": "sts.amazonaws.com", - "oidc.eks.us-west-2.amazonaws.com/id/1234567890123456789012345678901:sub": "system:serviceaccount:default:ingestor-sa" - } - } - } - ] - }, - "Description": "", - "MaxSessionDuration": 3600, - "Tags": [ - { - "Key": "alpha.eksctl.io/cluster-name", - "Value": "ind-ing-sep-demo" - }, - { - "Key": "alpha.eksctl.io/iamserviceaccount-name", - "Value": "default/ingestor-sa" - }, - { - "Key": "alpha.eksctl.io/eksctl-version", - "Value": "0.211.0" - }, - { - "Key": "eksctl.cluster.k8s.io/v1alpha1/cluster-name", - "Value": "ind-ing-sep-demo" - } - ], - "RoleLastUsed": { - "LastUsedDate": "2025-08-18T08:47:27+00:00", - "Region": "us-west-2" - } - } -} -``` - -``` -$ aws iam list-attached-role-policies --role-name eksctl-cluster-name-addon-iamserviceac-Role1-123456789123 -{ - "AttachedPolicies": [ - { - "PolicyName": "AmazonSQSFullAccess", - "PolicyArn": "arn:aws:iam::aws:policy/AmazonSQSFullAccess" - }, - { - "PolicyName": "AmazonS3FullAccess", - "PolicyArn": "arn:aws:iam::aws:policy/AmazonS3FullAccess" - } - ] -} -``` - -## Documentation References - -- [IAM Roles for Service Accounts on eksctl Docs](https://eksctl.io/usage/iamserviceaccounts/) - -# Horizontal Pod Autoscaler - -To automatically adjust the number of replicas to serve the ingestion traffic effectively, it is recommended to use Horizontal Pod Autoscaler which scales the workload based on the actual demand. It enables the user to provide the metrics which are used to make decisions on removing unwanted replicas if there is not too much traffic or setting up the new ones if the traffic is too big to be handled by currently running resources. - -## Example - -The exmaple presented below configures HorizontalPodAutoscaler named ingestor-hpa that resides in a default namespace (same namespace as resources it is managing) to scale IngestorCluster custom resource named ingestor. With average utilization set to 50, the HorizontalPodAutoscaler resource will try to keep the average utilization of the pods in the scaling target at 50%. It will be able to scale the replicas starting from the minimum number of 3 with the maximum number of 10 replicas. - -``` -apiVersion: autoscaling/v2 -kind: HorizontalPodAutoscaler -metadata: - name: ingestor-hpa -spec: - scaleTargetRef: - apiVersion: enterprise.splunk.com/v4 - kind: IngestorCluster - name: ingestor - minReplicas: 3 - maxReplicas: 10 - metrics: - - type: Resource - resource: - name: cpu - target: - type: Utilization - averageUtilization: 50 -``` - -## Documentation References - -- [Horizontal Pod Autoscaling on Kubernetes Docs](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/) - -# Grafana - -In order to monitor the resources, Grafana could be installed and configured on the cluster to present the setup on a dashabord in a series of useful diagrams and metrics. - -## Example - -In the following example, the dashboard presents ingestion and indexing data in the form of useful diagrams and metrics such as number of replicas or resource consumption. - -``` -{ - "id": null, - "uid": "splunk-autoscale", - "title": "Splunk Ingestion & Indexer Autoscaling with I/O & PV", - "schemaVersion": 27, - "version": 12, - "refresh": "5s", - "time": { "from": "now-30m", "to": "now" }, - "timezone": "browser", - "style": "dark", - "tags": ["splunk","autoscale","ingestion","indexer","io","pv"], - "graphTooltip": 1, - "panels": [ - { "id": 1, "type": "stat", "title": "Ingestion Replicas", "gridPos": {"x":0,"y":0,"w":4,"h":4}, "targets":[{"expr":"kube_statefulset_replicas{namespace=\"default\",statefulset=\"splunk-ingestor-ingestor\"}"}], "options": {"reduceOptions":{"calcs":["last"]},"orientation":"horizontal","colorMode":"value","graphMode":"none","textMode":"value","thresholds":{"mode":"absolute","steps":[{"value":null,"color":"#73BF69"},{"value":5,"color":"#EAB839"},{"value":8,"color":"#BF1B00"}]}}}, - { "id": 2, "type": "stat", "title": "Indexer Replicas", "gridPos": {"x":4,"y":0,"w":4,"h":4}, "targets":[{"expr":"kube_statefulset_replicas{namespace=\"default\",statefulset=\"splunk-indexer-indexer\"}"}], "options": {"reduceOptions":{"calcs":["last"]},"orientation":"horizontal","colorMode":"value","graphMode":"none","textMode":"value","thresholds":{"mode":"absolute","steps":[{"value":null,"color":"#73BF69"},{"value":5,"color":"#EAB839"},{"value":8,"color":"#BF1B00"}]}}}, - { "id": 3, "type": "timeseries","title": "Ingestion CPU (cores)","gridPos": {"x":8,"y":0,"w":8,"h":4},"targets":[{"expr":"sum(rate(container_cpu_usage_seconds_total{namespace=\"default\",pod=~\"splunk-ingestor-ingestor-.*\"}[1m]))","legendFormat":"CPU (cores)"}],"options":{"legend":{"displayMode":"list","placement":"bottom"},"yAxis":{"mode":"auto"},"color":{"mode":"fixed","fixedColor":"#FFA600"}}}, - { "id": 4, "type": "timeseries","title": "Ingestion Memory (MiB)","gridPos": {"x":16,"y":0,"w":8,"h":4},"targets":[{"expr":"sum(container_memory_usage_bytes{namespace=\"default\",pod=~\"splunk-ingestor-ingestor-.*\"}) / 1024 / 1024","legendFormat":"Memory (MiB)"}],"options":{"legend":{"displayMode":"list","placement":"bottom"},"yAxis":{"mode":"auto"},"color":{"mode":"fixed","fixedColor":"#00AF91"}}}, - { "id": 5, "type": "timeseries","title": "Ingestion Network In (KB/s)","gridPos": {"x":0,"y":8,"w":8,"h":4},"targets":[{"expr":"sum(rate(container_network_receive_bytes_total{namespace=\"default\",pod=~\"splunk-ingestor-ingestor-.*\"}[1m])) / 1024","legendFormat":"Net In (KB/s)"}],"options":{"legend":{"displayMode":"list","placement":"bottom"},"yAxis":{"mode":"auto"},"color":{"mode":"fixed","fixedColor":"#59A14F"}}}, - { "id": 6, "type": "timeseries","title": "Ingestion Network Out (KB/s)","gridPos": {"x":8,"y":8,"w":8,"h":4},"targets":[{"expr":"sum(rate(container_network_transmit_bytes_total{namespace=\"default\",pod=~\"splunk-ingestor-ingestor-.*\"}[1m])) / 1024","legendFormat":"Net Out (KB/s)"}],"options":{"legend":{"displayMode":"list","placement":"bottom"},"yAxis":{"mode":"auto"},"color":{"mode":"fixed","fixedColor":"#E15759"}}}, - { "id": 7, "type": "timeseries","title": "Indexer CPU (cores)","gridPos": {"x":16,"y":4,"w":8,"h":4},"targets":[{"expr":"sum(rate(container_cpu_usage_seconds_total{namespace=\"default\",pod=~\"splunk-indexer-indexer-.*\"}[1m]))","legendFormat":"CPU (cores)"}],"options":{"legend":{"displayMode":"list","placement":"bottom"},"yAxis":{"mode":"auto"},"color":{"mode":"fixed","fixedColor":"#7D4E57"}}}, - { "id":8, "type": "timeseries","title": "Indexer Memory (MiB)","gridPos": {"x":0,"y":12,"w":8,"h":4},"targets":[{"expr":"sum(container_memory_usage_bytes{namespace=\"default\",pod=~\"splunk-indexer-indexer-.*\"}) / 1024 / 1024","legendFormat":"Memory (MiB)"}],"options":{"legend":{"displayMode":"list","placement":"bottom"},"yAxis":{"mode":"auto"},"color":{"mode":"fixed","fixedColor":"#4E79A7"}}}, - { "id":9, "type": "timeseries","title": "Indexer Network In (KB/s)","gridPos": {"x":8,"y":12,"w":8,"h":4},"targets":[{"expr":"sum(rate(container_network_receive_bytes_total{namespace=\"default\",pod=~\"splunk-indexer-indexer-.*\"}[1m])) / 1024","legendFormat":"Net In (KB/s)"}],"options":{"legend":{"displayMode":"list","placement":"bottom"},"yAxis":{"mode":"auto"},"color":{"mode":"fixed","fixedColor":"#9467BD"}}}, - { "id":10, "type": "timeseries","title": "Indexer Network Out (KB/s)","gridPos": {"x":16,"y":12,"w":8,"h":4},"targets":[{"expr":"sum(rate(container_network_transmit_bytes_total{namespace=\"default\",pod=~\"splunk-indexer-indexer-.*\"}[1m])) / 1024","legendFormat":"Net Out (KB/s)"}],"options":{"legend":{"displayMode":"list","placement":"bottom"},"yAxis":{"mode":"auto"},"color":{"mode":"fixed","fixedColor":"#8C564B"}}}, - { "id":11, "type": "timeseries","title": "Ingestion Disk Read (KB/s)","gridPos": {"x":0,"y":16,"w":8,"h":4},"targets":[{"expr":"sum(rate(container_fs_reads_bytes_total{namespace=\"default\",pod=~\"splunk-ingestor-ingestor-.*\"}[1m])) / 1024","legendFormat":"Disk Read (KB/s)"}],"options":{"legend":{"displayMode":"list","placement":"bottom"},"yAxis":{"mode":"auto"},"color":{"mode":"fixed","fixedColor":"#1F77B4"}}}, - { "id":12, "type": "timeseries","title": "Ingestion Disk Write (KB/s)","gridPos": {"x":8,"y":16,"w":8,"h":4},"targets":[{"expr":"sum(rate(container_fs_writes_bytes_total{namespace=\"default\",pod=~\"splunk-ingestor-ingestor-.*\"}[1m])) / 1024","legendFormat":"Disk Write (KB/s)"}],"options":{"legend":{"displayMode":"list","placement":"bottom"},"yAxis":{"mode":"auto"},"color":{"mode":"fixed","fixedColor":"#FF7F0E"}}}, - { "id":13, "type": "timeseries","title": "Indexer PV Usage (GiB)","gridPos": {"x":0,"y":20,"w":8,"h":4},"targets":[{"expr":"kubelet_volume_stats_used_bytes{namespace=\"default\",persistentvolumeclaim=~\".*-indexer-.*\"} / 1024 / 1024 / 1024","legendFormat":"Used GiB"},{"expr":"kubelet_volume_stats_capacity_bytes{namespace=\"default\",persistentvolumeclaim=~\".*-indexer-.*\"} / 1024 / 1024 / 1024","legendFormat":"Capacity GiB"}],"options":{"legend":{"displayMode":"list","placement":"bottom"},"yAxis":{"mode":"auto"}}}, - { "id":14, "type": "timeseries","title": "Ingestion PV Usage (GiB)","gridPos": {"x":8,"y":20,"w":8,"h":4},"targets":[{"expr":"kubelet_volume_stats_used_bytes{namespace=\"default\",persistentvolumeclaim=~\".*-ingestor-.*\"} / 1024 / 1024 / 1024","legendFormat":"Used GiB"},{"expr":"kubelet_volume_stats_capacity_bytes{namespace=\"default\",persistentvolumeclaim=~\".*-ingestor-.*\"} / 1024 / 1024 / 1024","legendFormat":"Capacity GiB"}],"options":{"legend":{"displayMode":"list","placement":"bottom"},"yAxis":{"mode":"auto"}}} - ] -} -``` - -## Documentation References - -- [kube-prometheus-stack](https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack) - -# App Installation for Ingestor Cluster Instances - -Application installation is supported for Ingestor Cluster instances. However, as of now, applications are installed using local scope and if any application requires Splunk restart, there is no automated way to detect it and trigger automatically via Splunk Operator. - -Therefore, to be able to enforce Splunk restart for each of the Ingestor Cluster pods, it is recommended to add/update IngestorCluster CR annotations/labels and apply the new configuration which will trigger the rolling restart of Splunk pods for Ingestor Cluster. - -Ideally, update of annotations and labels should not trigger pod restart at all and it is under the investigation on how to stop this from happening and handle restart automatically. - -# Example - -1. Install CRDs and Splunk Operator for Kubernetes. - -- SOK_IMAGE_VERSION: version of the image for Splunk Operator for Kubernetes - -``` -$ make install -``` - -``` -$ kubectl apply -f ${SOK_IMAGE_VERSION}/splunk-operator-cluster.yaml --server-side -``` +Verify the ServiceAccount is annotated with the role ARN: -``` -$ kubectl get po -n splunk-operator -NAME READY STATUS RESTARTS AGE -splunk-operator-controller-manager-785b89d45c-dwfkd 2/2 Running 0 4d3h +```bash +kubectl describe sa ingestor-sa -n splunk +# Annotations: eks.amazonaws.com/role-arn: arn:aws:iam::111111111111:role/... ``` -2. Create a service account. +**Option B — Static credentials:** Create a Kubernetes Secret with `s3_access_key` and `s3_secret_key` keys. Reference this secret in `Queue.spec.sqs.volumes`. No ServiceAccount annotation is needed. -``` -$ eksctl create iamserviceaccount \ - --name ingestor-sa \ - --cluster ind-ing-sep-demo \ - --region us-west-2 \ - --attach-policy-arn arn:aws:iam::aws:policy/AmazonS3FullAccess \ - --attach-policy-arn arn:aws:iam::aws:policy/AmazonSQSFullAccess \ - --approve \ - --override-existing-serviceaccounts -``` - -``` -$ kubectl describe sa ingestor-sa -Name: ingestor-sa -Namespace: default -Labels: app.kubernetes.io/managed-by=eksctl -Annotations: eks.amazonaws.com/role-arn: arn:aws:iam::111111111111:role/eksctl-ind-ing-sep-demo-addon-iamserviceac-Role1-123456789123 -Image pull secrets: -Mountable secrets: -Tokens: -Events: -``` - -``` -$ aws iam get-role --role-name eksctl-ind-ing-sep-demo-addon-iamserviceac-Role1-123456789123 -{ - "Role": { - "Path": "/", - "RoleName": "eksctl-ind-ing-sep-demo-addon-iamserviceac-Role1-123456789123", - "RoleId": "123456789012345678901", - "Arn": "arn:aws:iam::111111111111:role/eksctl-ind-ing-sep-demo-addon-iamserviceac-Role1-123456789123", - "CreateDate": "2025-08-07T12:03:31+00:00", - "AssumeRolePolicyDocument": { - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Principal": { - "Federated": "arn:aws:iam::111111111111:oidc-provider/oidc.eks.us-west-2.amazonaws.com/id/1234567890123456789012345678901" - }, - "Action": "sts:AssumeRoleWithWebIdentity", - "Condition": { - "StringEquals": { - "oidc.eks.us-west-2.amazonaws.com/id/1234567890123456789012345678901:aud": "sts.amazonaws.com", - "oidc.eks.us-west-2.amazonaws.com/id/1234567890123456789012345678901:sub": "system:serviceaccount:default:ingestor-sa" - } - } - } - ] - }, - "Description": "", - "MaxSessionDuration": 3600, - "Tags": [ - { - "Key": "alpha.eksctl.io/cluster-name", - "Value": "ind-ing-sep-demo" - }, - { - "Key": "alpha.eksctl.io/iamserviceaccount-name", - "Value": "default/ingestor-sa" - }, - { - "Key": "alpha.eksctl.io/eksctl-version", - "Value": "0.211.0" - }, - { - "Key": "eksctl.cluster.k8s.io/v1alpha1/cluster-name", - "Value": "ind-ing-sep-demo" - } - ], - "RoleLastUsed": { - "LastUsedDate": "2025-08-18T08:47:27+00:00", - "Region": "us-west-2" - } - } -} +```bash +kubectl create secret generic s3-secret \ + --from-literal=s3_access_key=AKIAIOSFODNN7EXAMPLE \ + --from-literal=s3_secret_key=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY \ + -n splunk ``` -``` -$ aws iam list-attached-role-policies --role-name eksctl-ind-ing-sep-demo-addon-iamserviceac-Role1-123456789123 -{ - "AttachedPolicies": [ - { - "PolicyName": "AmazonSQSFullAccess", - "PolicyArn": "arn:aws:iam::aws:policy/AmazonSQSFullAccess" - }, - { - "PolicyName": "AmazonS3FullAccess", - "PolicyArn": "arn:aws:iam::aws:policy/AmazonS3FullAccess" - } - ] -} -``` +### Step 2 — Create the Queue CR -3. Install Queue resource. +Tell SOK about your SQS queue. The Queue CR is a shared reference — both the IngestorCluster and IndexerCluster will point to it. -``` -$ cat queue.yaml +```bash +kubectl apply -f - < -Annotations: -API Version: enterprise.splunk.com/v4 -Kind: Queue -Metadata: - Creation Timestamp: 2025-10-27T10:25:53Z - Finalizers: - enterprise.splunk.com/delete-pvc - Generation: 1 - Resource Version: 12345678 - UID: 12345678-1234-5678-1234-012345678911 -Spec: - Sqs: - Auth Region: us-west-2 - DLQ: sqs-dlq-test - Endpoint: https://sqs.us-west-2.amazonaws.com - Name: sqs-test - Provider: sqs -Status: - Message: - Phase: Ready - Resource Rev Map: -Events: -``` - -4. Install ObjectStorage resource. +Tell SOK about your S3 bucket for oversized messages. -``` -$ cat os.yaml +```bash +kubectl apply -f - < -Annotations: -API Version: enterprise.splunk.com/v4 -Kind: ObjectStorage -Metadata: - Creation Timestamp: 2025-10-27T10:25:53Z - Finalizers: - enterprise.splunk.com/delete-pvc - Generation: 1 - Resource Version: 12345678 - UID: 12345678-1234-5678-1234-012345678911 -Spec: - S3: - Endpoint: https://s3.us-west-2.amazonaws.com - Path: ingestion/smartbus-test - Provider: s3 -Status: - Message: - Phase: Ready - Resource Rev Map: -Events: -``` +### Step 4 — Deploy the IngestorCluster -5. Install IngestorCluster resource. +SOK will create the ingestor StatefulSet. On first boot, the operator builds the queue config ConfigMap and each pod's init container writes the Splunk app before Splunk starts. -``` -$ cat ingestor.yaml +```bash +kubectl apply -f - < -Annotations: -API Version: enterprise.splunk.com/v4 -Kind: IngestorCluster -Metadata: - Creation Timestamp: 2025-08-18T09:49:45Z - Generation: 1 - Resource Version: 12345678 - UID: 12345678-1234-1234-1234-1234567890123 -Spec: - Queue Ref: - Name: queue - Namespace: default - Image: splunk/splunk:${SPLUNK_IMAGE_VERSION} - Object Storage Ref: - Name: os - Namespace: default - Replicas: 3 - Service Account: ingestor-sa -Status: - App Context: - App Repo: - App Install Period Seconds: 90 - Defaults: - Premium Apps Props: - Es Defaults: - Install Max Retries: 2 - Bundle Push Status: - Is Deployment In Progress: false - Last App Info Check Time: 0 - Version: 0 - Credential Secret Version: 33744270 - Message: - Phase: Ready - Ready Replicas: 3 - Replicas: 3 - Resource Rev Map: - Selector: app.kubernetes.io/instance=splunk-ingestor-ingestor - Tel App Installed: true -Events: -``` - -``` -$ kubectl exec -it splunk-ingestor-ingestor-0 -- sh -$ kubectl exec -it splunk-ingestor-ingestor-1 -- sh -$ kubectl exec -it splunk-ingestor-ingestor-2 -- sh -sh-4.4$ env | grep AWS -AWS_DEFAULT_REGION=us-west-2 -AWS_WEB_IDENTITY_TOKEN_FILE=/var/run/secrets/eks.amazonaws.com/serviceaccount/token -AWS_REGION=us-west-2 -AWS_ROLE_ARN=arn:aws:iam::111111111111:role/eksctl-ind-ing-sep-demo-addon-iamserviceac-Role1-123456789123 -AWS_STS_REGIONAL_ENDPOINTS=regional -sh-4.4$ cat /opt/splunk/etc/system/local/default-mode.conf -[pipeline:remotequeueruleset] -disabled = false - -[pipeline:ruleset] -disabled = true - -[pipeline:remotequeuetyping] -disabled = false - -[pipeline:remotequeueoutput] -disabled = false - -[pipeline:typing] -disabled = true - -[pipeline:indexerPipe] -disabled = true - -sh-4.4$ cat /opt/splunk/etc/system/local/outputs.conf -[remote_queue:sqs-test] -remote_queue.sqs_smartbus.max_count.max_retries_per_part = 4 -remote_queue.sqs_smartbus.auth_region = us-west-2 -remote_queue.sqs_smartbus.dead_letter_queue.name = sqs-dlq-test -remote_queue.sqs_smartbus.encoding_format = s2s -remote_queue.sqs_smartbus.endpoint = https://sqs.us-west-2.amazonaws.com -remote_queue.sqs_smartbus.large_message_store.endpoint = https://s3.us-west-2.amazonaws.com -remote_queue.sqs_smartbus.large_message_store.path = s3://ingestion/smartbus-test -remote_queue.sqs_smartbus.retry_policy = max_count -remote_queue.sqs_smartbus.send_interval = 5s -remote_queue.type = sqs_smartbus -``` +### Step 5 — Deploy the Indexer Cluster -6. Install IndexerCluster resource. - -``` -$ cat idxc.yaml +```bash +kubectl apply -f - </50% 3 10 0 10s -``` +### Scaling Ingestors -``` -kubectl top pod -NAME CPU(cores) MEMORY(bytes) -hec-locust-load-29270124-f86gj 790m 221Mi -splunk-cm-cluster-manager-0 154m 1696Mi -splunk-indexer-indexer-0 107m 1339Mi -splunk-indexer-indexer-1 187m 1052Mi -splunk-indexer-indexer-2 203m 1703Mi -splunk-ingestor-ingestor-0 97m 517Mi -splunk-ingestor-ingestor-1 64m 585Mi -splunk-ingestor-ingestor-2 57m 565Mi -``` +To handle more ingestion throughput, increase replicas. New pods pick up the existing ConfigMap automatically — no reconfiguration is needed. -``` -$ kubectl get po -NAME READY STATUS RESTARTS AGE -hec-locust-load-29270126-szgv2 1/1 Running 0 30s -splunk-cm-cluster-manager-0 1/1 Running 0 41m -splunk-indexer-indexer-0 1/1 Running 0 38m -splunk-indexer-indexer-1 1/1 Running 0 38m -splunk-indexer-indexer-2 1/1 Running 0 38m -splunk-ingestor-ingestor-0 1/1 Running 0 53m -splunk-ingestor-ingestor-1 1/1 Running 0 55m -splunk-ingestor-ingestor-2 1/1 Running 0 57m -splunk-ingestor-ingestor-3 0/1 Running 0 116s -splunk-ingestor-ingestor-4 0/1 Running 0 116s +```bash +kubectl patch ingestorcluster ingestor -n splunk \ + --type=merge -p '{"spec":{"replicas":5}}' ``` -``` -kubectl top pod -NAME CPU(cores) MEMORY(bytes) -hec-locust-load-29270126-szgv2 532m 72Mi -splunk-cm-cluster-manager-0 91m 1260Mi -splunk-indexer-indexer-0 112m 865Mi -splunk-indexer-indexer-1 115m 855Mi -splunk-indexer-indexer-2 152m 1696Mi -splunk-ingestor-ingestor-0 115m 482Mi -splunk-ingestor-ingestor-1 76m 496Mi -splunk-ingestor-ingestor-2 156m 553Mi -splunk-ingestor-ingestor-3 355m 846Mi -splunk-ingestor-ingestor-4 1036m 979Mi -``` +### Scaling Indexers -``` -kubectl get hpa -NAME REFERENCE TARGETS MINPODS MAXPODS REPLICAS AGE -ing-hpa IngestorCluster/ingestor cpu: 115%/50% 3 10 10 8m54s +```bash +kubectl patch indexercluster indexer -n splunk \ + --type=merge -p '{"spec":{"replicas":5}}' ``` -8. Generate fake load. +### Rotating Credentials (Static Credential Mode) -- HEC_TOKEN: HEC token for making fake calls +Update the Kubernetes Secret referenced in `Queue.spec.sqs.volumes`. SOK detects the Secret's ResourceVersion change on the next reconcile, rebuilds the ConfigMap with new credentials, and triggers a rolling restart of ingestor pods. +```bash +kubectl create secret generic s3-secret \ + --from-literal=s3_access_key=NEW_ACCESS_KEY \ + --from-literal=s3_secret_key=NEW_SECRET_KEY \ + -n splunk \ + --dry-run=client -o yaml | kubectl apply -f - ``` -$ kubectl get secret splunk-default-secret -o yaml -apiVersion: v1 -data: - hec_token: HEC_TOKEN - idxc_secret: YWJjZGVmMTIzNDU2Cg== - pass4SymmKey: YWJjZGVmMTIzNDU2Cg== - password: YWJjZGVmMTIzNDU2Cg== - shc_secret: YWJjZGVmMTIzNDU2Cg== -kind: Secret -metadata: - creationTimestamp: "2025-08-26T10:15:11Z" - name: splunk-default-secret - namespace: default - ownerReferences: - - apiVersion: enterprise.splunk.com/v4 - controller: false - kind: IngestorCluster - name: ingestor - uid: 12345678-1234-1234-1234-1234567890123 - - apiVersion: enterprise.splunk.com/v4 - controller: false - kind: ClusterManager - name: cm - uid: 12345678-1234-1234-1234-1234567890125 - - apiVersion: enterprise.splunk.com/v4 - controller: false - kind: IndexerCluster - name: indexer - uid: 12345678-1234-1234-1234-1234567890124 - resourceVersion: "123456" - uid: 12345678-1234-1234-1234-1234567890126 -type: Opaque -``` +### Pausing Reconciliation + +To temporarily prevent SOK from making changes to an IngestorCluster (e.g. during a maintenance window): + +```bash +kubectl annotate ingestorcluster ingestor -n splunk \ + ingestorcluster.enterprise.splunk.com/paused=true ``` -$ echo HEC_TOKEN | base64 -d -HEC_TOKEN + +Remove the annotation to resume: +```bash +kubectl annotate ingestorcluster ingestor -n splunk \ + ingestorcluster.enterprise.splunk.com/paused- ``` +### Deploying Apps to Ingestors + +Use the `appRepo` field on the IngestorCluster spec to install Splunk apps via the App Framework. See [AppFramework](AppFramework.md) for configuration details. + +### Deleting the Deployment + +Delete in reverse dependency order to allow SOK to cleanly remove finalizers and deregister resources: + +```bash +kubectl delete ingestorcluster ingestor -n splunk +kubectl delete indexercluster indexer -n splunk +kubectl delete clustermanager cm -n splunk +kubectl delete objectstorage os -n splunk +kubectl delete queue queue -n splunk ``` -cat loadgen.yaml -apiVersion: v1 -kind: ConfigMap -metadata: - name: hec-locust-config -data: - requirements.txt: | - locust - requests - urllib3 - - locustfile.py: | - import urllib3 - from locust import HttpUser, task, between - - # disable insecure‐ssl warnings - urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) - - class HECUser(HttpUser): - wait_time = between(1, 2) - # use HTTPS and explicit port - host = "https://splunk-ingestor-ingestor-service:8088" - - def on_start(self): - # turn off SSL cert verification - self.client.verify = False - - @task - def send_event(self): - token = "HEC_TOKEN" - headers = { - "Authorization": f"Splunk {token}", - "Content-Type": "application/json" - } - payload = {"event": {"message": "load test", "value": 123}} - # this will POST to https://…:8088/services/collector/event - self.client.post( - "/services/collector/event", - json=payload, - headers=headers, - name="HEC POST" - ) + --- -apiVersion: batch/v1 -kind: CronJob -metadata: - name: hec-locust-load -spec: - schedule: "*/2 * * * *" - concurrencyPolicy: Replace - startingDeadlineSeconds: 60 - jobTemplate: - spec: - backoffLimit: 1 - template: - spec: - containers: - - name: locust - image: python:3.9-slim - command: - - sh - - -c - - | - pip install --no-cache-dir -r /app/requirements.txt \ - && exec locust \ - -f /app/locustfile.py \ - --headless \ - -u 200 \ - -r 50 \ - --run-time 1m50s - volumeMounts: - - name: app - mountPath: /app - restartPolicy: OnFailure - volumes: - - name: app - configMap: - name: hec-locust-config - defaultMode: 0755 -``` -``` -kubectl apply -f loadgen.yaml -``` +## Troubleshooting +### Ingestor pods stuck in Init state + +The init container `init-ingestor-queue-config` failed to run. Check why: + +```bash +kubectl describe pod splunk-ingestor-ingestor-0 -n splunk | grep -A 10 "Init Containers" +kubectl logs splunk-ingestor-ingestor-0 -n splunk -c init-ingestor-queue-config ``` -$ kubectl get cm -NAME DATA AGE -hec-locust-config 2 10s -kube-root-ca.crt 1 5d2h -splunk-cluster-manager-cm-configmap 1 28m -splunk-default-probe-configmap 3 58m -splunk-indexer-indexer-configmap 1 28m -splunk-ingestor-ingestor-configmap 1 48m -``` +Common causes: +- The ConfigMap `splunk-ingestor-ingestor-queue-config` was not yet created (wait for operator reconcile) +- The `serviceAccount` does not exist in the namespace + +### Queue config ConfigMap not found + +Verify the ConfigMap was created: +```bash +kubectl get cm splunk-ingestor-ingestor-queue-config -n splunk -o yaml ``` -$ kubectl get cj -NAME SCHEDULE TIMEZONE SUSPEND ACTIVE LAST SCHEDULE AGE -hec-locust-load */2 * * * * False 1 2s 26s + +If missing, check operator logs for errors from `buildAndApplyIngestorQueueConfigMap`: +```bash +kubectl logs -n splunk-operator deployment/splunk-operator-controller-manager | grep -i "ingestor-queue-config" ``` +### Ingestors cannot reach SQS + +Check that the pod's ServiceAccount has the correct IAM role annotation and that the role policy includes SQS permissions: +```bash +kubectl describe pod splunk-ingestor-ingestor-0 -n splunk | grep -i "service-account\|role-arn" ``` -$ kubectl get po -NAME READY STATUS RESTARTS AGE -hec-locust-load-29270114-zq7zz 1/1 Running 0 15s -splunk-cm-cluster-manager-0 1/1 Running 0 29m -splunk-indexer-indexer-0 1/1 Running 0 26m -splunk-indexer-indexer-1 1/1 Running 0 26m -splunk-indexer-indexer-2 1/1 Running 0 26m -splunk-ingestor-ingestor-0 1/1 Running 0 41m -splunk-ingestor-ingestor-1 1/1 Running 0 43m -splunk-ingestor-ingestor-2 1/1 Running 0 45m + +### Status and events + +```bash +# Overall status +kubectl get ingestorcluster,indexercluster,queue,objectstorage -n splunk + +# Detailed status with conditions and events +kubectl describe ingestorcluster ingestor -n splunk + +# Operator logs +kubectl logs -n splunk-operator deployment/splunk-operator-controller-manager --tail=100 ``` +--- + +## Helm Chart + +Queue, ObjectStorage, and IngestorCluster are included in the `splunk/splunk-enterprise` Helm chart. IndexerCluster has been extended with `queueRef` and `objectStorageRef` inputs. + +```yaml +queue: + enabled: true + name: queue + provider: sqs + sqs: + name: sqs-smartbus + authRegion: us-west-2 + dlq: sqs-smartbus-dlq + volumes: + - name: s3-sqs-creds + secretRef: s3-secret + +objectStorage: + enabled: true + name: os + provider: s3 + s3: + endpoint: https://s3.us-west-2.amazonaws.com + path: s3://my-smartbus-bucket/ingestion + +ingestorCluster: + enabled: true + name: ingestor + replicaCount: 3 + serviceAccount: ingestor-sa + queueRef: + name: queue + objectStorageRef: + name: os + +clusterManager: + enabled: true + name: cm + serviceAccount: ingestor-sa + +indexerCluster: + enabled: true + name: indexer + replicaCount: 3 + serviceAccount: ingestor-sa + clusterManagerRef: + name: cm + queueRef: + name: queue + objectStorageRef: + name: os ``` -$ aws s3 ls s3://ingestion/smartbus-test/ - PRE 29DDC1B4-D43E-47D1-AC04-C87AC7298201/ - PRE 43E16731-7146-4397-8553-D68B5C2C8634/ - PRE C8A4D060-DE0D-4DCB-9690-01D8902825DC/ -``` \ No newline at end of file + +--- + +## Reference + +- [Custom Resources](CustomResources.md) — CommonSplunkSpec fields shared by all CRs +- [App Framework](AppFramework.md) — Deploying Splunk apps to ingestor pods +- [SmartStore](SmartStore.md) — S3-backed index storage (separate from ingestion S3 bucket) +- [Security](Security.md) — RBAC, pod security, network policies diff --git a/docs/images/index_ingestion_separation.png b/docs/images/index_ingestion_separation.png new file mode 100644 index 000000000..e9db394b9 Binary files /dev/null and b/docs/images/index_ingestion_separation.png differ diff --git a/pkg/splunk/common/paths.go b/pkg/splunk/common/paths.go index 0aaf84d17..b72a15705 100644 --- a/pkg/splunk/common/paths.go +++ b/pkg/splunk/common/paths.go @@ -35,4 +35,22 @@ const ( //OperatorMountLocalServerConf OperatorMountLocalServerConf = "/mnt/splunk-operator/local/server.conf" + + //OperatorClusterManagerAppsLocalOutputsConf + OperatorClusterManagerAppsLocalOutputsConf = "/opt/splk/etc/manager-apps/splunk-operator/local/outputs.conf" + + //OperatorClusterManagerAppsLocalInputsConf + OperatorClusterManagerAppsLocalInputsConf = "/opt/splk/etc/manager-apps/splunk-operator/local/inputs.conf" + + //OperatorClusterManagerAppsLocalDefaultModeConf + OperatorClusterManagerAppsLocalDefaultModeConf = "/opt/splk/etc/manager-apps/splunk-operator/local/default-mode.conf" + + //OperatorMountLocalOutputsConf + OperatorMountLocalOutputsConf = "/mnt/splunk-operator/local/outputs.conf" + + //OperatorMountLocalInputsConf + OperatorMountLocalInputsConf = "/mnt/splunk-operator/local/inputs.conf" + + //OperatorMountLocalDefaultModeConf + OperatorMountLocalDefaultModeConf = "/mnt/splunk-operator/local/default-mode.conf" ) diff --git a/pkg/splunk/enterprise/clustermanager.go b/pkg/splunk/enterprise/clustermanager.go index 31835ee8e..fbf3799df 100644 --- a/pkg/splunk/enterprise/clustermanager.go +++ b/pkg/splunk/enterprise/clustermanager.go @@ -32,6 +32,7 @@ import ( appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" k8serrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/resource" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/log" @@ -313,12 +314,92 @@ func getClusterManagerStatefulSet(ctx context.Context, client splcommon.Controll if smartStoreConfigMap != nil { setupInitContainer(&ss.Spec.Template, cr.Spec.Image, cr.Spec.ImagePullPolicy, commandForCMSmartstore, cr.Spec.CommonSplunkSpec.EtcVolumeStorageConfig.EphemeralStorage) } + + // If a queue config ConfigMap exists for this CM, add a separate init container and volume. + setupCMQueueConfigInitContainer(ctx, client, cr, ss) // Setup App framework staging volume for apps setupAppsStagingVolume(ctx, client, cr, &ss.Spec.Template, &cr.Spec.AppFrameworkConfig) return ss, err } +// setupCMQueueConfigInitContainer adds a dedicated init container and ConfigMap volume for queue config +// to the ClusterManager StatefulSet if the queue config ConfigMap exists. This is a separate init container +// from the smartstore "init" container — it runs independently and symlinks outputs.conf, inputs.conf, +// and default-mode.conf from the queue config ConfigMap mount into manager-apps/splunk-operator/local/. +func setupCMQueueConfigInitContainer(ctx context.Context, client splcommon.ControllerClient, cr *enterpriseApi.ClusterManager, ss *appsv1.StatefulSet) { + configMapName := GetCMQueueConfigMapName(cr.GetName()) + // Only add the init container if the queue config ConfigMap exists. + _, err := splctrl.GetConfigMap(ctx, client, types.NamespacedName{Name: configMapName, Namespace: cr.GetNamespace()}) + if err != nil { + // ConfigMap doesn't exist yet — no queue config configured for this CM. + return + } + + // Add queue config ConfigMap volume to pod spec. + // defaultMode 420 (0644) must be set explicitly to match what Kubernetes stores — + // omitting it causes MergePodUpdates to see a diff on every reconcile (infinite update loop). + defaultMode := int32(420) + ss.Spec.Template.Spec.Volumes = append(ss.Spec.Template.Spec.Volumes, corev1.Volume{ + Name: cmQueueConfigVolName, + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: configMapName, + }, + DefaultMode: &defaultMode, + }, + }, + }) + + // Determine etc volume mount name (ephemeral vs PVC) + var etcVolMntName string + if cr.Spec.CommonSplunkSpec.EtcVolumeStorageConfig.EphemeralStorage { + etcVolMntName = fmt.Sprintf(splcommon.SplunkMountNamePrefix, splcommon.EtcVolumeStorage) + } else { + etcVolMntName = fmt.Sprintf(splcommon.PvcNamePrefix, splcommon.EtcVolumeStorage) + } + + runAsUser := int64(41812) + runAsNonRoot := true + privileged := false + + initContainer := corev1.Container{ + Name: "init-cm-queue-config", + Image: ss.Spec.Template.Spec.Containers[0].Image, + ImagePullPolicy: ss.Spec.Template.Spec.Containers[0].ImagePullPolicy, + Command: []string{"bash", "-c", commandForCMQueueConfig}, + VolumeMounts: []corev1.VolumeMount{ + {Name: etcVolMntName, MountPath: "/opt/splk/etc"}, + {Name: cmQueueConfigVolName, MountPath: cmQueueConfigMountPath}, + }, + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("0.25"), + corev1.ResourceMemory: resource.MustParse("128Mi"), + }, + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("1"), + corev1.ResourceMemory: resource.MustParse("512Mi"), + }, + }, + SecurityContext: &corev1.SecurityContext{ + RunAsUser: &runAsUser, + RunAsNonRoot: &runAsNonRoot, + AllowPrivilegeEscalation: &[]bool{false}[0], + Capabilities: &corev1.Capabilities{ + Drop: []corev1.Capability{"ALL"}, + Add: []corev1.Capability{"NET_BIND_SERVICE"}, + }, + Privileged: &privileged, + SeccompProfile: &corev1.SeccompProfile{ + Type: corev1.SeccompProfileTypeRuntimeDefault, + }, + }, + } + ss.Spec.Template.Spec.InitContainers = append(ss.Spec.Template.Spec.InitContainers, initContainer) +} + // CheckIfsmartstoreConfigMapUpdatedToPod checks if the smartstore configMap is updated on Pod or not func CheckIfsmartstoreConfigMapUpdatedToPod(ctx context.Context, c splcommon.ControllerClient, cr *enterpriseApi.ClusterManager, podExecClient splutil.PodExecClientImpl) error { reqLogger := log.FromContext(ctx) diff --git a/pkg/splunk/enterprise/clustermanager_test.go b/pkg/splunk/enterprise/clustermanager_test.go index 4a71bf8be..1752d4333 100644 --- a/pkg/splunk/enterprise/clustermanager_test.go +++ b/pkg/splunk/enterprise/clustermanager_test.go @@ -81,6 +81,7 @@ func TestApplyClusterManager(t *testing.T) { {MetaName: "*v1.Secret-test-splunk-stack1-cluster-manager-secret-v1"}, {MetaName: "*v1.ConfigMap-test-splunk-stack1-clustermanager-smartstore"}, {MetaName: "*v1.ConfigMap-test-splunk-stack1-clustermanager-smartstore"}, + {MetaName: "*v1.ConfigMap-test-splunk-stack1-clustermanager-queue-config"}, {MetaName: "*v1.StatefulSet-test-splunk-stack1-cluster-manager"}, {MetaName: "*v1.StatefulSet-test-splunk-stack1-cluster-manager"}, {MetaName: "*v4.ClusterManager-test-stack1"}, @@ -97,6 +98,7 @@ func TestApplyClusterManager(t *testing.T) { {MetaName: "*v1.Secret-test-splunk-stack1-cluster-manager-secret-v1"}, {MetaName: "*v1.ConfigMap-test-splunk-stack1-clustermanager-smartstore"}, {MetaName: "*v1.ConfigMap-test-splunk-stack1-clustermanager-smartstore"}, + {MetaName: "*v1.ConfigMap-test-splunk-stack1-clustermanager-queue-config"}, {MetaName: "*v1.StatefulSet-test-splunk-stack1-cluster-manager"}, {MetaName: "*v1.StatefulSet-test-splunk-stack1-cluster-manager"}, {MetaName: "*v1.StatefulSet-test-splunk-stack1-cluster-manager"}, @@ -578,6 +580,7 @@ func TestApplyClusterManagerWithSmartstore(t *testing.T) { {MetaName: "*v1.Secret-test-splunk-stack1-cluster-manager-secret-v1"}, {MetaName: "*v1.ConfigMap-test-splunk-stack1-clustermanager-smartstore"}, {MetaName: "*v1.ConfigMap-test-splunk-stack1-clustermanager-smartstore"}, + {MetaName: "*v1.ConfigMap-test-splunk-stack1-clustermanager-queue-config"}, {MetaName: "*v1.StatefulSet-test-splunk-stack1-cluster-manager"}, {MetaName: "*v1.StatefulSet-test-splunk-stack1-cluster-manager"}, {MetaName: "*v1.StatefulSet-test-splunk-stack1-cluster-manager"}, @@ -601,6 +604,7 @@ func TestApplyClusterManagerWithSmartstore(t *testing.T) { {MetaName: "*v1.Secret-test-splunk-stack1-cluster-manager-secret-v1"}, {MetaName: "*v1.ConfigMap-test-splunk-stack1-clustermanager-smartstore"}, {MetaName: "*v1.ConfigMap-test-splunk-stack1-clustermanager-smartstore"}, + {MetaName: "*v1.ConfigMap-test-splunk-stack1-clustermanager-queue-config"}, {MetaName: "*v1.StatefulSet-test-splunk-stack1-cluster-manager"}, {MetaName: "*v1.StatefulSet-test-splunk-stack1-cluster-manager"}, {MetaName: "*v1.StatefulSet-test-splunk-stack1-cluster-manager"}, diff --git a/pkg/splunk/enterprise/configuration.go b/pkg/splunk/enterprise/configuration.go index c9cc6838b..c3fc65909 100644 --- a/pkg/splunk/enterprise/configuration.go +++ b/pkg/splunk/enterprise/configuration.go @@ -880,17 +880,27 @@ func updateSplunkPodTemplateWithConfig(ctx context.Context, client splcommon.Con smartstoreConfigMap := getSmartstoreConfigMap(ctx, client, cr, instanceType) if smartstoreConfigMap != nil { + items := []corev1.KeyToPath{ + {Key: "indexes.conf", Path: "indexes.conf", Mode: &configMapVolDefaultMode}, + {Key: "server.conf", Path: "server.conf", Mode: &configMapVolDefaultMode}, + {Key: configToken, Path: configToken, Mode: &configMapVolDefaultMode}, + } + // When queue config keys are present (written by applyIdxcQueueConfigToCM), + // include them so the init container symlinks resolve correctly on the CM pod. + if _, ok := smartstoreConfigMap.Data["outputs.conf"]; ok { + items = append(items, + corev1.KeyToPath{Key: "outputs.conf", Path: "outputs.conf", Mode: &configMapVolDefaultMode}, + corev1.KeyToPath{Key: "inputs.conf", Path: "inputs.conf", Mode: &configMapVolDefaultMode}, + corev1.KeyToPath{Key: "default-mode.conf", Path: "default-mode.conf", Mode: &configMapVolDefaultMode}, + ) + } addSplunkVolumeToTemplate(podTemplateSpec, "mnt-splunk-operator", "/mnt/splunk-operator/local/", corev1.VolumeSource{ ConfigMap: &corev1.ConfigMapVolumeSource{ LocalObjectReference: corev1.LocalObjectReference{ Name: smartstoreConfigMap.GetName(), }, DefaultMode: &configMapVolDefaultMode, - Items: []corev1.KeyToPath{ - {Key: "indexes.conf", Path: "indexes.conf", Mode: &configMapVolDefaultMode}, - {Key: "server.conf", Path: "server.conf", Mode: &configMapVolDefaultMode}, - {Key: configToken, Path: configToken, Mode: &configMapVolDefaultMode}, - }, + Items: items, }, }) diff --git a/pkg/splunk/enterprise/indexercluster.go b/pkg/splunk/enterprise/indexercluster.go index c9e65e9f3..05c61d14a 100644 --- a/pkg/splunk/enterprise/indexercluster.go +++ b/pkg/splunk/enterprise/indexercluster.go @@ -44,6 +44,57 @@ import ( // NewSplunkClientFunc function pointer type type NewSplunkClientFunc func(managementURI, username, password string) *splclient.SplunkClient +// applyIdxcQueueConfigToCM builds and applies the ClusterManager queue config ConfigMap +// (splunk--clustermanager-queue-config) with outputs.conf, inputs.conf, and default-mode.conf. +// The CM bundle push infrastructure distributes these files to all indexer peers via manager-apps/ +// splunk-operator/local/ — no pod-by-pod REST calls needed. A dedicated init container on the CM pod +// symlinks the conf files from the ConfigMap mount before Splunk starts. +// Returns (true, nil) when content changed (bundle push trigger needed), (false, nil) when unchanged. +func applyIdxcQueueConfigToCM(ctx context.Context, client splcommon.ControllerClient, cr *enterpriseApi.IndexerCluster) (bool, error) { + cmName := cr.Spec.ClusterManagerRef.Name + cmNamespace := cr.GetNamespace() + + // Fetch the ClusterManager CR so we can update its BundlePushTracker on content change + cmCR := &enterpriseApi.ClusterManager{} + if err := client.Get(ctx, types.NamespacedName{Name: cmName, Namespace: cmNamespace}, cmCR); err != nil { + return false, fmt.Errorf("applyIdxcQueueConfigToCM: failed to get ClusterManager %s: %w", cmName, err) + } + + // Resolve Queue and ObjectStorage CRs to get credentials and endpoints + qosCfg, err := ResolveQueueAndObjectStorage(ctx, client, cr, cr.Spec.QueueRef, cr.Spec.ObjectStorageRef, cr.Spec.ServiceAccount) + if err != nil { + return false, fmt.Errorf("applyIdxcQueueConfigToCM: failed to resolve queue/OS config: %w", err) + } + + // Build the dedicated CM queue config ConfigMap data. + // outputs.conf and inputs.conf differ: outputs adds send_interval and encoding_format. + // default-mode.conf uses isIndexer=true (no indexerPipe stanza). + inputs, outputs := getQueueAndObjectStorageInputsForIndexerConfFiles(&qosCfg.Queue, &qosCfg.OS, qosCfg.AccessKey, qosCfg.SecretKey) + data := map[string]string{ + "app.conf": generateQueueConfigAppConf("Splunk Operator ClusterManager Queue Config"), + "outputs.conf": buildQueueConfStanza(qosCfg.Queue.SQS.Name, outputs), + "inputs.conf": buildQueueConfStanza(qosCfg.Queue.SQS.Name, inputs), + "default-mode.conf": generateIdxcDefaultModeConf(), + "local.meta": generateQueueConfigLocalMeta(), + } + + changed, err := applyQueueConfigMap(ctx, client, GetCMQueueConfigMapName(cmName), cmNamespace, cmCR, data) + if err != nil { + return false, fmt.Errorf("applyIdxcQueueConfigToCM: failed to apply ConfigMap: %w", err) + } + + if changed { + // Signal ClusterManager to run bundle push on next reconcile + cmCR.Status.BundlePushTracker.NeedToPushManagerApps = true + cmCR.Status.BundlePushTracker.LastCheckInterval = 0 + if err := client.Status().Update(ctx, cmCR); err != nil { + return true, fmt.Errorf("applyIdxcQueueConfigToCM: failed to update ClusterManager BundlePushTracker: %w", err) + } + } + + return changed, nil +} + // ApplyIndexerClusterManager reconciles the state of a Splunk Enterprise indexer cluster. func ApplyIndexerClusterManager(ctx context.Context, client splcommon.ControllerClient, cr *enterpriseApi.IndexerCluster) (reconcile.Result, error) { @@ -159,6 +210,16 @@ func ApplyIndexerClusterManager(ctx context.Context, client splcommon.Controller return result, err } + // Apply queue config to ClusterManager's smartstore ConfigMap for bundle push distribution. + // Called unconditionally — ApplyConfigMap skips write when content unchanged. + if cr.Spec.QueueRef.Name != "" && cr.Spec.ClusterManagerRef.Name != "" { + _, err = applyIdxcQueueConfigToCM(ctx, client, cr) + if err != nil { + eventPublisher.Warning(ctx, "applyIdxcQueueConfigToCM", fmt.Sprintf("failed to apply queue config to ClusterManager: %s", err.Error())) + return result, err + } + } + // create or update statefulset for the indexers statefulSet, err := getIndexerStatefulSet(ctx, client, cr) if err != nil { @@ -245,40 +306,6 @@ func ApplyIndexerClusterManager(ctx context.Context, client splcommon.Controller // no need to requeue if everything is ready if cr.Status.Phase == enterpriseApi.PhaseReady { - qosCfg, err := ResolveQueueAndObjectStorage(ctx, client, cr, cr.Spec.QueueRef, cr.Spec.ObjectStorageRef, cr.Spec.ServiceAccount) - if err != nil { - scopedLog.Error(err, "Failed to resolve Queue/ObjectStorage config") - return result, err - } - - secretChanged := cr.Status.CredentialSecretVersion != qosCfg.Version - serviceAccountChanged := cr.Status.ServiceAccount != cr.Spec.ServiceAccount - - // If queue is updated - if cr.Spec.QueueRef.Name != "" { - if secretChanged || serviceAccountChanged { - mgr := newIndexerClusterPodManager(scopedLog, cr, namespaceScopedSecret, splclient.NewSplunkClient, client) - err = mgr.updateIndexerConfFiles(ctx, cr, &qosCfg.Queue, &qosCfg.OS, qosCfg.AccessKey, qosCfg.SecretKey, client) - if err != nil { - eventPublisher.Warning(ctx, "ApplyIndexerClusterManager", fmt.Sprintf("Failed to update conf file for Queue/Pipeline config change after pod creation: %s", err.Error())) - scopedLog.Error(err, "Failed to update conf file for Queue/Pipeline config change after pod creation") - return result, err - } - - for i := int32(0); i < cr.Spec.Replicas; i++ { - idxcClient := mgr.getClient(ctx, i) - err = idxcClient.RestartSplunk() - if err != nil { - return result, err - } - scopedLog.Info("Restarted splunk", "indexer", i) - } - - cr.Status.CredentialSecretVersion = qosCfg.Version - cr.Status.ServiceAccount = cr.Spec.ServiceAccount - } - } - //update MC //Retrieve monitoring console ref from CM Spec cmMonitoringConsoleConfigRef, err := RetrieveCMSpec(ctx, client, cr) @@ -1383,6 +1410,17 @@ func getQueueAndPipelineInputsForIndexerConfFiles(queue *enterpriseApi.QueueSpec return } +// generateIdxcDefaultModeConf builds default-mode.conf INI content for an IndexerCluster peer. +// Uses getPipelineInputsForConfFile(true) — isIndexer=true omits the indexerPipe stanza. +func generateIdxcDefaultModeConf() string { + pipelineInputs := getPipelineInputsForConfFile(true) + var b strings.Builder + for _, input := range pipelineInputs { + fmt.Fprintf(&b, "[%s]\n%s = %s\n\n", input[0], input[1], input[2]) + } + return b.String() +} + // Tells if there is an image migration from 8.x.x to 9.x.x func imageUpdatedTo9(previousImage string, currentImage string) bool { // If there is no colon, version can't be detected diff --git a/pkg/splunk/enterprise/ingestorcluster.go b/pkg/splunk/enterprise/ingestorcluster.go index b0f866413..c807b5717 100644 --- a/pkg/splunk/enterprise/ingestorcluster.go +++ b/pkg/splunk/enterprise/ingestorcluster.go @@ -16,19 +16,20 @@ package enterprise import ( "context" + "crypto/sha256" "fmt" "reflect" "strings" "time" - "github.com/go-logr/logr" enterpriseApi "github.com/splunk/splunk-operator/api/v4" - splclient "github.com/splunk/splunk-operator/pkg/splunk/client" splcommon "github.com/splunk/splunk-operator/pkg/splunk/common" splctrl "github.com/splunk/splunk-operator/pkg/splunk/splkcontroller" splutil "github.com/splunk/splunk-operator/pkg/splunk/util" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/api/resource" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/log" @@ -70,10 +71,6 @@ func ApplyIngestorCluster(ctx context.Context, client client.Client, cr *enterpr // Update the CR Status defer updateCRStatus(ctx, client, cr, &err) - if cr.Status.Replicas < cr.Spec.Replicas { - cr.Status.CredentialSecretVersion = "0" - cr.Status.ServiceAccount = "" - } cr.Status.Replicas = cr.Spec.Replicas // If needed, migrate the app framework status @@ -97,7 +94,7 @@ func ApplyIngestorCluster(ctx context.Context, client client.Client, cr *enterpr cr.Status.Selector = fmt.Sprintf("app.kubernetes.io/instance=splunk-%s-ingestor", cr.GetName()) // Create or update general config resources - namespaceScopedSecret, err := ApplySplunkConfig(ctx, client, cr, cr.Spec.CommonSplunkSpec, SplunkIngestor) + _, err = ApplySplunkConfig(ctx, client, cr, cr.Spec.CommonSplunkSpec, SplunkIngestor) if err != nil { scopedLog.Error(err, "create or update general config failed", "error", err.Error()) eventPublisher.Warning(ctx, "ApplySplunkConfig", fmt.Sprintf("create or update general config failed with error %s", err.Error())) @@ -184,6 +181,20 @@ func ApplyIngestorCluster(ctx context.Context, client client.Client, cr *enterpr } } + // Resolve Queue/ObjectStorage CRs and build/apply queue config ConfigMap. + // Called unconditionally — splctrl.ApplyConfigMap skips write when content unchanged. + // Controller watches (Queue CR, ObjectStorage CR, Secret) ensure reconcile only fires on real changes. + qosCfg, err := ResolveQueueAndObjectStorage(ctx, client, cr, cr.Spec.QueueRef, cr.Spec.ObjectStorageRef, cr.Spec.ServiceAccount) + if err != nil { + eventPublisher.Warning(ctx, "ResolveQueueAndObjectStorage", fmt.Sprintf("failed to resolve queue/OS config: %s", err.Error())) + return result, err + } + _, err = buildAndApplyIngestorQueueConfigMap(ctx, client, cr, qosCfg) + if err != nil { + eventPublisher.Warning(ctx, "buildAndApplyIngestorQueueConfigMap", fmt.Sprintf("failed to build/apply queue config ConfigMap: %s", err.Error())) + return result, err + } + // Create or update statefulset for the ingestors statefulSet, err := getIngestorStatefulSet(ctx, client, cr) if err != nil { @@ -209,38 +220,6 @@ func ApplyIngestorCluster(ctx context.Context, client client.Client, cr *enterpr // No need to requeue if everything is ready if cr.Status.Phase == enterpriseApi.PhaseReady { - qosCfg, err := ResolveQueueAndObjectStorage(ctx, client, cr, cr.Spec.QueueRef, cr.Spec.ObjectStorageRef, cr.Spec.ServiceAccount) - if err != nil { - scopedLog.Error(err, "Failed to resolve Queue/ObjectStorage config") - return result, err - } - - secretChanged := cr.Status.CredentialSecretVersion != qosCfg.Version - serviceAccountChanged := cr.Status.ServiceAccount != cr.Spec.ServiceAccount - - // If queue is updated - if secretChanged || serviceAccountChanged { - mgr := newIngestorClusterPodManager(scopedLog, cr, namespaceScopedSecret, splclient.NewSplunkClient, client) - err = mgr.updateIngestorConfFiles(ctx, cr, &qosCfg.Queue, &qosCfg.OS, qosCfg.AccessKey, qosCfg.SecretKey, client) - if err != nil { - eventPublisher.Warning(ctx, "ApplyIngestorCluster", fmt.Sprintf("Failed to update conf file for Queue/Pipeline config change after pod creation: %s", err.Error())) - scopedLog.Error(err, "Failed to update conf file for Queue/Pipeline config change after pod creation") - return result, err - } - - for i := int32(0); i < cr.Spec.Replicas; i++ { - ingClient := mgr.getClient(ctx, i) - err = ingClient.RestartSplunk() - if err != nil { - return result, err - } - scopedLog.Info("Restarted splunk", "ingestor", i) - } - - cr.Status.CredentialSecretVersion = qosCfg.Version - cr.Status.ServiceAccount = cr.Spec.ServiceAccount - } - // Upgrade fron automated MC to MC CRD namespacedName := types.NamespacedName{Namespace: cr.GetNamespace(), Name: GetSplunkStatefulsetName(SplunkMonitoringConsole, cr.GetNamespace())} err = splctrl.DeleteReferencesToAutomatedMCIfExists(ctx, client, cr, namespacedName) @@ -281,27 +260,6 @@ func ApplyIngestorCluster(ctx context.Context, client client.Client, cr *enterpr return result, nil } -// getClient for ingestorClusterPodManager returns a SplunkClient for the member n -func (mgr *ingestorClusterPodManager) getClient(ctx context.Context, n int32) *splclient.SplunkClient { - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("ingestorClusterPodManager.getClient").WithValues("name", mgr.cr.GetName(), "namespace", mgr.cr.GetNamespace()) - - // Get Pod Name - memberName := GetSplunkStatefulsetPodName(SplunkIngestor, mgr.cr.GetName(), n) - - // Get Fully Qualified Domain Name - fqdnName := splcommon.GetServiceFQDN(mgr.cr.GetNamespace(), - fmt.Sprintf("%s.%s", memberName, GetSplunkServiceName(SplunkIngestor, mgr.cr.GetName(), true))) - - // Retrieve admin password from Pod - adminPwd, err := splutil.GetSpecificSecretTokenFromPod(ctx, mgr.c, memberName, mgr.cr.GetNamespace(), "password") - if err != nil { - scopedLog.Error(err, "Couldn't retrieve the admin password from pod") - } - - return mgr.newSplunkClient(fmt.Sprintf("https://%s:8089", fqdnName), "admin", adminPwd) -} - // validateIngestorClusterSpec checks validity and makes default updates to a IngestorClusterSpec and returns error if something is wrong func validateIngestorClusterSpec(ctx context.Context, c splcommon.ControllerClient, cr *enterpriseApi.IngestorCluster) error { // We cannot have 0 replicas in IngestorCluster spec since this refers to number of ingestion pods in the ingestor cluster @@ -319,6 +277,131 @@ func validateIngestorClusterSpec(ctx context.Context, c splcommon.ControllerClie return validateCommonSplunkSpec(ctx, c, &cr.Spec.CommonSplunkSpec, cr) } +// applyQueueConfigMap is a shared helper that builds and idempotently applies a queue config +// ConfigMap with the given name, namespace, owner, and data. +// Returns (true, nil) when content changed, (false, nil) when unchanged. +// Both IngestorCluster and ClusterManager (for IndexerCluster bundle push) use this. +func applyQueueConfigMap(ctx context.Context, c splcommon.ControllerClient, configMapName, namespace string, owner splcommon.MetaObject, data map[string]string) (bool, error) { + configMap := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: configMapName, + Namespace: namespace, + }, + Data: data, + } + configMap.SetOwnerReferences(append(configMap.GetOwnerReferences(), splcommon.AsOwner(owner, true))) + return splctrl.ApplyConfigMap(ctx, c, configMap) +} + +// buildAndApplyIngestorQueueConfigMap builds and idempotently applies the ingestor queue +// config ConfigMap. Returns (true, nil) when content changed, (false, nil) when unchanged. +// Called unconditionally on every reconcile — splctrl.ApplyConfigMap is the idempotency gate. +func buildAndApplyIngestorQueueConfigMap(ctx context.Context, c splcommon.ControllerClient, cr *enterpriseApi.IngestorCluster, qosCfg *QueueOSConfig) (bool, error) { + outputsConf := generateIngestorOutputsConf(&qosCfg.Queue, &qosCfg.OS, qosCfg.AccessKey, qosCfg.SecretKey) + defaultModeConf := generateIngestorDefaultModeConf() + data := map[string]string{ + "app.conf": generateQueueConfigAppConf("Splunk Operator Ingestor Queue Config"), + "outputs.conf": outputsConf, + "default-mode.conf": defaultModeConf, + "local.meta": generateQueueConfigLocalMeta(), + } + return applyQueueConfigMap(ctx, c, GetIngestorQueueConfigMapName(cr.GetName()), cr.GetNamespace(), cr, data) +} + +// setupIngestorInitContainer adds the queue config init container and ConfigMap volume to the +// StatefulSet pod template. The init container symlinks conf files and copies local.meta before +// Splunk starts, enabling zero-restart first-boot configuration. +func setupIngestorInitContainer(ctx context.Context, c splcommon.ControllerClient, cr *enterpriseApi.IngestorCluster, ss *appsv1.StatefulSet) error { + // Determine etc volume mount name (ephemeral vs PVC — mirrors setupInitContainer pattern) + var etcVolMntName string + if cr.Spec.CommonSplunkSpec.EtcVolumeStorageConfig.EphemeralStorage { + etcVolMntName = fmt.Sprintf(splcommon.SplunkMountNamePrefix, splcommon.EtcVolumeStorage) + } else { + etcVolMntName = fmt.Sprintf(splcommon.PvcNamePrefix, splcommon.EtcVolumeStorage) + } + + // Add ConfigMap volume to pod spec. defaultMode 420 (0644) matches what Kubernetes + // applies by default after admission — setting it explicitly prevents a spurious + // StatefulSet diff on every reconcile (current=420 vs revised=nil). + queueConfigVolName := "mnt-splunk-queue-config" + defaultMode := int32(420) + ss.Spec.Template.Spec.Volumes = append(ss.Spec.Template.Spec.Volumes, corev1.Volume{ + Name: queueConfigVolName, + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: GetIngestorQueueConfigMapName(cr.GetName()), + }, + DefaultMode: &defaultMode, + }, + }, + }) + + // Security context — same as setupInitContainer in util.go + runAsUser := int64(41812) + runAsNonRoot := true + privileged := false + + initContainer := corev1.Container{ + Name: "init-ingestor-queue-config", + Image: ss.Spec.Template.Spec.Containers[0].Image, + ImagePullPolicy: ss.Spec.Template.Spec.Containers[0].ImagePullPolicy, + Command: []string{"bash", "-c", commandForIngestorQueueConfig}, + VolumeMounts: []corev1.VolumeMount{ + {Name: etcVolMntName, MountPath: "/opt/splk/etc"}, + {Name: queueConfigVolName, MountPath: ingestorQueueConfigMountPath}, + }, + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("0.25"), + corev1.ResourceMemory: resource.MustParse("128Mi"), + }, + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("1"), + corev1.ResourceMemory: resource.MustParse("512Mi"), + }, + }, + SecurityContext: &corev1.SecurityContext{ + RunAsUser: &runAsUser, + RunAsNonRoot: &runAsNonRoot, + AllowPrivilegeEscalation: &[]bool{false}[0], + Capabilities: &corev1.Capabilities{ + Drop: []corev1.Capability{"ALL"}, + Add: []corev1.Capability{"NET_BIND_SERVICE"}, + }, + Privileged: &privileged, + SeccompProfile: &corev1.SeccompProfile{ + Type: corev1.SeccompProfileTypeRuntimeDefault, + }, + }, + } + ss.Spec.Template.Spec.InitContainers = append(ss.Spec.Template.Spec.InitContainers, initContainer) + + // Also mount the ConfigMap volume in the main Splunk container so that the symlinks + // created by the init container resolve correctly at runtime. + ss.Spec.Template.Spec.Containers[0].VolumeMounts = append(ss.Spec.Template.Spec.Containers[0].VolumeMounts, corev1.VolumeMount{ + Name: queueConfigVolName, + MountPath: ingestorQueueConfigMountPath, + ReadOnly: true, + }) + + // Set ingestorQueueConfigRev annotation to ConfigMap ResourceVersion. + // When ConfigMap content changes the RV increments, the annotation changes, + // and the Restart EPIC detects the pod template diff and triggers a rolling restart. + cmRV, err := splctrl.GetConfigMapResourceVersion(ctx, c, types.NamespacedName{ + Name: GetIngestorQueueConfigMapName(cr.GetName()), + Namespace: cr.GetNamespace(), + }) + if err == nil && cmRV != "" { + if ss.Spec.Template.ObjectMeta.Annotations == nil { + ss.Spec.Template.ObjectMeta.Annotations = make(map[string]string) + } + ss.Spec.Template.ObjectMeta.Annotations[ingestorQueueConfigRevAnnotation] = cmRV + } + + return nil +} + // getIngestorStatefulSet returns a Kubernetes StatefulSet object for Splunk Enterprise ingestors func getIngestorStatefulSet(ctx context.Context, client splcommon.ControllerClient, cr *enterpriseApi.IngestorCluster) (*appsv1.StatefulSet, error) { ss, err := getSplunkStatefulSet(ctx, client, cr, &cr.Spec.CommonSplunkSpec, SplunkIngestor, cr.Spec.Replicas, []corev1.EnvVar{}) @@ -329,74 +412,78 @@ func getIngestorStatefulSet(ctx context.Context, client splcommon.ControllerClie // Setup App framework staging volume for apps setupAppsStagingVolume(ctx, client, cr, &ss.Spec.Template, &cr.Spec.AppFrameworkConfig) + // Add queue config ConfigMap volume + init container + ingestorQueueConfigRev annotation + if err := setupIngestorInitContainer(ctx, client, cr, ss); err != nil { + return nil, err + } + return ss, nil } -// updateIngestorConfFiles checks if Queue or Pipeline inputs are created for the first time and updates the conf file if so -func (mgr *ingestorClusterPodManager) updateIngestorConfFiles(ctx context.Context, newCR *enterpriseApi.IngestorCluster, queue *enterpriseApi.QueueSpec, os *enterpriseApi.ObjectStorageSpec, accessKey, secretKey string, k8s client.Client) error { - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("updateIngestorConfFiles").WithValues("name", newCR.GetName(), "namespace", newCR.GetNamespace()) - - // Only update config for pods that exist - readyReplicas := newCR.Status.Replicas - - // List all pods for this IngestorCluster StatefulSet - var updateErr error - for n := 0; n < int(readyReplicas); n++ { - memberName := GetSplunkStatefulsetPodName(SplunkIngestor, newCR.GetName(), int32(n)) - fqdnName := splcommon.GetServiceFQDN(newCR.GetNamespace(), fmt.Sprintf("%s.%s", memberName, GetSplunkServiceName(SplunkIngestor, newCR.GetName(), true))) - adminPwd, err := splutil.GetSpecificSecretTokenFromPod(ctx, k8s, memberName, newCR.GetNamespace(), "password") - if err != nil { - return err - } - splunkClient := mgr.newSplunkClient(fmt.Sprintf("https://%s:8089", fqdnName), "admin", string(adminPwd)) - - queueInputs, pipelineInputs := getQueueAndPipelineInputsForIngestorConfFiles(queue, os, accessKey, secretKey) - - for _, input := range queueInputs { - if err := splunkClient.UpdateConfFile(scopedLog, "outputs", fmt.Sprintf("remote_queue:%s", queue.SQS.Name), [][]string{input}); err != nil { - updateErr = err - } - } +// computeIngestorConfChecksum returns a SHA-256 hex digest of the combined conf content. +// Stored in local.meta so Splunk detects content changes on app load. +func computeIngestorConfChecksum(outputsConf, defaultModeConf string) string { + h := sha256.New() + h.Write([]byte(outputsConf)) + h.Write([]byte(defaultModeConf)) + return fmt.Sprintf("%x", h.Sum(nil)) +} - for _, input := range pipelineInputs { - if err := splunkClient.UpdateConfFile(scopedLog, "default-mode", input[0], [][]string{{input[1], input[2]}}); err != nil { - updateErr = err - } - } +// buildQueueConfStanza builds an INI stanza for a remote_queue conf file. +// stanzaName is used as the stanza header (e.g. queue.SQS.Name), kvPairs is the list of key-value pairs. +// Shared by both IngestorCluster (outputs.conf) and ClusterManager (outputs.conf + inputs.conf). +func buildQueueConfStanza(stanzaName string, kvPairs [][]string) string { + var b strings.Builder + fmt.Fprintf(&b, "[remote_queue:%s]\n", stanzaName) + for _, kv := range kvPairs { + fmt.Fprintf(&b, "%s = %s\n", kv[0], kv[1]) } - - return updateErr + return b.String() } -// getQueueAndPipelineInputsForIngestorConfFiles returns a list of queue and pipeline inputs for ingestor pods conf files -func getQueueAndPipelineInputsForIngestorConfFiles(queue *enterpriseApi.QueueSpec, os *enterpriseApi.ObjectStorageSpec, accessKey, secretKey string) (queueInputs, pipelineInputs [][]string) { - // Queue Inputs - queueInputs = getQueueAndObjectStorageInputsForIngestorConfFiles(queue, os, accessKey, secretKey) - - // Pipeline inputs - pipelineInputs = getPipelineInputsForConfFile(false) +// generateIngestorOutputsConf builds outputs.conf INI content. +// Reuses getQueueAndObjectStorageInputsForIngestorConfFiles for key-value pairs. +// Credentials embedded when non-empty (same pattern as GetSmartstoreVolumesConfig). +func generateIngestorOutputsConf(queue *enterpriseApi.QueueSpec, os *enterpriseApi.ObjectStorageSpec, accessKey, secretKey string) string { + kvPairs := getQueueAndObjectStorageInputsForIngestorConfFiles(queue, os, accessKey, secretKey) + return buildQueueConfStanza(queue.SQS.Name, kvPairs) +} - return +// generateIngestorDefaultModeConf builds default-mode.conf INI content. +// Reuses getPipelineInputsForConfFile(false) for the six pipeline stanzas. +func generateIngestorDefaultModeConf() string { + pipelineInputs := getPipelineInputsForConfFile(false) + var b strings.Builder + for _, input := range pipelineInputs { + fmt.Fprintf(&b, "[%s]\n%s = %s\n\n", input[0], input[1], input[2]) + } + return b.String() } -type ingestorClusterPodManager struct { - c splcommon.ControllerClient - log logr.Logger - cr *enterpriseApi.IngestorCluster - secrets *corev1.Secret - newSplunkClient func(managementURI, username, password string) *splclient.SplunkClient +// generateQueueConfigAppConf builds app.conf INI content for a queue config Splunk app. +// label is shown in the Splunk UI (e.g. "Splunk Operator Ingestor Queue Config"). +func generateQueueConfigAppConf(label string) string { + return fmt.Sprintf(`[install] +state = enabled +allows_disable = false + +[package] +check_for_updates = false + +[ui] +is_visible = false +is_manageable = false +label = %s +`, label) } -// newIngestorClusterPodManager creates pod manager to handle unit test cases -var newIngestorClusterPodManager = func(log logr.Logger, cr *enterpriseApi.IngestorCluster, secret *corev1.Secret, newSplunkClient NewSplunkClientFunc, c splcommon.ControllerClient) ingestorClusterPodManager { - return ingestorClusterPodManager{ - log: log, - cr: cr, - secrets: secret, - newSplunkClient: newSplunkClient, - c: c, - } +// generateQueueConfigLocalMeta builds local.meta with system-level access. +// Shared by both IngestorCluster and ClusterManager queue config apps. +func generateQueueConfigLocalMeta() string { + return `[] +access = read : [ * ], write : [ admin ] +export = system +` } // getPipelineInputsForConfFile returns a list of pipeline inputs for conf file diff --git a/pkg/splunk/enterprise/ingestorcluster_test.go b/pkg/splunk/enterprise/ingestorcluster_test.go index e96002372..b59a7c731 100644 --- a/pkg/splunk/enterprise/ingestorcluster_test.go +++ b/pkg/splunk/enterprise/ingestorcluster_test.go @@ -16,17 +16,11 @@ package enterprise import ( "context" - "fmt" - "net/http" "os" "path/filepath" - "strings" "testing" - "github.com/go-logr/logr" enterpriseApi "github.com/splunk/splunk-operator/api/v4" - splclient "github.com/splunk/splunk-operator/pkg/splunk/client" - splcommon "github.com/splunk/splunk-operator/pkg/splunk/common" spltest "github.com/splunk/splunk-operator/pkg/splunk/test" splutil "github.com/splunk/splunk-operator/pkg/splunk/util" "github.com/stretchr/testify/assert" @@ -64,8 +58,6 @@ func TestApplyIngestorCluster(t *testing.T) { c := fake.NewClientBuilder().WithScheme(scheme).Build() // Object definitions - provider := "sqs_smartbus" - queue := &enterpriseApi.Queue{ TypeMeta: metav1.TypeMeta{ Kind: "Queue", @@ -249,68 +241,7 @@ func TestApplyIngestorCluster(t *testing.T) { assert.True(t, result.Requeue) assert.NotEqual(t, enterpriseApi.PhaseError, cr.Status.Phase) - // outputs.conf - origNew := newIngestorClusterPodManager - mockHTTPClient := &spltest.MockHTTPClient{} - newIngestorClusterPodManager = func(l logr.Logger, cr *enterpriseApi.IngestorCluster, secret *corev1.Secret, _ NewSplunkClientFunc, c splcommon.ControllerClient) ingestorClusterPodManager { - return ingestorClusterPodManager{ - c: c, - log: l, cr: cr, secrets: secret, - newSplunkClient: func(uri, user, pass string) *splclient.SplunkClient { - return &splclient.SplunkClient{ManagementURI: uri, Username: user, Password: pass, Client: mockHTTPClient} - }, - } - } - defer func() { newIngestorClusterPodManager = origNew }() - - propertyKVList := [][]string{ - {"remote_queue.type", provider}, - {fmt.Sprintf("remote_queue.%s.encoding_format", provider), "s2s"}, - {fmt.Sprintf("remote_queue.%s.auth_region", provider), queue.Spec.SQS.AuthRegion}, - {fmt.Sprintf("remote_queue.%s.endpoint", provider), queue.Spec.SQS.Endpoint}, - {fmt.Sprintf("remote_queue.%s.large_message_store.endpoint", provider), os.Spec.S3.Endpoint}, - {fmt.Sprintf("remote_queue.%s.large_message_store.path", provider), os.Spec.S3.Path}, - {fmt.Sprintf("remote_queue.%s.dead_letter_queue.name", provider), queue.Spec.SQS.DLQ}, - {fmt.Sprintf("remote_queue.%s.max_count.max_retries_per_part", provider), "4"}, - {fmt.Sprintf("remote_queue.%s.retry_policy", provider), "max_count"}, - {fmt.Sprintf("remote_queue.%s.send_interval", provider), "5s"}, - } - - body := buildFormBody(propertyKVList) - addRemoteQueueHandlersForIngestor(mockHTTPClient, cr, &queue.Spec, "conf-outputs", body) - - // default-mode.conf - propertyKVList = [][]string{ - {"pipeline:remotequeueruleset", "disabled", "false"}, - {"pipeline:ruleset", "disabled", "true"}, - {"pipeline:remotequeuetyping", "disabled", "false"}, - {"pipeline:remotequeueoutput", "disabled", "false"}, - {"pipeline:typing", "disabled", "true"}, - {"pipeline:indexerPipe", "disabled", "true"}, - } - - for i := 0; i < int(cr.Status.ReadyReplicas); i++ { - podName := fmt.Sprintf("splunk-test-ingestor-%d", i) - baseURL := fmt.Sprintf("https://%s.splunk-%s-ingestor-headless.%s.svc.cluster.local:8089/servicesNS/nobody/system/configs/conf-default-mode", podName, cr.GetName(), cr.GetNamespace()) - - for _, field := range propertyKVList { - req, _ := http.NewRequest("POST", baseURL, strings.NewReader(fmt.Sprintf("name=%s", field[0]))) - mockHTTPClient.AddHandler(req, 200, "", nil) - - updateURL := fmt.Sprintf("%s/%s", baseURL, field[0]) - req, _ = http.NewRequest("POST", updateURL, strings.NewReader(fmt.Sprintf("%s=%s", field[1], field[2]))) - mockHTTPClient.AddHandler(req, 200, "", nil) - } - } - - for i := 0; i < int(cr.Status.ReadyReplicas); i++ { - podName := fmt.Sprintf("splunk-test-ingestor-%d", i) - baseURL := fmt.Sprintf("https://%s.splunk-%s-ingestor-headless.%s.svc.cluster.local:8089/services/server/control/restart", podName, cr.GetName(), cr.GetNamespace()) - req, _ := http.NewRequest("POST", baseURL, nil) - mockHTTPClient.AddHandler(req, 200, "", nil) - } - - // Second reconcile should now yield Ready + // Second reconcile with telemetry already installed should yield Ready cr.Status.TelAppInstalled = true result, err = ApplyIngestorCluster(ctx, c, cr) assert.NoError(t, err) @@ -371,7 +302,9 @@ func TestGetIngestorStatefulSet(t *testing.T) { } return getIngestorStatefulSet(ctx, c, &cr) } - configTester(t, "getIngestorStatefulSet()", f, want) + // Use configTester2 (no space-stripping) because the init container command string + // contains meaningful spaces that must be preserved in comparison. + configTester2(t, "getIngestorStatefulSet()", f, want) } // Define additional service port in CR and verify the statefulset has the new port @@ -404,374 +337,78 @@ func TestGetIngestorStatefulSet(t *testing.T) { test(loadFixture(t, "statefulset_ingestor_with_labels.json")) } -func TestGetQueueAndPipelineInputsForIngestorConfFiles(t *testing.T) { - provider := "sqs_smartbus" - - queue := enterpriseApi.Queue{ - TypeMeta: metav1.TypeMeta{ - Kind: "Queue", - APIVersion: "enterprise.splunk.com/v4", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "queue", - }, - Spec: enterpriseApi.QueueSpec{ - Provider: "sqs", - SQS: enterpriseApi.SQSSpec{ - Name: "test-queue", - AuthRegion: "us-west-2", - Endpoint: "https://sqs.us-west-2.amazonaws.com", - DLQ: "sqs-dlq-test", - VolList: []enterpriseApi.VolumeSpec{ - {SecretRef: "secret"}, - }, - }, - }, - } - - os := enterpriseApi.ObjectStorage{ - TypeMeta: metav1.TypeMeta{ - Kind: "ObjectStorage", - APIVersion: "enterprise.splunk.com/v4", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "os", - }, - Spec: enterpriseApi.ObjectStorageSpec{ - Provider: "s3", - S3: enterpriseApi.S3Spec{ - Endpoint: "https://s3.us-west-2.amazonaws.com", - Path: "bucket/key", - }, - }, +func TestComputeIngestorConfChecksum(t *testing.T) { + checksum := computeIngestorConfChecksum("outputs", "defaultmode") + // SHA-256 produces 64 hex chars + if len(checksum) != 64 { + t.Errorf("expected 64-char hex, got %d: %s", len(checksum), checksum) } - key := "key" - secret := "secret" - - queueInputs, pipelineInputs := getQueueAndPipelineInputsForIngestorConfFiles(&queue.Spec, &os.Spec, key, secret) - - assert.Equal(t, 12, len(queueInputs)) - assert.Equal(t, [][]string{ - {"remote_queue.type", provider}, - {fmt.Sprintf("remote_queue.%s.auth_region", provider), queue.Spec.SQS.AuthRegion}, - {fmt.Sprintf("remote_queue.%s.endpoint", provider), queue.Spec.SQS.Endpoint}, - {fmt.Sprintf("remote_queue.%s.large_message_store.endpoint", provider), os.Spec.S3.Endpoint}, - {fmt.Sprintf("remote_queue.%s.large_message_store.path", provider), "s3://" + os.Spec.S3.Path}, - {fmt.Sprintf("remote_queue.%s.dead_letter_queue.name", provider), queue.Spec.SQS.DLQ}, - {fmt.Sprintf("remote_queue.%s.encoding_format", provider), "s2s"}, - {fmt.Sprintf("remote_queue.%s.max_count.max_retries_per_part", provider), "4"}, - {fmt.Sprintf("remote_queue.%s.retry_policy", provider), "max_count"}, - {fmt.Sprintf("remote_queue.%s.send_interval", provider), "5s"}, - {fmt.Sprintf("remote_queue.%s.access_key", provider), key}, - {fmt.Sprintf("remote_queue.%s.secret_key", provider), secret}, - }, queueInputs) - - assert.Equal(t, 6, len(pipelineInputs)) - assert.Equal(t, [][]string{ - {"pipeline:remotequeueruleset", "disabled", "false"}, - {"pipeline:ruleset", "disabled", "true"}, - {"pipeline:remotequeuetyping", "disabled", "false"}, - {"pipeline:remotequeueoutput", "disabled", "false"}, - {"pipeline:typing", "disabled", "true"}, - {"pipeline:indexerPipe", "disabled", "true"}, - }, pipelineInputs) -} - -func TestGetQueueAndPipelineInputsForIngestorConfFilesSQSCP(t *testing.T) { - provider := "sqs_smartbus_cp" - - queue := enterpriseApi.Queue{ - TypeMeta: metav1.TypeMeta{ - Kind: "Queue", - APIVersion: "enterprise.splunk.com/v4", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "queue", - }, - Spec: enterpriseApi.QueueSpec{ - Provider: "sqs_cp", - SQS: enterpriseApi.SQSSpec{ - Name: "test-queue", - AuthRegion: "us-west-2", - Endpoint: "https://sqs.us-west-2.amazonaws.com", - DLQ: "sqs-dlq-test", - VolList: []enterpriseApi.VolumeSpec{ - {SecretRef: "secret"}, - }, - }, - }, + // Deterministic + if computeIngestorConfChecksum("outputs", "defaultmode") != checksum { + t.Error("checksum is not deterministic") } - os := enterpriseApi.ObjectStorage{ - TypeMeta: metav1.TypeMeta{ - Kind: "ObjectStorage", - APIVersion: "enterprise.splunk.com/v4", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "os", - }, - Spec: enterpriseApi.ObjectStorageSpec{ - Provider: "s3", - S3: enterpriseApi.S3Spec{ - Endpoint: "https://s3.us-west-2.amazonaws.com", - Path: "bucket/key", - }, - }, + // Sensitive to content change + if computeIngestorConfChecksum("outputs2", "defaultmode") == checksum { + t.Error("checksum did not change when outputs changed") } - - key := "key" - secret := "secret" - - queueInputs, pipelineInputs := getQueueAndPipelineInputsForIngestorConfFiles(&queue.Spec, &os.Spec, key, secret) - - assert.Equal(t, 12, len(queueInputs)) - assert.Equal(t, [][]string{ - {"remote_queue.type", provider}, - {fmt.Sprintf("remote_queue.%s.auth_region", provider), queue.Spec.SQS.AuthRegion}, - {fmt.Sprintf("remote_queue.%s.endpoint", provider), queue.Spec.SQS.Endpoint}, - {fmt.Sprintf("remote_queue.%s.large_message_store.endpoint", provider), os.Spec.S3.Endpoint}, - {fmt.Sprintf("remote_queue.%s.large_message_store.path", provider), "s3://" + os.Spec.S3.Path}, - {fmt.Sprintf("remote_queue.%s.dead_letter_queue.name", provider), queue.Spec.SQS.DLQ}, - {fmt.Sprintf("remote_queue.%s.encoding_format", provider), "s2s"}, - {fmt.Sprintf("remote_queue.%s.max_count.max_retries_per_part", provider), "4"}, - {fmt.Sprintf("remote_queue.%s.retry_policy", provider), "max_count"}, - {fmt.Sprintf("remote_queue.%s.send_interval", provider), "5s"}, - {fmt.Sprintf("remote_queue.%s.access_key", provider), key}, - {fmt.Sprintf("remote_queue.%s.secret_key", provider), secret}, - }, queueInputs) - - assert.Equal(t, 6, len(pipelineInputs)) - assert.Equal(t, [][]string{ - {"pipeline:remotequeueruleset", "disabled", "false"}, - {"pipeline:ruleset", "disabled", "true"}, - {"pipeline:remotequeuetyping", "disabled", "false"}, - {"pipeline:remotequeueoutput", "disabled", "false"}, - {"pipeline:typing", "disabled", "true"}, - {"pipeline:indexerPipe", "disabled", "true"}, - }, pipelineInputs) } -func TestUpdateIngestorConfFiles(t *testing.T) { - c := spltest.NewMockClient() - ctx := context.TODO() - - // Object definitions - provider := "sqs_smartbus" - - accessKey := "accessKey" - secretKey := "secretKey" - - queue := &enterpriseApi.Queue{ - TypeMeta: metav1.TypeMeta{ - Kind: "Queue", - APIVersion: "enterprise.splunk.com/v4", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "queue", - }, - Spec: enterpriseApi.QueueSpec{ - Provider: "sqs", - SQS: enterpriseApi.SQSSpec{ - Name: "test-queue", - AuthRegion: "us-west-2", - Endpoint: "https://sqs.us-west-2.amazonaws.com", - DLQ: "sqs-dlq-test", - }, - }, - } - - os := &enterpriseApi.ObjectStorage{ - TypeMeta: metav1.TypeMeta{ - Kind: "ObjectStorage", - APIVersion: "enterprise.splunk.com/v4", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "os", - }, - Spec: enterpriseApi.ObjectStorageSpec{ - Provider: "s3", - S3: enterpriseApi.S3Spec{ - Endpoint: "https://s3.us-west-2.amazonaws.com", - Path: "bucket/key", - }, - }, - } - - cr := &enterpriseApi.IngestorCluster{ - TypeMeta: metav1.TypeMeta{ - Kind: "IngestorCluster", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "test", - Namespace: "test", - }, - Spec: enterpriseApi.IngestorClusterSpec{ - QueueRef: corev1.ObjectReference{ - Name: queue.Name, - }, - ObjectStorageRef: corev1.ObjectReference{ - Name: os.Name, - }, - }, - Status: enterpriseApi.IngestorClusterStatus{ - Replicas: 3, - ReadyReplicas: 3, - CredentialSecretVersion: "123", - }, - } - - pod0 := &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "splunk-test-ingestor-0", - Namespace: "test", - Labels: map[string]string{ - "app.kubernetes.io/instance": "splunk-test-ingestor", - }, - }, - Spec: corev1.PodSpec{ - Volumes: []corev1.Volume{ - { - Name: "dummy-volume", - VolumeSource: corev1.VolumeSource{ - EmptyDir: &corev1.EmptyDirVolumeSource{}, - }, - }, - { - Name: "mnt-splunk-secrets", - VolumeSource: corev1.VolumeSource{ - Secret: &corev1.SecretVolumeSource{ - SecretName: "test-secrets", - }, - }, - }, - }, - }, - Status: corev1.PodStatus{ - Phase: corev1.PodRunning, - ContainerStatuses: []corev1.ContainerStatus{ - {Ready: true}, - }, +func TestGenerateIngestorOutputsConf(t *testing.T) { + queue := enterpriseApi.QueueSpec{ + Provider: "sqs", + SQS: enterpriseApi.SQSSpec{ + Name: "test-queue", + AuthRegion: "us-west-2", + Endpoint: "https://sqs.us-west-2.amazonaws.com", + DLQ: "dlq", }, } - - pod1 := pod0.DeepCopy() - pod1.ObjectMeta.Name = "splunk-test-ingestor-1" - - pod2 := pod0.DeepCopy() - pod2.ObjectMeta.Name = "splunk-test-ingestor-2" - - c.Create(ctx, pod0) - c.Create(ctx, pod1) - c.Create(ctx, pod2) - - secret := &corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-secrets", - Namespace: "test", - }, - Data: map[string][]byte{ - "password": []byte("dummy"), + os := enterpriseApi.ObjectStorageSpec{ + Provider: "s3", + S3: enterpriseApi.S3Spec{ + Endpoint: "https://s3.amazonaws.com", + Path: "bucket/key", }, } - // Negative test case: secret not found - mgr := &ingestorClusterPodManager{} - - err := mgr.updateIngestorConfFiles(ctx, cr, &queue.Spec, &os.Spec, accessKey, secretKey, c) - assert.NotNil(t, err) - - // Mock secret - c.Create(ctx, secret) - - mockHTTPClient := &spltest.MockHTTPClient{} - - // Negative test case: failure in creating remote queue stanza - mgr = newTestIngestorQueuePipelineManager(mockHTTPClient) - - err = mgr.updateIngestorConfFiles(ctx, cr, &queue.Spec, &os.Spec, accessKey, secretKey, c) - assert.NotNil(t, err) - - // outputs.conf - propertyKVList := [][]string{ - {fmt.Sprintf("remote_queue.%s.encoding_format", provider), "s2s"}, - {fmt.Sprintf("remote_queue.%s.auth_region", provider), queue.Spec.SQS.AuthRegion}, - {fmt.Sprintf("remote_queue.%s.endpoint", provider), queue.Spec.SQS.Endpoint}, - {fmt.Sprintf("remote_queue.%s.large_message_store.endpoint", provider), os.Spec.S3.Endpoint}, - {fmt.Sprintf("remote_queue.%s.large_message_store.path", provider), os.Spec.S3.Path}, - {fmt.Sprintf("remote_queue.%s.dead_letter_queue.name", provider), queue.Spec.SQS.DLQ}, - {fmt.Sprintf("remote_queue.max_count.%s.max_retries_per_part", provider), "4"}, - {fmt.Sprintf("remote_queue.%s.retry_policy", provider), "max_count"}, - {fmt.Sprintf("remote_queue.%s.send_interval", provider), "5s"}, - } - - body := buildFormBody(propertyKVList) - addRemoteQueueHandlersForIngestor(mockHTTPClient, cr, &queue.Spec, "conf-outputs", body) + // IRSA: no credentials embedded + conf := generateIngestorOutputsConf(&queue, &os, "", "") + assert.Contains(t, conf, "[remote_queue:test-queue]") + assert.NotContains(t, conf, "access_key") - // Negative test case: failure in creating remote queue stanza - mgr = newTestIngestorQueuePipelineManager(mockHTTPClient) - - err = mgr.updateIngestorConfFiles(ctx, cr, &queue.Spec, &os.Spec, accessKey, secretKey, c) - assert.NotNil(t, err) - - // default-mode.conf - propertyKVList = [][]string{ - {"pipeline:remotequeueruleset", "disabled", "false"}, - {"pipeline:ruleset", "disabled", "true"}, - {"pipeline:remotequeuetyping", "disabled", "false"}, - {"pipeline:remotequeueoutput", "disabled", "false"}, - {"pipeline:typing", "disabled", "true"}, - {"pipeline:indexerPipe", "disabled", "true"}, - } - - for i := 0; i < int(cr.Status.ReadyReplicas); i++ { - podName := fmt.Sprintf("splunk-test-ingestor-%d", i) - baseURL := fmt.Sprintf("https://%s.splunk-%s-ingestor-headless.%s.svc.cluster.local:8089/servicesNS/nobody/system/configs/conf-default-mode", podName, cr.GetName(), cr.GetNamespace()) - - for _, field := range propertyKVList { - req, _ := http.NewRequest("POST", baseURL, strings.NewReader(fmt.Sprintf("name=%s", field[0]))) - mockHTTPClient.AddHandler(req, 200, "", nil) + // Static creds: credentials embedded + conf = generateIngestorOutputsConf(&queue, &os, "AKID", "secret") + assert.Contains(t, conf, "access_key") +} - updateURL := fmt.Sprintf("%s/%s", baseURL, field[0]) - req, _ = http.NewRequest("POST", updateURL, strings.NewReader(fmt.Sprintf("%s=%s", field[1], field[2]))) - mockHTTPClient.AddHandler(req, 200, "", nil) - } +func TestGenerateIngestorDefaultModeConf(t *testing.T) { + conf := generateIngestorDefaultModeConf() + for _, stanza := range []string{ + "pipeline:remotequeueruleset", + "pipeline:ruleset", + "pipeline:remotequeuetyping", + "pipeline:remotequeueoutput", + "pipeline:typing", + "pipeline:indexerPipe", + } { + assert.Contains(t, conf, stanza) } - - mgr = newTestIngestorQueuePipelineManager(mockHTTPClient) - - err = mgr.updateIngestorConfFiles(ctx, cr, &queue.Spec, &os.Spec, accessKey, secretKey, c) - assert.Nil(t, err) } -func addRemoteQueueHandlersForIngestor(mockHTTPClient *spltest.MockHTTPClient, cr *enterpriseApi.IngestorCluster, queue *enterpriseApi.QueueSpec, confName, body string) { - for i := 0; i < int(cr.Status.ReadyReplicas); i++ { - podName := fmt.Sprintf("splunk-%s-ingestor-%d", cr.GetName(), i) - baseURL := fmt.Sprintf( - "https://%s.splunk-%s-ingestor-headless.%s.svc.cluster.local:8089/servicesNS/nobody/system/configs/%s", - podName, cr.GetName(), cr.GetNamespace(), confName, - ) - - createReqBody := fmt.Sprintf("name=%s", fmt.Sprintf("remote_queue:%s", queue.SQS.Name)) - reqCreate, _ := http.NewRequest("POST", baseURL, strings.NewReader(createReqBody)) - mockHTTPClient.AddHandler(reqCreate, 200, "", nil) - - updateURL := fmt.Sprintf("%s/%s", baseURL, fmt.Sprintf("remote_queue:%s", queue.SQS.Name)) - reqUpdate, _ := http.NewRequest("POST", updateURL, strings.NewReader(body)) - mockHTTPClient.AddHandler(reqUpdate, 200, "", nil) - } +func TestGenerateIngestorAppConf(t *testing.T) { + conf := generateQueueConfigAppConf("Splunk Operator Ingestor Queue Config") + assert.Contains(t, conf, "[install]") + assert.Contains(t, conf, "state = enabled") + assert.Contains(t, conf, "[package]") + assert.Contains(t, conf, "[ui]") } -func newTestIngestorQueuePipelineManager(mockHTTPClient *spltest.MockHTTPClient) *ingestorClusterPodManager { - newSplunkClientForQueuePipeline := func(uri, user, pass string) *splclient.SplunkClient { - return &splclient.SplunkClient{ - ManagementURI: uri, - Username: user, - Password: pass, - Client: mockHTTPClient, - } - } - return &ingestorClusterPodManager{ - newSplunkClient: newSplunkClientForQueuePipeline, - } +func TestGenerateIngestorLocalMeta(t *testing.T) { + conf := generateQueueConfigLocalMeta() + // install_source_checksum is not a valid local.meta field; it was removed to prevent parse errors. + assert.NotContains(t, conf, "install_source_checksum") + assert.Contains(t, conf, "export = system") + assert.Contains(t, conf, "access = read : [ * ], write : [ admin ]") } diff --git a/pkg/splunk/enterprise/names.go b/pkg/splunk/enterprise/names.go index 623f361f8..c0eb50ff8 100644 --- a/pkg/splunk/enterprise/names.go +++ b/pkg/splunk/enterprise/names.go @@ -134,6 +134,29 @@ const ( // setSymbolicLinkCmanager setSymbolicLinkCmanager = "ln -sfn /mnt/splunk-operator/local/indexes.conf /opt/splunk/etc/manager-apps/splunk-operator/local/indexes.conf && ln -sfn /mnt/splunk-operator/local/server.conf /opt/splunk/etc/manager-apps/splunk-operator/local/server.conf" + // cmQueueConfigMapTemplateStr is the ConfigMap name pattern: splunk--clustermanager-queue-config + cmQueueConfigMapTemplateStr = "splunk-%s-clustermanager-queue-config" + + // cmQueueConfigVolName is the volume name for the CM queue config ConfigMap mount. + cmQueueConfigVolName = "mnt-splunk-cm-queue-config" + + // cmQueueConfigMountPath is where the queue config ConfigMap is mounted on the CM pod. + cmQueueConfigMountPath = "/mnt/splunk-cm-queue-config" + + // commandForCMQueueConfig is used in a dedicated init container on the ClusterManager pod + // to symlink queue config files (outputs.conf, inputs.conf, default-mode.conf) from the + // queue config ConfigMap mount into manager-apps/splunk-operator/local/. + // This init container runs independently of the smartstore init container. + commandForCMQueueConfig = "mkdir -p " + splcommon.OperatorClusterManagerAppsLocal + + " && ln -sfn " + cmQueueConfigMountPath + "/outputs.conf " + splcommon.OperatorClusterManagerAppsLocalOutputsConf + + " && ln -sfn " + cmQueueConfigMountPath + "/inputs.conf " + splcommon.OperatorClusterManagerAppsLocalInputsConf + + " && ln -sfn " + cmQueueConfigMountPath + "/default-mode.conf " + splcommon.OperatorClusterManagerAppsLocalDefaultModeConf + + // setSymbolicLinkCmanagerQueueConfig resets queue config symlinks on the CM pod after a bundle push. + setSymbolicLinkCmanagerQueueConfig = "ln -sfn " + cmQueueConfigMountPath + "/outputs.conf " + splcommon.OperatorClusterManagerAppsLocalOutputsConf + + " && ln -sfn " + cmQueueConfigMountPath + "/inputs.conf " + splcommon.OperatorClusterManagerAppsLocalInputsConf + + " && ln -sfn " + cmQueueConfigMountPath + "/default-mode.conf " + splcommon.OperatorClusterManagerAppsLocalDefaultModeConf + // configToken used to track if the config is reflecting on Pod or not configToken = "conftoken" @@ -218,6 +241,30 @@ access = read : [ * ], write : [ admin ] telLicenseInfoKey = "license_info" managerConfigMapTemplateStr = "%smanager-config" + + // ingestorQueueConfigAppName is the Splunk app directory name for ingestor queue config + ingestorQueueConfigAppName = "100-sok-ingestorcluster" + + // ingestorQueueConfigTemplateStr is the ConfigMap name pattern: splunk--ingestor-queue-config + ingestorQueueConfigTemplateStr = "splunk-%s-ingestor-queue-config" + + // ingestorQueueConfigRevAnnotation is the pod annotation key set to ConfigMap ResourceVersion. + // When content changes (credentials rotate, topology changes), RV increments → annotation changes + // → Restart EPIC detects change and restarts pods (PDB-aware). reload.outputs.remote_queue=never. + ingestorQueueConfigRevAnnotation = "ingestorQueueConfigRev" + + // ingestorQueueConfigMountPath is the ConfigMap mount path inside the pod + ingestorQueueConfigMountPath = "/mnt/splunk-queue-config" + + // commandForIngestorQueueConfig is the init container shell command. + // Creates app directory, symlinks conf files from ConfigMap mount, copies local.meta + // (NOT symlinked — Splunk replaces metadata symlinks with regular files when writing stanzas). + commandForIngestorQueueConfig = "mkdir -p /opt/splk/etc/apps/" + ingestorQueueConfigAppName + "/local && " + + "mkdir -p /opt/splk/etc/apps/" + ingestorQueueConfigAppName + "/metadata && " + + "ln -sfn " + ingestorQueueConfigMountPath + "/app.conf /opt/splk/etc/apps/" + ingestorQueueConfigAppName + "/local/app.conf && " + + "ln -sfn " + ingestorQueueConfigMountPath + "/outputs.conf /opt/splk/etc/apps/" + ingestorQueueConfigAppName + "/local/outputs.conf && " + + "ln -sfn " + ingestorQueueConfigMountPath + "/default-mode.conf /opt/splk/etc/apps/" + ingestorQueueConfigAppName + "/local/default-mode.conf && " + + "cp " + ingestorQueueConfigMountPath + "/local.meta /opt/splk/etc/apps/" + ingestorQueueConfigAppName + "/metadata/local.meta" ) const ( @@ -383,3 +430,13 @@ func GetTelemetryConfigMapName(namePrefix string) string { func GetManagerConfigMapName(namePrefix string) string { return fmt.Sprintf(managerConfigMapTemplateStr, namePrefix) } + +// GetIngestorQueueConfigMapName returns the name of the ingestor queue config ConfigMap +func GetIngestorQueueConfigMapName(crName string) string { + return fmt.Sprintf(ingestorQueueConfigTemplateStr, crName) +} + +// GetCMQueueConfigMapName returns the name of the ClusterManager queue config ConfigMap +func GetCMQueueConfigMapName(cmName string) string { + return fmt.Sprintf(cmQueueConfigMapTemplateStr, cmName) +} diff --git a/pkg/splunk/enterprise/testdata/fixtures/statefulset_ingestor.json b/pkg/splunk/enterprise/testdata/fixtures/statefulset_ingestor.json index 933f26f73..075e974b9 100644 --- a/pkg/splunk/enterprise/testdata/fixtures/statefulset_ingestor.json +++ b/pkg/splunk/enterprise/testdata/fixtures/statefulset_ingestor.json @@ -1,207 +1 @@ -{ - "kind": "StatefulSet", - "apiVersion": "apps/v1", - "metadata": { - "name": "splunk-test-ingestor", - "namespace": "test", - "creationTimestamp": null, - "labels": { - "app.kubernetes.io/component": "ingestor", - "app.kubernetes.io/instance": "splunk-test-ingestor", - "app.kubernetes.io/managed-by": "splunk-operator", - "app.kubernetes.io/name": "ingestor", - "app.kubernetes.io/part-of": "splunk-test-ingestor" - }, - "ownerReferences": [ - { - "apiVersion": "", - "kind": "IngestorCluster", - "name": "test", - "uid": "", - "controller": true - } - ] - }, - "spec": { - "replicas": 1, - "selector": { - "matchLabels": { - "app.kubernetes.io/component": "ingestor", - "app.kubernetes.io/instance": "splunk-test-ingestor", - "app.kubernetes.io/managed-by": "splunk-operator", - "app.kubernetes.io/name": "ingestor", - "app.kubernetes.io/part-of": "splunk-test-ingestor" - } - }, - "template": { - "metadata": { - "creationTimestamp": null, - "labels": { - "app.kubernetes.io/component": "ingestor", - "app.kubernetes.io/instance": "splunk-test-ingestor", - "app.kubernetes.io/managed-by": "splunk-operator", - "app.kubernetes.io/name": "ingestor", - "app.kubernetes.io/part-of": "splunk-test-ingestor" - }, - "annotations": { - "traffic.sidecar.istio.io/excludeOutboundPorts": "8089,8191,9997", - "traffic.sidecar.istio.io/includeInboundPorts": "8000,8088" - } - }, - "spec": { - "volumes": [ - { - "name": "splunk-test-probe-configmap", - "configMap": { - "name": "splunk-test-probe-configmap", - "defaultMode": 365 - } - }, - { - "name": "mnt-splunk-secrets", - "secret": { - "secretName": "splunk-test-ingestor-secret-v1", - "defaultMode": 420 - } - } - ], - "containers": [ - { - "name": "splunk", - "image": "splunk/splunk", - "ports": [ - { "name": "http-splunkweb", "containerPort": 8000, "protocol": "TCP" }, - { "name": "http-hec", "containerPort": 8088, "protocol": "TCP" }, - { "name": "https-splunkd", "containerPort": 8089, "protocol": "TCP" }, - { "name": "tcp-s2s", "containerPort": 9997, "protocol": "TCP" }, - { "name": "user-defined", "containerPort": 32000, "protocol": "UDP" } - ], - "env": [ - { "name": "SPLUNK_HOME", "value": "/opt/splunk" }, - { "name": "SPLUNK_START_ARGS", "value": "--accept-license" }, - { "name": "SPLUNK_DEFAULTS_URL", "value": "/mnt/splunk-secrets/default.yml" }, - { "name": "SPLUNK_HOME_OWNERSHIP_ENFORCEMENT", "value": "false" }, - { "name": "SPLUNK_ROLE", "value": "splunk_ingestor" }, - { "name": "SPLUNK_DECLARATIVE_ADMIN_PASSWORD", "value": "true" }, - { "name": "SPLUNK_OPERATOR_K8_LIVENESS_DRIVER_FILE_PATH", "value": "/tmp/splunk_operator_k8s/probes/k8_liveness_driver.sh" }, - { "name": "SPLUNK_GENERAL_TERMS", "value": "--accept-sgt-current-at-splunk-com" }, - { "name": "SPLUNK_SKIP_CLUSTER_BUNDLE_PUSH", "value": "true" } - ], - "resources": { - "limits": { "cpu": "4", "memory": "8Gi" }, - "requests": { "cpu": "100m", "memory": "512Mi" } - }, - "volumeMounts": [ - { "name": "pvc-etc", "mountPath": "/opt/splunk/etc" }, - { "name": "pvc-var", "mountPath": "/opt/splunk/var" }, - { "name": "splunk-test-probe-configmap", "mountPath": "/mnt/probes" }, - { "name": "mnt-splunk-secrets", "mountPath": "/mnt/splunk-secrets" } - ], - "livenessProbe": { - "exec": { "command": ["/mnt/probes/livenessProbe.sh"] }, - "initialDelaySeconds": 30, - "timeoutSeconds": 30, - "periodSeconds": 30, - "failureThreshold": 3 - }, - "readinessProbe": { - "exec": { "command": ["/mnt/probes/readinessProbe.sh"] }, - "initialDelaySeconds": 10, - "timeoutSeconds": 5, - "periodSeconds": 5, - "failureThreshold": 3 - }, - "startupProbe": { - "exec": { "command": ["/mnt/probes/startupProbe.sh"] }, - "initialDelaySeconds": 40, - "timeoutSeconds": 30, - "periodSeconds": 30, - "failureThreshold": 12 - }, - "imagePullPolicy": "IfNotPresent", - "securityContext": { - "capabilities": { "add": ["NET_BIND_SERVICE"], "drop": ["ALL"] }, - "privileged": false, - "runAsUser": 41812, - "runAsNonRoot": true, - "allowPrivilegeEscalation": false, - "seccompProfile": { "type": "RuntimeDefault" } - } - } - ], - "securityContext": { - "runAsUser": 41812, - "runAsNonRoot": true, - "fsGroup": 41812, - "fsGroupChangePolicy": "OnRootMismatch" - }, - "affinity": { - "podAntiAffinity": { - "preferredDuringSchedulingIgnoredDuringExecution": [ - { - "weight": 100, - "podAffinityTerm": { - "labelSelector": { - "matchExpressions": [ - { - "key": "app.kubernetes.io/instance", - "operator": "In", - "values": ["splunk-test-ingestor"] - } - ] - }, - "topologyKey": "kubernetes.io/hostname" - } - } - ] - } - }, - "schedulerName": "default-scheduler" - } - }, - "volumeClaimTemplates": [ - { - "metadata": { - "name": "pvc-etc", - "namespace": "test", - "creationTimestamp": null, - "labels": { - "app.kubernetes.io/component": "ingestor", - "app.kubernetes.io/instance": "splunk-test-ingestor", - "app.kubernetes.io/managed-by": "splunk-operator", - "app.kubernetes.io/name": "ingestor", - "app.kubernetes.io/part-of": "splunk-test-ingestor" - } - }, - "spec": { - "accessModes": ["ReadWriteOnce"], - "resources": { "requests": { "storage": "10Gi" } } - }, - "status": {} - }, - { - "metadata": { - "name": "pvc-var", - "namespace": "test", - "creationTimestamp": null, - "labels": { - "app.kubernetes.io/component": "ingestor", - "app.kubernetes.io/instance": "splunk-test-ingestor", - "app.kubernetes.io/managed-by": "splunk-operator", - "app.kubernetes.io/name": "ingestor", - "app.kubernetes.io/part-of": "splunk-test-ingestor" - } - }, - "spec": { - "accessModes": ["ReadWriteOnce"], - "resources": { "requests": { "storage": "100Gi" } } - }, - "status": {} - } - ], - "serviceName": "splunk-test-ingestor-headless", - "podManagementPolicy": "Parallel", - "updateStrategy": { "type": "OnDelete" } - }, - "status": { "replicas": 0, "availableReplicas": 0 } -} \ No newline at end of file +{"kind":"StatefulSet","apiVersion":"apps/v1","metadata":{"name":"splunk-test-ingestor","namespace":"test","creationTimestamp":null,"labels":{"app.kubernetes.io/component":"ingestor","app.kubernetes.io/instance":"splunk-test-ingestor","app.kubernetes.io/managed-by":"splunk-operator","app.kubernetes.io/name":"ingestor","app.kubernetes.io/part-of":"splunk-test-ingestor"},"ownerReferences":[{"apiVersion":"","kind":"IngestorCluster","name":"test","uid":"","controller":true}]},"spec":{"replicas":1,"selector":{"matchLabels":{"app.kubernetes.io/component":"ingestor","app.kubernetes.io/instance":"splunk-test-ingestor","app.kubernetes.io/managed-by":"splunk-operator","app.kubernetes.io/name":"ingestor","app.kubernetes.io/part-of":"splunk-test-ingestor"}},"template":{"metadata":{"creationTimestamp":null,"labels":{"app.kubernetes.io/component":"ingestor","app.kubernetes.io/instance":"splunk-test-ingestor","app.kubernetes.io/managed-by":"splunk-operator","app.kubernetes.io/name":"ingestor","app.kubernetes.io/part-of":"splunk-test-ingestor"},"annotations":{"traffic.sidecar.istio.io/excludeOutboundPorts":"8089,8191,9997","traffic.sidecar.istio.io/includeInboundPorts":"8000,8088"}},"spec":{"volumes":[{"name":"splunk-test-probe-configmap","configMap":{"name":"splunk-test-probe-configmap","defaultMode":365}},{"name":"mnt-splunk-secrets","secret":{"secretName":"splunk-test-ingestor-secret-v1","defaultMode":420}},{"name":"mnt-splunk-queue-config","configMap":{"name":"splunk-test-ingestor-queue-config","defaultMode":420}}],"initContainers":[{"name":"init-ingestor-queue-config","image":"splunk/splunk","command":["bash","-c","mkdir -p /opt/splk/etc/apps/100-sok-ingestorcluster/local \u0026\u0026 mkdir -p /opt/splk/etc/apps/100-sok-ingestorcluster/metadata \u0026\u0026 ln -sfn /mnt/splunk-queue-config/app.conf /opt/splk/etc/apps/100-sok-ingestorcluster/local/app.conf \u0026\u0026 ln -sfn /mnt/splunk-queue-config/outputs.conf /opt/splk/etc/apps/100-sok-ingestorcluster/local/outputs.conf \u0026\u0026 ln -sfn /mnt/splunk-queue-config/default-mode.conf /opt/splk/etc/apps/100-sok-ingestorcluster/local/default-mode.conf \u0026\u0026 cp /mnt/splunk-queue-config/local.meta /opt/splk/etc/apps/100-sok-ingestorcluster/metadata/local.meta"],"resources":{"limits":{"cpu":"1","memory":"512Mi"},"requests":{"cpu":"250m","memory":"128Mi"}},"volumeMounts":[{"name":"pvc-etc","mountPath":"/opt/splk/etc"},{"name":"mnt-splunk-queue-config","mountPath":"/mnt/splunk-queue-config"}],"imagePullPolicy":"IfNotPresent","securityContext":{"capabilities":{"add":["NET_BIND_SERVICE"],"drop":["ALL"]},"privileged":false,"runAsUser":41812,"runAsNonRoot":true,"allowPrivilegeEscalation":false,"seccompProfile":{"type":"RuntimeDefault"}}}],"containers":[{"name":"splunk","image":"splunk/splunk","ports":[{"name":"http-splunkweb","containerPort":8000,"protocol":"TCP"},{"name":"http-hec","containerPort":8088,"protocol":"TCP"},{"name":"https-splunkd","containerPort":8089,"protocol":"TCP"},{"name":"tcp-s2s","containerPort":9997,"protocol":"TCP"},{"name":"user-defined","containerPort":32000,"protocol":"UDP"}],"env":[{"name":"SPLUNK_HOME","value":"/opt/splunk"},{"name":"SPLUNK_START_ARGS","value":"--accept-license"},{"name":"SPLUNK_DEFAULTS_URL","value":"/mnt/splunk-secrets/default.yml"},{"name":"SPLUNK_HOME_OWNERSHIP_ENFORCEMENT","value":"false"},{"name":"SPLUNK_ROLE","value":"splunk_ingestor"},{"name":"SPLUNK_DECLARATIVE_ADMIN_PASSWORD","value":"true"},{"name":"SPLUNK_OPERATOR_K8_LIVENESS_DRIVER_FILE_PATH","value":"/tmp/splunk_operator_k8s/probes/k8_liveness_driver.sh"},{"name":"SPLUNK_GENERAL_TERMS","value":"--accept-sgt-current-at-splunk-com"},{"name":"SPLUNK_SKIP_CLUSTER_BUNDLE_PUSH","value":"true"}],"resources":{"limits":{"cpu":"4","memory":"8Gi"},"requests":{"cpu":"100m","memory":"512Mi"}},"volumeMounts":[{"name":"pvc-etc","mountPath":"/opt/splunk/etc"},{"name":"pvc-var","mountPath":"/opt/splunk/var"},{"name":"splunk-test-probe-configmap","mountPath":"/mnt/probes"},{"name":"mnt-splunk-secrets","mountPath":"/mnt/splunk-secrets"},{"name":"mnt-splunk-queue-config","readOnly":true,"mountPath":"/mnt/splunk-queue-config"}],"livenessProbe":{"exec":{"command":["/mnt/probes/livenessProbe.sh"]},"initialDelaySeconds":30,"timeoutSeconds":30,"periodSeconds":30,"failureThreshold":3},"readinessProbe":{"exec":{"command":["/mnt/probes/readinessProbe.sh"]},"initialDelaySeconds":10,"timeoutSeconds":5,"periodSeconds":5,"failureThreshold":3},"startupProbe":{"exec":{"command":["/mnt/probes/startupProbe.sh"]},"initialDelaySeconds":40,"timeoutSeconds":30,"periodSeconds":30,"failureThreshold":12},"imagePullPolicy":"IfNotPresent","securityContext":{"capabilities":{"add":["NET_BIND_SERVICE"],"drop":["ALL"]},"privileged":false,"runAsUser":41812,"runAsNonRoot":true,"allowPrivilegeEscalation":false,"seccompProfile":{"type":"RuntimeDefault"}}}],"securityContext":{"runAsUser":41812,"runAsNonRoot":true,"fsGroup":41812,"fsGroupChangePolicy":"OnRootMismatch"},"affinity":{"podAntiAffinity":{"preferredDuringSchedulingIgnoredDuringExecution":[{"weight":100,"podAffinityTerm":{"labelSelector":{"matchExpressions":[{"key":"app.kubernetes.io/instance","operator":"In","values":["splunk-test-ingestor"]}]},"topologyKey":"kubernetes.io/hostname"}}]}},"schedulerName":"default-scheduler"}},"volumeClaimTemplates":[{"metadata":{"name":"pvc-etc","namespace":"test","creationTimestamp":null,"labels":{"app.kubernetes.io/component":"ingestor","app.kubernetes.io/instance":"splunk-test-ingestor","app.kubernetes.io/managed-by":"splunk-operator","app.kubernetes.io/name":"ingestor","app.kubernetes.io/part-of":"splunk-test-ingestor"}},"spec":{"accessModes":["ReadWriteOnce"],"resources":{"requests":{"storage":"10Gi"}}},"status":{}},{"metadata":{"name":"pvc-var","namespace":"test","creationTimestamp":null,"labels":{"app.kubernetes.io/component":"ingestor","app.kubernetes.io/instance":"splunk-test-ingestor","app.kubernetes.io/managed-by":"splunk-operator","app.kubernetes.io/name":"ingestor","app.kubernetes.io/part-of":"splunk-test-ingestor"}},"spec":{"accessModes":["ReadWriteOnce"],"resources":{"requests":{"storage":"100Gi"}}},"status":{}}],"serviceName":"splunk-test-ingestor-headless","podManagementPolicy":"Parallel","updateStrategy":{"type":"OnDelete"}},"status":{"replicas":0,"availableReplicas":0}} \ No newline at end of file diff --git a/pkg/splunk/enterprise/testdata/fixtures/statefulset_ingestor_with_extraenv.json b/pkg/splunk/enterprise/testdata/fixtures/statefulset_ingestor_with_extraenv.json index 581598ecf..178ee6ec4 100644 --- a/pkg/splunk/enterprise/testdata/fixtures/statefulset_ingestor_with_extraenv.json +++ b/pkg/splunk/enterprise/testdata/fixtures/statefulset_ingestor_with_extraenv.json @@ -1,209 +1 @@ -{ - "kind": "StatefulSet", - "apiVersion": "apps/v1", - "metadata": { - "name": "splunk-test-ingestor", - "namespace": "test", - "creationTimestamp": null, - "labels": { - "app.kubernetes.io/component": "ingestor", - "app.kubernetes.io/instance": "splunk-test-ingestor", - "app.kubernetes.io/managed-by": "splunk-operator", - "app.kubernetes.io/name": "ingestor", - "app.kubernetes.io/part-of": "splunk-test-ingestor" - }, - "ownerReferences": [ - { - "apiVersion": "", - "kind": "IngestorCluster", - "name": "test", - "uid": "", - "controller": true - } - ] - }, - "spec": { - "replicas": 1, - "selector": { - "matchLabels": { - "app.kubernetes.io/component": "ingestor", - "app.kubernetes.io/instance": "splunk-test-ingestor", - "app.kubernetes.io/managed-by": "splunk-operator", - "app.kubernetes.io/name": "ingestor", - "app.kubernetes.io/part-of": "splunk-test-ingestor" - } - }, - "template": { - "metadata": { - "creationTimestamp": null, - "labels": { - "app.kubernetes.io/component": "ingestor", - "app.kubernetes.io/instance": "splunk-test-ingestor", - "app.kubernetes.io/managed-by": "splunk-operator", - "app.kubernetes.io/name": "ingestor", - "app.kubernetes.io/part-of": "splunk-test-ingestor" - }, - "annotations": { - "traffic.sidecar.istio.io/excludeOutboundPorts": "8089,8191,9997", - "traffic.sidecar.istio.io/includeInboundPorts": "8000,8088" - } - }, - "spec": { - "volumes": [ - { - "name": "splunk-test-probe-configmap", - "configMap": { - "name": "splunk-test-probe-configmap", - "defaultMode": 365 - } - }, - { - "name": "mnt-splunk-secrets", - "secret": { - "secretName": "splunk-test-ingestor-secret-v1", - "defaultMode": 420 - } - } - ], - "containers": [ - { - "name": "splunk", - "image": "splunk/splunk", - "ports": [ - { "name": "http-splunkweb", "containerPort": 8000, "protocol": "TCP" }, - { "name": "http-hec", "containerPort": 8088, "protocol": "TCP" }, - { "name": "https-splunkd", "containerPort": 8089, "protocol": "TCP" }, - { "name": "tcp-s2s", "containerPort": 9997, "protocol": "TCP" }, - { "name": "user-defined", "containerPort": 32000, "protocol": "UDP" } - ], - "env": [ - { "name": "TEST_ENV_VAR", "value": "test_value" }, - { "name": "SPLUNK_HOME", "value": "/opt/splunk" }, - { "name": "SPLUNK_START_ARGS", "value": "--accept-license" }, - { "name": "SPLUNK_DEFAULTS_URL", "value": "/mnt/splunk-secrets/default.yml" }, - { "name": "SPLUNK_HOME_OWNERSHIP_ENFORCEMENT", "value": "false" }, - { "name": "SPLUNK_ROLE", "value": "splunk_ingestor" }, - { "name": "SPLUNK_DECLARATIVE_ADMIN_PASSWORD", "value": "true" }, - { "name": "SPLUNK_OPERATOR_K8_LIVENESS_DRIVER_FILE_PATH", "value": "/tmp/splunk_operator_k8s/probes/k8_liveness_driver.sh" }, - { "name": "SPLUNK_GENERAL_TERMS", "value": "--accept-sgt-current-at-splunk-com" }, - { "name": "SPLUNK_SKIP_CLUSTER_BUNDLE_PUSH", "value": "true" } - ], - "resources": { - "limits": { "cpu": "4", "memory": "8Gi" }, - "requests": { "cpu": "100m", "memory": "512Mi" } - }, - "volumeMounts": [ - { "name": "pvc-etc", "mountPath": "/opt/splunk/etc" }, - { "name": "pvc-var", "mountPath": "/opt/splunk/var" }, - { "name": "splunk-test-probe-configmap", "mountPath": "/mnt/probes" }, - { "name": "mnt-splunk-secrets", "mountPath": "/mnt/splunk-secrets" } - ], - "livenessProbe": { - "exec": { "command": ["/mnt/probes/livenessProbe.sh"] }, - "initialDelaySeconds": 30, - "timeoutSeconds": 30, - "periodSeconds": 30, - "failureThreshold": 3 - }, - "readinessProbe": { - "exec": { "command": ["/mnt/probes/readinessProbe.sh"] }, - "initialDelaySeconds": 10, - "timeoutSeconds": 5, - "periodSeconds": 5, - "failureThreshold": 3 - }, - "startupProbe": { - "exec": { "command": ["/mnt/probes/startupProbe.sh"] }, - "initialDelaySeconds": 40, - "timeoutSeconds": 30, - "periodSeconds": 30, - "failureThreshold": 12 - }, - "imagePullPolicy": "IfNotPresent", - "securityContext": { - "capabilities": { "add": ["NET_BIND_SERVICE"], "drop": ["ALL"] }, - "privileged": false, - "runAsUser": 41812, - "runAsNonRoot": true, - "allowPrivilegeEscalation": false, - "seccompProfile": { "type": "RuntimeDefault" } - } - } - ], - "serviceAccountName": "defaults", - "securityContext": { - "runAsUser": 41812, - "runAsNonRoot": true, - "fsGroup": 41812, - "fsGroupChangePolicy": "OnRootMismatch" - }, - "affinity": { - "podAntiAffinity": { - "preferredDuringSchedulingIgnoredDuringExecution": [ - { - "weight": 100, - "podAffinityTerm": { - "labelSelector": { - "matchExpressions": [ - { - "key": "app.kubernetes.io/instance", - "operator": "In", - "values": ["splunk-test-ingestor"] - } - ] - }, - "topologyKey": "kubernetes.io/hostname" - } - } - ] - } - }, - "schedulerName": "default-scheduler" - } - }, - "volumeClaimTemplates": [ - { - "metadata": { - "name": "pvc-etc", - "namespace": "test", - "creationTimestamp": null, - "labels": { - "app.kubernetes.io/component": "ingestor", - "app.kubernetes.io/instance": "splunk-test-ingestor", - "app.kubernetes.io/managed-by": "splunk-operator", - "app.kubernetes.io/name": "ingestor", - "app.kubernetes.io/part-of": "splunk-test-ingestor" - } - }, - "spec": { - "accessModes": ["ReadWriteOnce"], - "resources": { "requests": { "storage": "10Gi" } } - }, - "status": {} - }, - { - "metadata": { - "name": "pvc-var", - "namespace": "test", - "creationTimestamp": null, - "labels": { - "app.kubernetes.io/component": "ingestor", - "app.kubernetes.io/instance": "splunk-test-ingestor", - "app.kubernetes.io/managed-by": "splunk-operator", - "app.kubernetes.io/name": "ingestor", - "app.kubernetes.io/part-of": "splunk-test-ingestor" - } - }, - "spec": { - "accessModes": ["ReadWriteOnce"], - "resources": { "requests": { "storage": "100Gi" } } - }, - "status": {} - } - ], - "serviceName": "splunk-test-ingestor-headless", - "podManagementPolicy": "Parallel", - "updateStrategy": { "type": "OnDelete" } - }, - "status": { "replicas": 0, "availableReplicas": 0 } -} \ No newline at end of file +{"kind":"StatefulSet","apiVersion":"apps/v1","metadata":{"name":"splunk-test-ingestor","namespace":"test","creationTimestamp":null,"labels":{"app.kubernetes.io/component":"ingestor","app.kubernetes.io/instance":"splunk-test-ingestor","app.kubernetes.io/managed-by":"splunk-operator","app.kubernetes.io/name":"ingestor","app.kubernetes.io/part-of":"splunk-test-ingestor"},"ownerReferences":[{"apiVersion":"","kind":"IngestorCluster","name":"test","uid":"","controller":true}]},"spec":{"replicas":1,"selector":{"matchLabels":{"app.kubernetes.io/component":"ingestor","app.kubernetes.io/instance":"splunk-test-ingestor","app.kubernetes.io/managed-by":"splunk-operator","app.kubernetes.io/name":"ingestor","app.kubernetes.io/part-of":"splunk-test-ingestor"}},"template":{"metadata":{"creationTimestamp":null,"labels":{"app.kubernetes.io/component":"ingestor","app.kubernetes.io/instance":"splunk-test-ingestor","app.kubernetes.io/managed-by":"splunk-operator","app.kubernetes.io/name":"ingestor","app.kubernetes.io/part-of":"splunk-test-ingestor"},"annotations":{"traffic.sidecar.istio.io/excludeOutboundPorts":"8089,8191,9997","traffic.sidecar.istio.io/includeInboundPorts":"8000,8088"}},"spec":{"volumes":[{"name":"splunk-test-probe-configmap","configMap":{"name":"splunk-test-probe-configmap","defaultMode":365}},{"name":"mnt-splunk-secrets","secret":{"secretName":"splunk-test-ingestor-secret-v1","defaultMode":420}},{"name":"mnt-splunk-queue-config","configMap":{"name":"splunk-test-ingestor-queue-config","defaultMode":420}}],"initContainers":[{"name":"init-ingestor-queue-config","image":"splunk/splunk","command":["bash","-c","mkdir -p /opt/splk/etc/apps/100-sok-ingestorcluster/local \u0026\u0026 mkdir -p /opt/splk/etc/apps/100-sok-ingestorcluster/metadata \u0026\u0026 ln -sfn /mnt/splunk-queue-config/app.conf /opt/splk/etc/apps/100-sok-ingestorcluster/local/app.conf \u0026\u0026 ln -sfn /mnt/splunk-queue-config/outputs.conf /opt/splk/etc/apps/100-sok-ingestorcluster/local/outputs.conf \u0026\u0026 ln -sfn /mnt/splunk-queue-config/default-mode.conf /opt/splk/etc/apps/100-sok-ingestorcluster/local/default-mode.conf \u0026\u0026 cp /mnt/splunk-queue-config/local.meta /opt/splk/etc/apps/100-sok-ingestorcluster/metadata/local.meta"],"resources":{"limits":{"cpu":"1","memory":"512Mi"},"requests":{"cpu":"250m","memory":"128Mi"}},"volumeMounts":[{"name":"pvc-etc","mountPath":"/opt/splk/etc"},{"name":"mnt-splunk-queue-config","mountPath":"/mnt/splunk-queue-config"}],"imagePullPolicy":"IfNotPresent","securityContext":{"capabilities":{"add":["NET_BIND_SERVICE"],"drop":["ALL"]},"privileged":false,"runAsUser":41812,"runAsNonRoot":true,"allowPrivilegeEscalation":false,"seccompProfile":{"type":"RuntimeDefault"}}}],"containers":[{"name":"splunk","image":"splunk/splunk","ports":[{"name":"http-splunkweb","containerPort":8000,"protocol":"TCP"},{"name":"http-hec","containerPort":8088,"protocol":"TCP"},{"name":"https-splunkd","containerPort":8089,"protocol":"TCP"},{"name":"tcp-s2s","containerPort":9997,"protocol":"TCP"},{"name":"user-defined","containerPort":32000,"protocol":"UDP"}],"env":[{"name":"TEST_ENV_VAR","value":"test_value"},{"name":"SPLUNK_HOME","value":"/opt/splunk"},{"name":"SPLUNK_START_ARGS","value":"--accept-license"},{"name":"SPLUNK_DEFAULTS_URL","value":"/mnt/splunk-secrets/default.yml"},{"name":"SPLUNK_HOME_OWNERSHIP_ENFORCEMENT","value":"false"},{"name":"SPLUNK_ROLE","value":"splunk_ingestor"},{"name":"SPLUNK_DECLARATIVE_ADMIN_PASSWORD","value":"true"},{"name":"SPLUNK_OPERATOR_K8_LIVENESS_DRIVER_FILE_PATH","value":"/tmp/splunk_operator_k8s/probes/k8_liveness_driver.sh"},{"name":"SPLUNK_GENERAL_TERMS","value":"--accept-sgt-current-at-splunk-com"},{"name":"SPLUNK_SKIP_CLUSTER_BUNDLE_PUSH","value":"true"}],"resources":{"limits":{"cpu":"4","memory":"8Gi"},"requests":{"cpu":"100m","memory":"512Mi"}},"volumeMounts":[{"name":"pvc-etc","mountPath":"/opt/splunk/etc"},{"name":"pvc-var","mountPath":"/opt/splunk/var"},{"name":"splunk-test-probe-configmap","mountPath":"/mnt/probes"},{"name":"mnt-splunk-secrets","mountPath":"/mnt/splunk-secrets"},{"name":"mnt-splunk-queue-config","readOnly":true,"mountPath":"/mnt/splunk-queue-config"}],"livenessProbe":{"exec":{"command":["/mnt/probes/livenessProbe.sh"]},"initialDelaySeconds":30,"timeoutSeconds":30,"periodSeconds":30,"failureThreshold":3},"readinessProbe":{"exec":{"command":["/mnt/probes/readinessProbe.sh"]},"initialDelaySeconds":10,"timeoutSeconds":5,"periodSeconds":5,"failureThreshold":3},"startupProbe":{"exec":{"command":["/mnt/probes/startupProbe.sh"]},"initialDelaySeconds":40,"timeoutSeconds":30,"periodSeconds":30,"failureThreshold":12},"imagePullPolicy":"IfNotPresent","securityContext":{"capabilities":{"add":["NET_BIND_SERVICE"],"drop":["ALL"]},"privileged":false,"runAsUser":41812,"runAsNonRoot":true,"allowPrivilegeEscalation":false,"seccompProfile":{"type":"RuntimeDefault"}}}],"serviceAccountName":"defaults","securityContext":{"runAsUser":41812,"runAsNonRoot":true,"fsGroup":41812,"fsGroupChangePolicy":"OnRootMismatch"},"affinity":{"podAntiAffinity":{"preferredDuringSchedulingIgnoredDuringExecution":[{"weight":100,"podAffinityTerm":{"labelSelector":{"matchExpressions":[{"key":"app.kubernetes.io/instance","operator":"In","values":["splunk-test-ingestor"]}]},"topologyKey":"kubernetes.io/hostname"}}]}},"schedulerName":"default-scheduler"}},"volumeClaimTemplates":[{"metadata":{"name":"pvc-etc","namespace":"test","creationTimestamp":null,"labels":{"app.kubernetes.io/component":"ingestor","app.kubernetes.io/instance":"splunk-test-ingestor","app.kubernetes.io/managed-by":"splunk-operator","app.kubernetes.io/name":"ingestor","app.kubernetes.io/part-of":"splunk-test-ingestor"}},"spec":{"accessModes":["ReadWriteOnce"],"resources":{"requests":{"storage":"10Gi"}}},"status":{}},{"metadata":{"name":"pvc-var","namespace":"test","creationTimestamp":null,"labels":{"app.kubernetes.io/component":"ingestor","app.kubernetes.io/instance":"splunk-test-ingestor","app.kubernetes.io/managed-by":"splunk-operator","app.kubernetes.io/name":"ingestor","app.kubernetes.io/part-of":"splunk-test-ingestor"}},"spec":{"accessModes":["ReadWriteOnce"],"resources":{"requests":{"storage":"100Gi"}}},"status":{}}],"serviceName":"splunk-test-ingestor-headless","podManagementPolicy":"Parallel","updateStrategy":{"type":"OnDelete"}},"status":{"replicas":0,"availableReplicas":0}} \ No newline at end of file diff --git a/pkg/splunk/enterprise/testdata/fixtures/statefulset_ingestor_with_labels.json b/pkg/splunk/enterprise/testdata/fixtures/statefulset_ingestor_with_labels.json index 9a35ffab7..92e69feb7 100644 --- a/pkg/splunk/enterprise/testdata/fixtures/statefulset_ingestor_with_labels.json +++ b/pkg/splunk/enterprise/testdata/fixtures/statefulset_ingestor_with_labels.json @@ -1,213 +1 @@ -{ - "kind": "StatefulSet", - "apiVersion": "apps/v1", - "metadata": { - "name": "splunk-test-ingestor", - "namespace": "test", - "creationTimestamp": null, - "labels": { - "app.kubernetes.io/component": "ingestor", - "app.kubernetes.io/instance": "splunk-test-ingestor", - "app.kubernetes.io/managed-by": "splunk-operator", - "app.kubernetes.io/name": "ingestor", - "app.kubernetes.io/part-of": "splunk-test-ingestor", - "app.kubernetes.io/test-extra-label": "test-extra-label-value" - }, - "ownerReferences": [ - { - "apiVersion": "", - "kind": "IngestorCluster", - "name": "test", - "uid": "", - "controller": true - } - ] - }, - "spec": { - "replicas": 1, - "selector": { - "matchLabels": { - "app.kubernetes.io/component": "ingestor", - "app.kubernetes.io/instance": "splunk-test-ingestor", - "app.kubernetes.io/managed-by": "splunk-operator", - "app.kubernetes.io/name": "ingestor", - "app.kubernetes.io/part-of": "splunk-test-ingestor" - } - }, - "template": { - "metadata": { - "creationTimestamp": null, - "labels": { - "app.kubernetes.io/component": "ingestor", - "app.kubernetes.io/instance": "splunk-test-ingestor", - "app.kubernetes.io/managed-by": "splunk-operator", - "app.kubernetes.io/name": "ingestor", - "app.kubernetes.io/part-of": "splunk-test-ingestor", - "app.kubernetes.io/test-extra-label": "test-extra-label-value" - }, - "annotations": { - "traffic.sidecar.istio.io/excludeOutboundPorts": "8089,8191,9997", - "traffic.sidecar.istio.io/includeInboundPorts": "8000,8088" - } - }, - "spec": { - "volumes": [ - { - "name": "splunk-test-probe-configmap", - "configMap": { - "name": "splunk-test-probe-configmap", - "defaultMode": 365 - } - }, - { - "name": "mnt-splunk-secrets", - "secret": { - "secretName": "splunk-test-ingestor-secret-v1", - "defaultMode": 420 - } - } - ], - "containers": [ - { - "name": "splunk", - "image": "splunk/splunk", - "ports": [ - { "name": "http-splunkweb", "containerPort": 8000, "protocol": "TCP" }, - { "name": "http-hec", "containerPort": 8088, "protocol": "TCP" }, - { "name": "https-splunkd", "containerPort": 8089, "protocol": "TCP" }, - { "name": "tcp-s2s", "containerPort": 9997, "protocol": "TCP" }, - { "name": "user-defined", "containerPort": 32000, "protocol": "UDP" } - ], - "env": [ - { "name": "TEST_ENV_VAR", "value": "test_value" }, - { "name": "SPLUNK_HOME", "value": "/opt/splunk" }, - { "name": "SPLUNK_START_ARGS", "value": "--accept-license" }, - { "name": "SPLUNK_DEFAULTS_URL", "value": "/mnt/splunk-secrets/default.yml" }, - { "name": "SPLUNK_HOME_OWNERSHIP_ENFORCEMENT", "value": "false" }, - { "name": "SPLUNK_ROLE", "value": "splunk_ingestor" }, - { "name": "SPLUNK_DECLARATIVE_ADMIN_PASSWORD", "value": "true" }, - { "name": "SPLUNK_OPERATOR_K8_LIVENESS_DRIVER_FILE_PATH", "value": "/tmp/splunk_operator_k8s/probes/k8_liveness_driver.sh" }, - { "name": "SPLUNK_GENERAL_TERMS", "value": "--accept-sgt-current-at-splunk-com" }, - { "name": "SPLUNK_SKIP_CLUSTER_BUNDLE_PUSH", "value": "true" } - ], - "resources": { - "limits": { "cpu": "4", "memory": "8Gi" }, - "requests": { "cpu": "100m", "memory": "512Mi" } - }, - "volumeMounts": [ - { "name": "pvc-etc", "mountPath": "/opt/splunk/etc" }, - { "name": "pvc-var", "mountPath": "/opt/splunk/var" }, - { "name": "splunk-test-probe-configmap", "mountPath": "/mnt/probes" }, - { "name": "mnt-splunk-secrets", "mountPath": "/mnt/splunk-secrets" } - ], - "livenessProbe": { - "exec": { "command": ["/mnt/probes/livenessProbe.sh"] }, - "initialDelaySeconds": 30, - "timeoutSeconds": 30, - "periodSeconds": 30, - "failureThreshold": 3 - }, - "readinessProbe": { - "exec": { "command": ["/mnt/probes/readinessProbe.sh"] }, - "initialDelaySeconds": 10, - "timeoutSeconds": 5, - "periodSeconds": 5, - "failureThreshold": 3 - }, - "startupProbe": { - "exec": { "command": ["/mnt/probes/startupProbe.sh"] }, - "initialDelaySeconds": 40, - "timeoutSeconds": 30, - "periodSeconds": 30, - "failureThreshold": 12 - }, - "imagePullPolicy": "IfNotPresent", - "securityContext": { - "capabilities": { "add": ["NET_BIND_SERVICE"], "drop": ["ALL"] }, - "privileged": false, - "runAsUser": 41812, - "runAsNonRoot": true, - "allowPrivilegeEscalation": false, - "seccompProfile": { "type": "RuntimeDefault" } - } - } - ], - "serviceAccountName": "defaults", - "securityContext": { - "runAsUser": 41812, - "runAsNonRoot": true, - "fsGroup": 41812, - "fsGroupChangePolicy": "OnRootMismatch" - }, - "affinity": { - "podAntiAffinity": { - "preferredDuringSchedulingIgnoredDuringExecution": [ - { - "weight": 100, - "podAffinityTerm": { - "labelSelector": { - "matchExpressions": [ - { - "key": "app.kubernetes.io/instance", - "operator": "In", - "values": ["splunk-test-ingestor"] - } - ] - }, - "topologyKey": "kubernetes.io/hostname" - } - } - ] - } - }, - "schedulerName": "default-scheduler" - } - }, - "volumeClaimTemplates": [ - { - "metadata": { - "name": "pvc-etc", - "namespace": "test", - "creationTimestamp": null, - "labels": { - "app.kubernetes.io/component": "ingestor", - "app.kubernetes.io/instance": "splunk-test-ingestor", - "app.kubernetes.io/managed-by": "splunk-operator", - "app.kubernetes.io/name": "ingestor", - "app.kubernetes.io/part-of": "splunk-test-ingestor", - "app.kubernetes.io/test-extra-label": "test-extra-label-value" - } - }, - "spec": { - "accessModes": ["ReadWriteOnce"], - "resources": { "requests": { "storage": "10Gi" } } - }, - "status": {} - }, - { - "metadata": { - "name": "pvc-var", - "namespace": "test", - "creationTimestamp": null, - "labels": { - "app.kubernetes.io/component": "ingestor", - "app.kubernetes.io/instance": "splunk-test-ingestor", - "app.kubernetes.io/managed-by": "splunk-operator", - "app.kubernetes.io/name": "ingestor", - "app.kubernetes.io/part-of": "splunk-test-ingestor", - "app.kubernetes.io/test-extra-label": "test-extra-label-value" - } - }, - "spec": { - "accessModes": ["ReadWriteOnce"], - "resources": { "requests": { "storage": "100Gi" } } - }, - "status": {} - } - ], - "serviceName": "splunk-test-ingestor-headless", - "podManagementPolicy": "Parallel", - "updateStrategy": { "type": "OnDelete" } - }, - "status": { "replicas": 0, "availableReplicas": 0 } -} \ No newline at end of file +{"kind":"StatefulSet","apiVersion":"apps/v1","metadata":{"name":"splunk-test-ingestor","namespace":"test","creationTimestamp":null,"labels":{"app.kubernetes.io/component":"ingestor","app.kubernetes.io/instance":"splunk-test-ingestor","app.kubernetes.io/managed-by":"splunk-operator","app.kubernetes.io/name":"ingestor","app.kubernetes.io/part-of":"splunk-test-ingestor","app.kubernetes.io/test-extra-label":"test-extra-label-value"},"ownerReferences":[{"apiVersion":"","kind":"IngestorCluster","name":"test","uid":"","controller":true}]},"spec":{"replicas":1,"selector":{"matchLabels":{"app.kubernetes.io/component":"ingestor","app.kubernetes.io/instance":"splunk-test-ingestor","app.kubernetes.io/managed-by":"splunk-operator","app.kubernetes.io/name":"ingestor","app.kubernetes.io/part-of":"splunk-test-ingestor"}},"template":{"metadata":{"creationTimestamp":null,"labels":{"app.kubernetes.io/component":"ingestor","app.kubernetes.io/instance":"splunk-test-ingestor","app.kubernetes.io/managed-by":"splunk-operator","app.kubernetes.io/name":"ingestor","app.kubernetes.io/part-of":"splunk-test-ingestor","app.kubernetes.io/test-extra-label":"test-extra-label-value"},"annotations":{"traffic.sidecar.istio.io/excludeOutboundPorts":"8089,8191,9997","traffic.sidecar.istio.io/includeInboundPorts":"8000,8088"}},"spec":{"volumes":[{"name":"splunk-test-probe-configmap","configMap":{"name":"splunk-test-probe-configmap","defaultMode":365}},{"name":"mnt-splunk-secrets","secret":{"secretName":"splunk-test-ingestor-secret-v1","defaultMode":420}},{"name":"mnt-splunk-queue-config","configMap":{"name":"splunk-test-ingestor-queue-config","defaultMode":420}}],"initContainers":[{"name":"init-ingestor-queue-config","image":"splunk/splunk","command":["bash","-c","mkdir -p /opt/splk/etc/apps/100-sok-ingestorcluster/local \u0026\u0026 mkdir -p /opt/splk/etc/apps/100-sok-ingestorcluster/metadata \u0026\u0026 ln -sfn /mnt/splunk-queue-config/app.conf /opt/splk/etc/apps/100-sok-ingestorcluster/local/app.conf \u0026\u0026 ln -sfn /mnt/splunk-queue-config/outputs.conf /opt/splk/etc/apps/100-sok-ingestorcluster/local/outputs.conf \u0026\u0026 ln -sfn /mnt/splunk-queue-config/default-mode.conf /opt/splk/etc/apps/100-sok-ingestorcluster/local/default-mode.conf \u0026\u0026 cp /mnt/splunk-queue-config/local.meta /opt/splk/etc/apps/100-sok-ingestorcluster/metadata/local.meta"],"resources":{"limits":{"cpu":"1","memory":"512Mi"},"requests":{"cpu":"250m","memory":"128Mi"}},"volumeMounts":[{"name":"pvc-etc","mountPath":"/opt/splk/etc"},{"name":"mnt-splunk-queue-config","mountPath":"/mnt/splunk-queue-config"}],"imagePullPolicy":"IfNotPresent","securityContext":{"capabilities":{"add":["NET_BIND_SERVICE"],"drop":["ALL"]},"privileged":false,"runAsUser":41812,"runAsNonRoot":true,"allowPrivilegeEscalation":false,"seccompProfile":{"type":"RuntimeDefault"}}}],"containers":[{"name":"splunk","image":"splunk/splunk","ports":[{"name":"http-splunkweb","containerPort":8000,"protocol":"TCP"},{"name":"http-hec","containerPort":8088,"protocol":"TCP"},{"name":"https-splunkd","containerPort":8089,"protocol":"TCP"},{"name":"tcp-s2s","containerPort":9997,"protocol":"TCP"},{"name":"user-defined","containerPort":32000,"protocol":"UDP"}],"env":[{"name":"TEST_ENV_VAR","value":"test_value"},{"name":"SPLUNK_HOME","value":"/opt/splunk"},{"name":"SPLUNK_START_ARGS","value":"--accept-license"},{"name":"SPLUNK_DEFAULTS_URL","value":"/mnt/splunk-secrets/default.yml"},{"name":"SPLUNK_HOME_OWNERSHIP_ENFORCEMENT","value":"false"},{"name":"SPLUNK_ROLE","value":"splunk_ingestor"},{"name":"SPLUNK_DECLARATIVE_ADMIN_PASSWORD","value":"true"},{"name":"SPLUNK_OPERATOR_K8_LIVENESS_DRIVER_FILE_PATH","value":"/tmp/splunk_operator_k8s/probes/k8_liveness_driver.sh"},{"name":"SPLUNK_GENERAL_TERMS","value":"--accept-sgt-current-at-splunk-com"},{"name":"SPLUNK_SKIP_CLUSTER_BUNDLE_PUSH","value":"true"}],"resources":{"limits":{"cpu":"4","memory":"8Gi"},"requests":{"cpu":"100m","memory":"512Mi"}},"volumeMounts":[{"name":"pvc-etc","mountPath":"/opt/splunk/etc"},{"name":"pvc-var","mountPath":"/opt/splunk/var"},{"name":"splunk-test-probe-configmap","mountPath":"/mnt/probes"},{"name":"mnt-splunk-secrets","mountPath":"/mnt/splunk-secrets"},{"name":"mnt-splunk-queue-config","readOnly":true,"mountPath":"/mnt/splunk-queue-config"}],"livenessProbe":{"exec":{"command":["/mnt/probes/livenessProbe.sh"]},"initialDelaySeconds":30,"timeoutSeconds":30,"periodSeconds":30,"failureThreshold":3},"readinessProbe":{"exec":{"command":["/mnt/probes/readinessProbe.sh"]},"initialDelaySeconds":10,"timeoutSeconds":5,"periodSeconds":5,"failureThreshold":3},"startupProbe":{"exec":{"command":["/mnt/probes/startupProbe.sh"]},"initialDelaySeconds":40,"timeoutSeconds":30,"periodSeconds":30,"failureThreshold":12},"imagePullPolicy":"IfNotPresent","securityContext":{"capabilities":{"add":["NET_BIND_SERVICE"],"drop":["ALL"]},"privileged":false,"runAsUser":41812,"runAsNonRoot":true,"allowPrivilegeEscalation":false,"seccompProfile":{"type":"RuntimeDefault"}}}],"serviceAccountName":"defaults","securityContext":{"runAsUser":41812,"runAsNonRoot":true,"fsGroup":41812,"fsGroupChangePolicy":"OnRootMismatch"},"affinity":{"podAntiAffinity":{"preferredDuringSchedulingIgnoredDuringExecution":[{"weight":100,"podAffinityTerm":{"labelSelector":{"matchExpressions":[{"key":"app.kubernetes.io/instance","operator":"In","values":["splunk-test-ingestor"]}]},"topologyKey":"kubernetes.io/hostname"}}]}},"schedulerName":"default-scheduler"}},"volumeClaimTemplates":[{"metadata":{"name":"pvc-etc","namespace":"test","creationTimestamp":null,"labels":{"app.kubernetes.io/component":"ingestor","app.kubernetes.io/instance":"splunk-test-ingestor","app.kubernetes.io/managed-by":"splunk-operator","app.kubernetes.io/name":"ingestor","app.kubernetes.io/part-of":"splunk-test-ingestor","app.kubernetes.io/test-extra-label":"test-extra-label-value"}},"spec":{"accessModes":["ReadWriteOnce"],"resources":{"requests":{"storage":"10Gi"}}},"status":{}},{"metadata":{"name":"pvc-var","namespace":"test","creationTimestamp":null,"labels":{"app.kubernetes.io/component":"ingestor","app.kubernetes.io/instance":"splunk-test-ingestor","app.kubernetes.io/managed-by":"splunk-operator","app.kubernetes.io/name":"ingestor","app.kubernetes.io/part-of":"splunk-test-ingestor","app.kubernetes.io/test-extra-label":"test-extra-label-value"}},"spec":{"accessModes":["ReadWriteOnce"],"resources":{"requests":{"storage":"100Gi"}}},"status":{}}],"serviceName":"splunk-test-ingestor-headless","podManagementPolicy":"Parallel","updateStrategy":{"type":"OnDelete"}},"status":{"replicas":0,"availableReplicas":0}} \ No newline at end of file diff --git a/pkg/splunk/enterprise/testdata/fixtures/statefulset_ingestor_with_serviceaccount.json b/pkg/splunk/enterprise/testdata/fixtures/statefulset_ingestor_with_serviceaccount.json index eb261195d..27d544829 100644 --- a/pkg/splunk/enterprise/testdata/fixtures/statefulset_ingestor_with_serviceaccount.json +++ b/pkg/splunk/enterprise/testdata/fixtures/statefulset_ingestor_with_serviceaccount.json @@ -1,208 +1 @@ -{ - "kind": "StatefulSet", - "apiVersion": "apps/v1", - "metadata": { - "name": "splunk-test-ingestor", - "namespace": "test", - "creationTimestamp": null, - "labels": { - "app.kubernetes.io/component": "ingestor", - "app.kubernetes.io/instance": "splunk-test-ingestor", - "app.kubernetes.io/managed-by": "splunk-operator", - "app.kubernetes.io/name": "ingestor", - "app.kubernetes.io/part-of": "splunk-test-ingestor" - }, - "ownerReferences": [ - { - "apiVersion": "", - "kind": "IngestorCluster", - "name": "test", - "uid": "", - "controller": true - } - ] - }, - "spec": { - "replicas": 1, - "selector": { - "matchLabels": { - "app.kubernetes.io/component": "ingestor", - "app.kubernetes.io/instance": "splunk-test-ingestor", - "app.kubernetes.io/managed-by": "splunk-operator", - "app.kubernetes.io/name": "ingestor", - "app.kubernetes.io/part-of": "splunk-test-ingestor" - } - }, - "template": { - "metadata": { - "creationTimestamp": null, - "labels": { - "app.kubernetes.io/component": "ingestor", - "app.kubernetes.io/instance": "splunk-test-ingestor", - "app.kubernetes.io/managed-by": "splunk-operator", - "app.kubernetes.io/name": "ingestor", - "app.kubernetes.io/part-of": "splunk-test-ingestor" - }, - "annotations": { - "traffic.sidecar.istio.io/excludeOutboundPorts": "8089,8191,9997", - "traffic.sidecar.istio.io/includeInboundPorts": "8000,8088" - } - }, - "spec": { - "volumes": [ - { - "name": "splunk-test-probe-configmap", - "configMap": { - "name": "splunk-test-probe-configmap", - "defaultMode": 365 - } - }, - { - "name": "mnt-splunk-secrets", - "secret": { - "secretName": "splunk-test-ingestor-secret-v1", - "defaultMode": 420 - } - } - ], - "containers": [ - { - "name": "splunk", - "image": "splunk/splunk", - "ports": [ - { "name": "http-splunkweb", "containerPort": 8000, "protocol": "TCP" }, - { "name": "http-hec", "containerPort": 8088, "protocol": "TCP" }, - { "name": "https-splunkd", "containerPort": 8089, "protocol": "TCP" }, - { "name": "tcp-s2s", "containerPort": 9997, "protocol": "TCP" }, - { "name": "user-defined", "containerPort": 32000, "protocol": "UDP" } - ], - "env": [ - { "name": "SPLUNK_HOME", "value": "/opt/splunk" }, - { "name": "SPLUNK_START_ARGS", "value": "--accept-license" }, - { "name": "SPLUNK_DEFAULTS_URL", "value": "/mnt/splunk-secrets/default.yml" }, - { "name": "SPLUNK_HOME_OWNERSHIP_ENFORCEMENT", "value": "false" }, - { "name": "SPLUNK_ROLE", "value": "splunk_ingestor" }, - { "name": "SPLUNK_DECLARATIVE_ADMIN_PASSWORD", "value": "true" }, - { "name": "SPLUNK_OPERATOR_K8_LIVENESS_DRIVER_FILE_PATH", "value": "/tmp/splunk_operator_k8s/probes/k8_liveness_driver.sh" }, - { "name": "SPLUNK_GENERAL_TERMS", "value": "--accept-sgt-current-at-splunk-com" }, - { "name": "SPLUNK_SKIP_CLUSTER_BUNDLE_PUSH", "value": "true" } - ], - "resources": { - "limits": { "cpu": "4", "memory": "8Gi" }, - "requests": { "cpu": "100m", "memory": "512Mi" } - }, - "volumeMounts": [ - { "name": "pvc-etc", "mountPath": "/opt/splunk/etc" }, - { "name": "pvc-var", "mountPath": "/opt/splunk/var" }, - { "name": "splunk-test-probe-configmap", "mountPath": "/mnt/probes" }, - { "name": "mnt-splunk-secrets", "mountPath": "/mnt/splunk-secrets" } - ], - "livenessProbe": { - "exec": { "command": ["/mnt/probes/livenessProbe.sh"] }, - "initialDelaySeconds": 30, - "timeoutSeconds": 30, - "periodSeconds": 30, - "failureThreshold": 3 - }, - "readinessProbe": { - "exec": { "command": ["/mnt/probes/readinessProbe.sh"] }, - "initialDelaySeconds": 10, - "timeoutSeconds": 5, - "periodSeconds": 5, - "failureThreshold": 3 - }, - "startupProbe": { - "exec": { "command": ["/mnt/probes/startupProbe.sh"] }, - "initialDelaySeconds": 40, - "timeoutSeconds": 30, - "periodSeconds": 30, - "failureThreshold": 12 - }, - "imagePullPolicy": "IfNotPresent", - "securityContext": { - "capabilities": { "add": ["NET_BIND_SERVICE"], "drop": ["ALL"] }, - "privileged": false, - "runAsUser": 41812, - "runAsNonRoot": true, - "allowPrivilegeEscalation": false, - "seccompProfile": { "type": "RuntimeDefault" } - } - } - ], - "serviceAccountName": "defaults", - "securityContext": { - "runAsUser": 41812, - "runAsNonRoot": true, - "fsGroup": 41812, - "fsGroupChangePolicy": "OnRootMismatch" - }, - "affinity": { - "podAntiAffinity": { - "preferredDuringSchedulingIgnoredDuringExecution": [ - { - "weight": 100, - "podAffinityTerm": { - "labelSelector": { - "matchExpressions": [ - { - "key": "app.kubernetes.io/instance", - "operator": "In", - "values": ["splunk-test-ingestor"] - } - ] - }, - "topologyKey": "kubernetes.io/hostname" - } - } - ] - } - }, - "schedulerName": "default-scheduler" - } - }, - "volumeClaimTemplates": [ - { - "metadata": { - "name": "pvc-etc", - "namespace": "test", - "creationTimestamp": null, - "labels": { - "app.kubernetes.io/component": "ingestor", - "app.kubernetes.io/instance": "splunk-test-ingestor", - "app.kubernetes.io/managed-by": "splunk-operator", - "app.kubernetes.io/name": "ingestor", - "app.kubernetes.io/part-of": "splunk-test-ingestor" - } - }, - "spec": { - "accessModes": ["ReadWriteOnce"], - "resources": { "requests": { "storage": "10Gi" } } - }, - "status": {} - }, - { - "metadata": { - "name": "pvc-var", - "namespace": "test", - "creationTimestamp": null, - "labels": { - "app.kubernetes.io/component": "ingestor", - "app.kubernetes.io/instance": "splunk-test-ingestor", - "app.kubernetes.io/managed-by": "splunk-operator", - "app.kubernetes.io/name": "ingestor", - "app.kubernetes.io/part-of": "splunk-test-ingestor" - } - }, - "spec": { - "accessModes": ["ReadWriteOnce"], - "resources": { "requests": { "storage": "100Gi" } } - }, - "status": {} - } - ], - "serviceName": "splunk-test-ingestor-headless", - "podManagementPolicy": "Parallel", - "updateStrategy": { "type": "OnDelete" } - }, - "status": { "replicas": 0, "availableReplicas": 0 } -} \ No newline at end of file +{"kind":"StatefulSet","apiVersion":"apps/v1","metadata":{"name":"splunk-test-ingestor","namespace":"test","creationTimestamp":null,"labels":{"app.kubernetes.io/component":"ingestor","app.kubernetes.io/instance":"splunk-test-ingestor","app.kubernetes.io/managed-by":"splunk-operator","app.kubernetes.io/name":"ingestor","app.kubernetes.io/part-of":"splunk-test-ingestor"},"ownerReferences":[{"apiVersion":"","kind":"IngestorCluster","name":"test","uid":"","controller":true}]},"spec":{"replicas":1,"selector":{"matchLabels":{"app.kubernetes.io/component":"ingestor","app.kubernetes.io/instance":"splunk-test-ingestor","app.kubernetes.io/managed-by":"splunk-operator","app.kubernetes.io/name":"ingestor","app.kubernetes.io/part-of":"splunk-test-ingestor"}},"template":{"metadata":{"creationTimestamp":null,"labels":{"app.kubernetes.io/component":"ingestor","app.kubernetes.io/instance":"splunk-test-ingestor","app.kubernetes.io/managed-by":"splunk-operator","app.kubernetes.io/name":"ingestor","app.kubernetes.io/part-of":"splunk-test-ingestor"},"annotations":{"traffic.sidecar.istio.io/excludeOutboundPorts":"8089,8191,9997","traffic.sidecar.istio.io/includeInboundPorts":"8000,8088"}},"spec":{"volumes":[{"name":"splunk-test-probe-configmap","configMap":{"name":"splunk-test-probe-configmap","defaultMode":365}},{"name":"mnt-splunk-secrets","secret":{"secretName":"splunk-test-ingestor-secret-v1","defaultMode":420}},{"name":"mnt-splunk-queue-config","configMap":{"name":"splunk-test-ingestor-queue-config","defaultMode":420}}],"initContainers":[{"name":"init-ingestor-queue-config","image":"splunk/splunk","command":["bash","-c","mkdir -p /opt/splk/etc/apps/100-sok-ingestorcluster/local \u0026\u0026 mkdir -p /opt/splk/etc/apps/100-sok-ingestorcluster/metadata \u0026\u0026 ln -sfn /mnt/splunk-queue-config/app.conf /opt/splk/etc/apps/100-sok-ingestorcluster/local/app.conf \u0026\u0026 ln -sfn /mnt/splunk-queue-config/outputs.conf /opt/splk/etc/apps/100-sok-ingestorcluster/local/outputs.conf \u0026\u0026 ln -sfn /mnt/splunk-queue-config/default-mode.conf /opt/splk/etc/apps/100-sok-ingestorcluster/local/default-mode.conf \u0026\u0026 cp /mnt/splunk-queue-config/local.meta /opt/splk/etc/apps/100-sok-ingestorcluster/metadata/local.meta"],"resources":{"limits":{"cpu":"1","memory":"512Mi"},"requests":{"cpu":"250m","memory":"128Mi"}},"volumeMounts":[{"name":"pvc-etc","mountPath":"/opt/splk/etc"},{"name":"mnt-splunk-queue-config","mountPath":"/mnt/splunk-queue-config"}],"imagePullPolicy":"IfNotPresent","securityContext":{"capabilities":{"add":["NET_BIND_SERVICE"],"drop":["ALL"]},"privileged":false,"runAsUser":41812,"runAsNonRoot":true,"allowPrivilegeEscalation":false,"seccompProfile":{"type":"RuntimeDefault"}}}],"containers":[{"name":"splunk","image":"splunk/splunk","ports":[{"name":"http-splunkweb","containerPort":8000,"protocol":"TCP"},{"name":"http-hec","containerPort":8088,"protocol":"TCP"},{"name":"https-splunkd","containerPort":8089,"protocol":"TCP"},{"name":"tcp-s2s","containerPort":9997,"protocol":"TCP"},{"name":"user-defined","containerPort":32000,"protocol":"UDP"}],"env":[{"name":"SPLUNK_HOME","value":"/opt/splunk"},{"name":"SPLUNK_START_ARGS","value":"--accept-license"},{"name":"SPLUNK_DEFAULTS_URL","value":"/mnt/splunk-secrets/default.yml"},{"name":"SPLUNK_HOME_OWNERSHIP_ENFORCEMENT","value":"false"},{"name":"SPLUNK_ROLE","value":"splunk_ingestor"},{"name":"SPLUNK_DECLARATIVE_ADMIN_PASSWORD","value":"true"},{"name":"SPLUNK_OPERATOR_K8_LIVENESS_DRIVER_FILE_PATH","value":"/tmp/splunk_operator_k8s/probes/k8_liveness_driver.sh"},{"name":"SPLUNK_GENERAL_TERMS","value":"--accept-sgt-current-at-splunk-com"},{"name":"SPLUNK_SKIP_CLUSTER_BUNDLE_PUSH","value":"true"}],"resources":{"limits":{"cpu":"4","memory":"8Gi"},"requests":{"cpu":"100m","memory":"512Mi"}},"volumeMounts":[{"name":"pvc-etc","mountPath":"/opt/splunk/etc"},{"name":"pvc-var","mountPath":"/opt/splunk/var"},{"name":"splunk-test-probe-configmap","mountPath":"/mnt/probes"},{"name":"mnt-splunk-secrets","mountPath":"/mnt/splunk-secrets"},{"name":"mnt-splunk-queue-config","readOnly":true,"mountPath":"/mnt/splunk-queue-config"}],"livenessProbe":{"exec":{"command":["/mnt/probes/livenessProbe.sh"]},"initialDelaySeconds":30,"timeoutSeconds":30,"periodSeconds":30,"failureThreshold":3},"readinessProbe":{"exec":{"command":["/mnt/probes/readinessProbe.sh"]},"initialDelaySeconds":10,"timeoutSeconds":5,"periodSeconds":5,"failureThreshold":3},"startupProbe":{"exec":{"command":["/mnt/probes/startupProbe.sh"]},"initialDelaySeconds":40,"timeoutSeconds":30,"periodSeconds":30,"failureThreshold":12},"imagePullPolicy":"IfNotPresent","securityContext":{"capabilities":{"add":["NET_BIND_SERVICE"],"drop":["ALL"]},"privileged":false,"runAsUser":41812,"runAsNonRoot":true,"allowPrivilegeEscalation":false,"seccompProfile":{"type":"RuntimeDefault"}}}],"serviceAccountName":"defaults","securityContext":{"runAsUser":41812,"runAsNonRoot":true,"fsGroup":41812,"fsGroupChangePolicy":"OnRootMismatch"},"affinity":{"podAntiAffinity":{"preferredDuringSchedulingIgnoredDuringExecution":[{"weight":100,"podAffinityTerm":{"labelSelector":{"matchExpressions":[{"key":"app.kubernetes.io/instance","operator":"In","values":["splunk-test-ingestor"]}]},"topologyKey":"kubernetes.io/hostname"}}]}},"schedulerName":"default-scheduler"}},"volumeClaimTemplates":[{"metadata":{"name":"pvc-etc","namespace":"test","creationTimestamp":null,"labels":{"app.kubernetes.io/component":"ingestor","app.kubernetes.io/instance":"splunk-test-ingestor","app.kubernetes.io/managed-by":"splunk-operator","app.kubernetes.io/name":"ingestor","app.kubernetes.io/part-of":"splunk-test-ingestor"}},"spec":{"accessModes":["ReadWriteOnce"],"resources":{"requests":{"storage":"10Gi"}}},"status":{}},{"metadata":{"name":"pvc-var","namespace":"test","creationTimestamp":null,"labels":{"app.kubernetes.io/component":"ingestor","app.kubernetes.io/instance":"splunk-test-ingestor","app.kubernetes.io/managed-by":"splunk-operator","app.kubernetes.io/name":"ingestor","app.kubernetes.io/part-of":"splunk-test-ingestor"}},"spec":{"accessModes":["ReadWriteOnce"],"resources":{"requests":{"storage":"100Gi"}}},"status":{}}],"serviceName":"splunk-test-ingestor-headless","podManagementPolicy":"Parallel","updateStrategy":{"type":"OnDelete"}},"status":{"replicas":0,"availableReplicas":0}} \ No newline at end of file diff --git a/pkg/splunk/enterprise/util.go b/pkg/splunk/enterprise/util.go index cc48f69a7..6084b25f8 100644 --- a/pkg/splunk/enterprise/util.go +++ b/pkg/splunk/enterprise/util.go @@ -466,9 +466,9 @@ func GetQueueRemoteVolumeSecrets(ctx context.Context, volume enterpriseApi.Volum version := namespaceScopedSecret.ResourceVersion - if accessKey == "" { - return "", "", "", errors.New("access Key is missing") - } else if secretKey == "" { + // Empty credentials are valid for IRSA mode (pod identity via service account token). + // Only return an error when one key is present but the other is missing. + if accessKey != "" && secretKey == "" { return "", "", "", errors.New("secret Key is missing") } @@ -722,13 +722,20 @@ func ApplySmartstoreConfigMap(ctx context.Context, client splcommon.ControllerCl SplunkOperatorAppConfigMap.SetOwnerReferences(append(SplunkOperatorAppConfigMap.GetOwnerReferences(), splcommon.AsOwner(cr, true))) - // if existing configmap contains key conftoken then add that back + // if existing configmap contains key conftoken then add that back. + // Also preserve queue config keys (outputs.conf, inputs.conf, default-mode.conf) written by + // applyIdxcQueueConfigToCM so that CM reconcile does not overwrite them. namespacedName := types.NamespacedName{Namespace: cr.GetNamespace(), Name: configMapName} configMap, err := splctrl.GetConfigMap(ctx, client, namespacedName) if err == nil && configMap != nil && configMap.Data != nil && reflect.ValueOf(configMap.Data).Kind() == reflect.Map { if _, ok := configMap.Data[configToken]; ok { SplunkOperatorAppConfigMap.Data[configToken] = configMap.Data[configToken] } + for _, queueKey := range []string{"outputs.conf", "inputs.conf", "default-mode.conf"} { + if v, ok := configMap.Data[queueKey]; ok { + SplunkOperatorAppConfigMap.Data[queueKey] = v + } + } } configMapDataChanged, err = splctrl.ApplyConfigMap(ctx, client, SplunkOperatorAppConfigMap) @@ -769,7 +776,7 @@ var resetSymbolicLinks = func(ctx context.Context, client splcommon.ControllerCl reqLogger := log.FromContext(ctx) scopedLog := reqLogger.WithName("ResetSymbolicLinks").WithValues("kind", crKind, "name", cr.GetName(), "namespace", cr.GetNamespace()) - // Create command for symbolic link creation + // Create command for symbolic link creation. var command string if crKind == "ClusterManager" || crKind == "ClusterMaster" { command = setSymbolicLinkCmanager @@ -784,6 +791,17 @@ var resetSymbolicLinks = func(ctx context.Context, client splcommon.ControllerCl return err } + // Also reset queue config symlinks if the queue config ConfigMap exists. + queueConfigCMName := GetCMQueueConfigMapName(cr.GetName()) + _, queueCMErr := splctrl.GetConfigMap(ctx, client, types.NamespacedName{Name: queueConfigCMName, Namespace: cr.GetNamespace()}) + if queueCMErr == nil { + err = runCustomCommandOnSplunkPods(ctx, cr, replicas, setSymbolicLinkCmanagerQueueConfig, podExecClient) + if err != nil { + scopedLog.Error(err, "unable to reset queue config symbolic links on splunk pod") + return err + } + } + scopedLog.Info("Reset symbolic links successfully") // All good diff --git a/test/index_and_ingestion_separation/index_and_ingestion_separation_test.go b/test/index_and_ingestion_separation/index_and_ingestion_separation_test.go index 85c7de276..4719cff4a 100644 --- a/test/index_and_ingestion_separation/index_and_ingestion_separation_test.go +++ b/test/index_and_ingestion_separation/index_and_ingestion_separation_test.go @@ -315,10 +315,44 @@ var _ = Describe("indingsep test", func() { err = deployment.GetInstance(ctx, deployment.GetName()+"-ingest", ingest) Expect(err).To(Succeed(), "Failed to get instance of Ingestor Cluster") - // Verify Ingestor Cluster Status - testcaseEnvInst.Log.Info("Verify Ingestor Cluster Status") - Expect(ingest.Status.CredentialSecretVersion).To(Not(Equal("")), "Ingestor queue status credential access secret version is empty") - Expect(ingest.Status.CredentialSecretVersion).To(Not(Equal("0")), "Ingestor queue status credential access secret version is 0") + // Verify ingestor queue config ConfigMap exists and has correct content + ingestorCMName := enterprise.GetIngestorQueueConfigMapName(ingest.GetName()) + testcaseEnvInst.Log.Info("Verify ingestor queue config ConfigMap", "configMapName", ingestorCMName, "namespace", testcaseEnvInst.GetName()) + cm, err := testenv.GetConfigMap(ctx, deployment, testcaseEnvInst.GetName(), ingestorCMName) + Expect(err).To(Succeed(), "Failed to get ingestor queue config ConfigMap", "ConfigMap", ingestorCMName) + testcaseEnvInst.Log.Info("Ingestor queue config ConfigMap found", "configMapName", cm.Name, "keyCount", len(cm.Data)) + Expect(cm.Data).To(HaveKey("outputs.conf"), "ConfigMap missing outputs.conf") + Expect(cm.Data).To(HaveKey("default-mode.conf"), "ConfigMap missing default-mode.conf") + Expect(cm.Data).To(HaveKey("app.conf"), "ConfigMap missing app.conf") + Expect(cm.Data).To(HaveKey("local.meta"), "ConfigMap missing local.meta") + snippetLen := min(120, len(cm.Data["outputs.conf"])) + testcaseEnvInst.Log.Info("outputs.conf content verified", "configMapName", ingestorCMName, "snippet", cm.Data["outputs.conf"][:snippetLen]) + Expect(cm.Data["outputs.conf"]).To(ContainSubstring("remote_queue:"), "outputs.conf missing remote_queue stanza") + Expect(cm.Data["default-mode.conf"]).To(ContainSubstring("pipeline:remotequeueruleset"), "default-mode.conf missing pipeline stanza") + + // Verify init container completed successfully on each ingestor pod + testcaseEnvInst.Log.Info("Verify init container status on ingestor pods", "ingestorName", ingest.GetName()) + ingestorPodPrefix := "splunk-" + ingest.GetName() + "-ingestor" + allPods := testenv.DumpGetPods(testcaseEnvInst.GetName()) + for _, pod := range allPods { + if !strings.Contains(pod, ingestorPodPrefix) { + continue + } + testcaseEnvInst.Log.Info("Checking init container on pod", "pod", pod) + podObj := &v1.Pod{} + err = deployment.GetInstance(ctx, pod, podObj) + Expect(err).To(Succeed(), "Failed to get pod", "pod", pod) + foundInitContainer := false + for _, ics := range podObj.Status.InitContainerStatuses { + if ics.Name == "init-ingestor-queue-config" { + foundInitContainer = true + testcaseEnvInst.Log.Info("Init container status", "pod", pod, "container", ics.Name, "ready", ics.Ready, "restartCount", ics.RestartCount) + Expect(ics.Ready).To(BeTrue(), "Init container not ready", "pod", pod, "container", ics.Name) + Expect(ics.RestartCount).To(BeEquivalentTo(0), "Init container restarted", "pod", pod, "container", ics.Name) + } + } + Expect(foundInitContainer).To(BeTrue(), "init-ingestor-queue-config container not found on pod", "pod", pod) + } // Get instance of current Indexer Cluster CR with latest config testcaseEnvInst.Log.Info("Get instance of current Indexer Cluster CR with latest config") @@ -338,37 +372,50 @@ var _ = Describe("indingsep test", func() { defaultsConf := "" if strings.Contains(pod, "ingest") || strings.Contains(pod, "idxc") { + // Select conf paths based on pod type: + // ingestor pods use the ConfigMap-based app path; indexer pods use system/local + var outputsPath, defaultsPath string + if strings.Contains(pod, "ingest") { + outputsPath = "opt/splunk/etc/apps/100-sok-ingestorcluster/local/outputs.conf" + defaultsPath = "opt/splunk/etc/apps/100-sok-ingestorcluster/local/default-mode.conf" + } else { + outputsPath = "opt/splunk/etc/system/local/outputs.conf" + defaultsPath = "opt/splunk/etc/system/local/default-mode.conf" + } + // Verify outputs.conf - testcaseEnvInst.Log.Info("Verify outputs.conf") - outputsPath := "opt/splunk/etc/system/local/outputs.conf" + testcaseEnvInst.Log.Info("Verify outputs.conf", "pod", pod, "path", outputsPath) outputsConf, err := testenv.GetConfFile(pod, outputsPath, deployment.GetName()) - Expect(err).To(Succeed(), "Failed to get outputs.conf from Ingestor Cluster pod") + Expect(err).To(Succeed(), "Failed to get outputs.conf from pod", "pod", pod) + snippetLen := min(80, len(outputsConf)) + testcaseEnvInst.Log.Info("outputs.conf retrieved", "pod", pod, "snippet", outputsConf[:snippetLen]) testenv.ValidateContent(outputsConf, outputs, true) // Verify default-mode.conf - testcaseEnvInst.Log.Info("Verify default-mode.conf") - defaultsPath := "opt/splunk/etc/system/local/default-mode.conf" - defaultsConf, err := testenv.GetConfFile(pod, defaultsPath, deployment.GetName()) - Expect(err).To(Succeed(), "Failed to get default-mode.conf from Ingestor Cluster pod") + testcaseEnvInst.Log.Info("Verify default-mode.conf", "pod", pod, "path", defaultsPath) + defaultsConf, err = testenv.GetConfFile(pod, defaultsPath, deployment.GetName()) + Expect(err).To(Succeed(), "Failed to get default-mode.conf from pod", "pod", pod) + snippetLen = min(80, len(defaultsConf)) + testcaseEnvInst.Log.Info("default-mode.conf retrieved", "pod", pod, "snippet", defaultsConf[:snippetLen]) testenv.ValidateContent(defaultsConf, defaultsAll, true) // Verify AWS env variables - testcaseEnvInst.Log.Info("Verify AWS env variables") + testcaseEnvInst.Log.Info("Verify AWS env variables", "pod", pod) envVars, err := testenv.GetAWSEnv(pod, deployment.GetName()) - Expect(err).To(Succeed(), "Failed to get AWS env variables from Ingestor Cluster pod") + Expect(err).To(Succeed(), "Failed to get AWS env variables from pod", "pod", pod) testenv.ValidateContent(envVars, awsEnvVars, true) } if strings.Contains(pod, "ingest") { - // Verify default-mode.conf - testcaseEnvInst.Log.Info("Verify default-mode.conf") + // Verify ingest-specific default-mode.conf content + testcaseEnvInst.Log.Info("Verify ingest-specific default-mode.conf content", "pod", pod) testenv.ValidateContent(defaultsConf, defaultsIngest, true) } else if strings.Contains(pod, "idxc") { // Verify inputs.conf - testcaseEnvInst.Log.Info("Verify inputs.conf") + testcaseEnvInst.Log.Info("Verify inputs.conf", "pod", pod) inputsPath := "opt/splunk/etc/system/local/inputs.conf" inputsConf, err := testenv.GetConfFile(pod, inputsPath, deployment.GetName()) - Expect(err).To(Succeed(), "Failed to get inputs.conf from Indexer Cluster pod") + Expect(err).To(Succeed(), "Failed to get inputs.conf from Indexer Cluster pod", "pod", pod) testenv.ValidateContent(inputsConf, inputs, true) } } diff --git a/test/testenv/testcaseenv.go b/test/testenv/testcaseenv.go index cb3c8a107..8dd79dd49 100644 --- a/test/testenv/testcaseenv.go +++ b/test/testenv/testcaseenv.go @@ -275,6 +275,32 @@ func (testenv *TestCaseEnv) createNamespace() error { return err } + // Copy ECR pull secret into this namespace if IMAGE_PULL_SECRET is set + if secretName := os.Getenv("IMAGE_PULL_SECRET"); secretName != "" { + srcSecret := &corev1.Secret{} + srcKey := client.ObjectKey{Name: secretName, Namespace: os.Getenv("IMAGE_PULL_SECRET_NAMESPACE")} + if srcKey.Namespace == "" { + srcKey.Namespace = "splunk-operator" + } + if err := testenv.GetKubeClient().Get(context.TODO(), srcKey, srcSecret); err != nil { + testenv.Log.Info("IMAGE_PULL_SECRET not found in source namespace, skipping copy", "secret", secretName, "srcNamespace", srcKey.Namespace) + } else { + dstSecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: srcSecret.Name, + Namespace: testenv.namespace, + }, + Type: srcSecret.Type, + Data: srcSecret.Data, + } + if err := testenv.GetKubeClient().Create(context.TODO(), dstSecret); err != nil { + testenv.Log.Info("Unable to copy IMAGE_PULL_SECRET to namespace", "secret", secretName, "namespace", testenv.namespace, "err", err) + } else { + testenv.Log.Info("Copied IMAGE_PULL_SECRET to namespace", "secret", secretName, "namespace", testenv.namespace) + } + } + } + return nil } @@ -347,45 +373,52 @@ func (testenv *TestCaseEnv) createRoleBinding() error { } func (testenv *TestCaseEnv) attachPVCToOperator(name string) error { - var err error - // volume name which refers to PVC to be attached volumeName := "app-staging" - namespacedName := client.ObjectKey{Name: testenv.operatorName, Namespace: testenv.namespace} - operator := &appsv1.Deployment{} - err = testenv.GetKubeClient().Get(context.TODO(), namespacedName, operator) - if err != nil { - testenv.Log.Error(err, "Unable to get operator", "operator name", testenv.operatorName) - return err - } - - volume := corev1.Volume{ - Name: volumeName, - VolumeSource: corev1.VolumeSource{ - PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ - ClaimName: name, - }, - }, - } - operator.Spec.Template.Spec.Volumes = append(operator.Spec.Template.Spec.Volumes, volume) - - volumeMount := corev1.VolumeMount{ - Name: volumeName, - MountPath: splcommon.AppDownloadVolume, - } + // Retry on 409 Conflict — Kubernetes may mutate the Deployment between Get and Update + for attempt := 0; attempt < 5; attempt++ { + operator := &appsv1.Deployment{} + if err := testenv.GetKubeClient().Get(context.TODO(), namespacedName, operator); err != nil { + testenv.Log.Error(err, "Unable to get operator", "operator name", testenv.operatorName) + return err + } - operator.Spec.Template.Spec.Containers[0].VolumeMounts = append(operator.Spec.Template.Spec.Containers[0].VolumeMounts, volumeMount) + // Only append if not already present (idempotent on retry) + hasVolume := false + for _, v := range operator.Spec.Template.Spec.Volumes { + if v.Name == volumeName { + hasVolume = true + break + } + } + if !hasVolume { + operator.Spec.Template.Spec.Volumes = append(operator.Spec.Template.Spec.Volumes, corev1.Volume{ + Name: volumeName, + VolumeSource: corev1.VolumeSource{ + PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ + ClaimName: name, + }, + }, + }) + operator.Spec.Template.Spec.Containers[0].VolumeMounts = append( + operator.Spec.Template.Spec.Containers[0].VolumeMounts, + corev1.VolumeMount{Name: volumeName, MountPath: splcommon.AppDownloadVolume}, + ) + } - // update the operator deployment now - err = testenv.GetKubeClient().Update(context.TODO(), operator) - if err != nil { - testenv.Log.Error(err, "Unable to update operator", "operator name", testenv.operatorName) - return err + err := testenv.GetKubeClient().Update(context.TODO(), operator) + if err == nil { + return nil + } + if !errors.IsConflict(err) { + testenv.Log.Error(err, "Unable to update operator", "operator name", testenv.operatorName) + return err + } + testenv.Log.Info("Conflict updating operator deployment, retrying", "attempt", attempt+1) } - - return err + return fmt.Errorf("failed to attach PVC to operator %s after retries", testenv.operatorName) } func (testenv *TestCaseEnv) createOperator() error { diff --git a/test/testenv/util.go b/test/testenv/util.go index 366ea3668..d2c36f139 100644 --- a/test/testenv/util.go +++ b/test/testenv/util.go @@ -92,6 +92,7 @@ func newStandalone(name, ns, splunkImage string) *enterpriseApi.Standalone { ImagePullPolicy: "Always", Image: splunkImage, }, + ImagePullSecrets: getImagePullSecrets(), Volumes: []corev1.Volume{}, }, }, @@ -149,6 +150,7 @@ func newLicenseManager(name, ns, licenseConfigMapName, splunkImage string) *ente ImagePullPolicy: "Always", Image: splunkImage, }, + ImagePullSecrets: getImagePullSecrets(), }, }, } @@ -187,6 +189,7 @@ func newLicenseMaster(name, ns, licenseConfigMapName, splunkImage string) *enter ImagePullPolicy: "Always", Image: splunkImage, }, + ImagePullSecrets: getImagePullSecrets(), }, }, } @@ -234,6 +237,7 @@ func newClusterManager(name, ns, licenseManagerName, ansibleConfig, splunkImage ImagePullPolicy: "Always", Image: splunkImage, }, + ImagePullSecrets: getImagePullSecrets(), LicenseManagerRef: corev1.ObjectReference{ Name: licenseManagerRef, }, @@ -270,6 +274,7 @@ func newClusterMaster(name, ns, licenseManagerName, ansibleConfig, splunkImage s ImagePullPolicy: "Always", Image: splunkImage, }, + ImagePullSecrets: getImagePullSecrets(), LicenseManagerRef: corev1.ObjectReference{ Name: licenseManagerRef, }, @@ -307,6 +312,7 @@ func newClusterManagerWithGivenIndexes(name, ns, licenseManagerName, ansibleConf ImagePullPolicy: "Always", Image: splunkImage, }, + ImagePullSecrets: getImagePullSecrets(), LicenseManagerRef: corev1.ObjectReference{ Name: licenseManagerRef, }, @@ -344,6 +350,7 @@ func newClusterMasterWithGivenIndexes(name, ns, licenseManagerName, ansibleConfi ImagePullPolicy: "Always", Image: splunkImage, }, + ImagePullSecrets: getImagePullSecrets(), LicenseManagerRef: corev1.ObjectReference{ Name: licenseManagerRef, }, @@ -382,6 +389,7 @@ func newIndexerCluster(name, ns, licenseManagerName string, replicas int, cluste ImagePullPolicy: "Always", Image: splunkImage, }, + ImagePullSecrets: getImagePullSecrets(), ClusterManagerRef: corev1.ObjectReference{ Name: clusterManagerRef, }, @@ -425,6 +433,7 @@ func newIngestorCluster(name, ns string, replicas int, splunkImage string, queue ImagePullPolicy: "Always", Image: splunkImage, }, + ImagePullSecrets: getImagePullSecrets(), }, Replicas: int32(replicas), QueueRef: queue, @@ -483,6 +492,7 @@ func newSearchHeadCluster(name, ns, clusterManagerRef, licenseManagerName, ansib ImagePullPolicy: "Always", Image: splunkImage, }, + ImagePullSecrets: getImagePullSecrets(), ClusterManagerRef: corev1.ObjectReference{ Name: clusterManagerRef, }, @@ -632,6 +642,7 @@ func newOperator(name, ns, account, operatorImageAndTag, splunkEnterpriseImageAn FSGroup: &OperatorFSGroup, }, ServiceAccountName: account, + ImagePullSecrets: getImagePullSecrets(), Containers: []corev1.Container{ { Name: name, @@ -673,6 +684,14 @@ func newOperator(name, ns, account, operatorImageAndTag, splunkEnterpriseImageAn return &operator } +// getImagePullSecrets returns imagePullSecrets for operator pods when IMAGE_PULL_SECRET env var is set. +func getImagePullSecrets() []corev1.LocalObjectReference { + if secret := os.Getenv("IMAGE_PULL_SECRET"); secret != "" { + return []corev1.LocalObjectReference{{Name: secret}} + } + return nil +} + // newStandaloneWithLM creates and initializes CR for Standalone Kind with License Manager func newStandaloneWithLM(name, ns, licenseManagerName, splunkImage string) *enterpriseApi.Standalone { @@ -694,6 +713,7 @@ func newStandaloneWithLM(name, ns, licenseManagerName, splunkImage string) *ente ImagePullPolicy: "Always", Image: splunkImage, }, + ImagePullSecrets: getImagePullSecrets(), LicenseManagerRef: corev1.ObjectReference{ Name: licenseManagerRef, }, @@ -764,6 +784,7 @@ func newMonitoringConsoleSpec(name, ns, LicenseManagerRef, splunkImage string) * ImagePullPolicy: "Always", Image: splunkImage, }, + ImagePullSecrets: getImagePullSecrets(), LicenseManagerRef: corev1.ObjectReference{ Name: licenseManagerRef, }, @@ -993,6 +1014,9 @@ func GetConfigMap(ctx context.Context, deployment *Deployment, ns string, config // newClusterManagerWithGivenSpec creates and initialize the CR for ClusterManager Kind func newClusterManagerWithGivenSpec(name string, ns string, spec enterpriseApi.ClusterManagerSpec) *enterpriseApi.ClusterManager { + if len(spec.CommonSplunkSpec.ImagePullSecrets) == 0 { + spec.CommonSplunkSpec.ImagePullSecrets = getImagePullSecrets() + } new := enterpriseApi.ClusterManager{ TypeMeta: metav1.TypeMeta{ Kind: "ClusterManager",