From 1a7a45cf7d92f0e1f8bffc54d18b92c2f6768eae Mon Sep 17 00:00:00 2001
From: Daniel Mepham <daniel.mepham@cloudzero.com>
Date: Thu, 14 Nov 2024 07:15:09 -0500
Subject: [PATCH 01/26] CP-22731: add insights-controller chart (#97)

* CP-22731: include cz-insights-controller as subchart

* increase replicacount for tag server

* CP-22731: add beta testing

* update release process for insights controller

* update release workflow

* make most resources off by default

* update readme

* use global for secret names

* incorporate changes from 0.0.30-beta

* add beta release doc

* use local chart for testing

---------

Co-authored-by: josephbarnett <joe.barnett@cloudzero.com>
---
 .../build-and-publish-beta-chart.yml          |  25 +-
 .github/workflows/build-and-publish-chart.yml |  28 ++-
 .github/workflows/test-chart.yml              |  30 ++-
 charts/cloudzero-agent/Chart.lock             |  10 +-
 charts/cloudzero-agent/Chart.yaml             |   9 +-
 charts/cloudzero-agent/README.md              | 224 ++++++------------
 .../configuration.example.yaml                |  28 +++
 .../docs/releases/1.0.0-beta.md               |  53 +++++
 charts/cloudzero-agent/templates/_helpers.tpl |   8 +-
 charts/cloudzero-agent/templates/cm.yaml      |  20 ++
 charts/cloudzero-agent/templates/deploy.yaml  |   9 +-
 charts/cloudzero-agent/templates/secret.yaml  |   4 +-
 .../templates/validatorcm.yaml                |   2 +-
 charts/cloudzero-agent/values.yaml            |  35 ++-
 .../cloudzero-insights-controller/Chart.lock  |   6 +
 .../cloudzero-insights-controller/Chart.yaml  |  12 +
 .../cloudzero-insights-controller/README.md   | 145 ++++++++++++
 .../templates/_helpers.tpl                    | 206 ++++++++++++++++
 .../templates/certificate.yaml                |  28 +++
 .../templates/clusterrole.yaml                |  14 ++
 .../templates/clusterrolebinding.yaml         |  14 ++
 .../templates/cm.yaml                         |  64 +++++
 .../templates/deploy.yaml                     | 107 +++++++++
 .../templates/init-job.yaml                   |  27 +++
 .../templates/issuer.yaml                     |  10 +
 .../templates/service.yaml                    |  15 ++
 .../templates/serviceaccount.yaml             |  13 +
 .../templates/webhooks.yaml                   |  32 +++
 .../cloudzero-insights-controller/values.yaml | 112 +++++++++
 29 files changed, 1102 insertions(+), 188 deletions(-)
 create mode 100644 charts/cloudzero-agent/configuration.example.yaml
 create mode 100644 charts/cloudzero-agent/docs/releases/1.0.0-beta.md
 create mode 100644 charts/cloudzero-insights-controller/Chart.lock
 create mode 100644 charts/cloudzero-insights-controller/Chart.yaml
 create mode 100644 charts/cloudzero-insights-controller/README.md
 create mode 100644 charts/cloudzero-insights-controller/templates/_helpers.tpl
 create mode 100644 charts/cloudzero-insights-controller/templates/certificate.yaml
 create mode 100644 charts/cloudzero-insights-controller/templates/clusterrole.yaml
 create mode 100644 charts/cloudzero-insights-controller/templates/clusterrolebinding.yaml
 create mode 100644 charts/cloudzero-insights-controller/templates/cm.yaml
 create mode 100644 charts/cloudzero-insights-controller/templates/deploy.yaml
 create mode 100644 charts/cloudzero-insights-controller/templates/init-job.yaml
 create mode 100644 charts/cloudzero-insights-controller/templates/issuer.yaml
 create mode 100644 charts/cloudzero-insights-controller/templates/service.yaml
 create mode 100644 charts/cloudzero-insights-controller/templates/serviceaccount.yaml
 create mode 100644 charts/cloudzero-insights-controller/templates/webhooks.yaml
 create mode 100644 charts/cloudzero-insights-controller/values.yaml

diff --git a/.github/workflows/build-and-publish-beta-chart.yml b/.github/workflows/build-and-publish-beta-chart.yml
index 9acf2509..ba54b316 100644
--- a/.github/workflows/build-and-publish-beta-chart.yml
+++ b/.github/workflows/build-and-publish-beta-chart.yml
@@ -42,9 +42,18 @@ jobs:
       - name: Install Helm
         uses: azure/setup-helm@v3
 
-      - name: Build Dependencies
+      - name: Build Insights Controller Dependencies
+        run: helm dependency update charts/cloudzero-insights-controller/
+
+      - name: Package Insights Controller Chart
+        run: helm package charts/cloudzero-insights-controller/ --destination .deploy
+
+      - name: Build Cloudzero Agent Dependencies
         run: helm dependency update charts/cloudzero-agent/
 
+      - name: Package Cloudzero Agent Chart
+        run: helm package charts/cloudzero-agent/ --destination .deploy
+
       # Step 3: Determine Version
       - name: Get Github Tag Version
         id: version
@@ -63,6 +72,11 @@ jobs:
           VERSION_LINE=$(awk '/version:/ && !done {print NR; done=1}' charts/cloudzero-agent/Chart.yaml)
           sed -i ''$VERSION_LINE's/.*/version: ${{ env.NEW_VERSION }}/' charts/cloudzero-agent/Chart.yaml
 
+      - name: Update Chart Version for insights-controller
+        run: |
+          VERSION_LINE=$(awk '/version:/ && !done {print NR; done=1}' charts/cloudzero-insights-controller/Chart.yaml)
+          sed -i ''$VERSION_LINE's/.*/version: ${{ env.NEW_VERSION }}/' charts/cloudzero-insights-controller/Chart.yaml
+
       - name: Validate Release Notes are Present
         run: |
           if [ ! -f "docs/releases/${{ env.NEW_VERSION }}.md" ]; then
@@ -84,7 +98,13 @@ jobs:
         continue-on-error: true
 
       # Step 7: Handle Artifacts and Update Pages
-      - name: Upload Chart as Artifact
+      - name: Upload Insight Controller Chart as Artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: agent-chart
+          path: .deploy/cloudzero-insights-controller-${{ env.NEW_VERSION }}.tgz
+
+      - name: Upload Cloudzero Agent Chart as Artifact
         uses: actions/upload-artifact@v4
         with:
           name: agent-chart
@@ -97,6 +117,7 @@ jobs:
 
       - name: Move release Tarball
         run: |
+          cp .deploy/cloudzero-insights-controller-${{ env.NEW_VERSION }}.tgz ./beta/
           cp .deploy/cloudzero-agent-${{ env.NEW_VERSION }}.tgz ./beta/
           rm -fr .deploy
 
diff --git a/.github/workflows/build-and-publish-chart.yml b/.github/workflows/build-and-publish-chart.yml
index 0fcdb838..74b50aa8 100644
--- a/.github/workflows/build-and-publish-chart.yml
+++ b/.github/workflows/build-and-publish-chart.yml
@@ -66,6 +66,11 @@ jobs:
           VERSION_LINE=$(awk '/version:/ && !done {print NR; done=1}' charts/cloudzero-agent/Chart.yaml)
           sed -i ''$VERSION_LINE's/.*/version: ${{ env.NEW_VERSION }}/' charts/cloudzero-agent/Chart.yaml
 
+      - name: Update Chart Version for insights-controller
+        run: |
+          VERSION_LINE=$(awk '/version:/ && !done {print NR; done=1}' charts/cloudzero-insights-controller/Chart.yaml)
+          sed -i ''$VERSION_LINE's/.*/version: ${{ env.NEW_VERSION }}/' charts/cloudzero-insights-controller/Chart.yaml
+
       - name: Validate Release Notes are Present
         run: |
           if [ ! -f "charts/cloudzero-agent/docs/releases/${{ env.NEW_VERSION }}.md" ]; then
@@ -73,11 +78,16 @@ jobs:
             exit 1
           fi
 
-      - name: Build Dependencies
+      - name: Build Insights Controller Dependencies
+        run: helm dependency update charts/cloudzero-insights-controller/
+
+      - name: Package Insights Controller Chart
+        run: helm package charts/cloudzero-insights-controller/ --destination .deploy
+
+      - name: Build Cloudzero Agent Dependencies
         run: helm dependency update charts/cloudzero-agent/
 
-      # Step 5: Package and Commit Chart
-      - name: Package Chart
+      - name: Package Cloudzero Agent Chart
         run: helm package charts/cloudzero-agent/ --destination .deploy
 
       - name: Get Main Changelog Beginning Hash
@@ -104,18 +114,26 @@ jobs:
           echo "::set-output name=changes::${CHANGES}"
 
       # Step 7: Handle Artifacts and Update Pages
-      - name: Upload Chart as Artifact
+      - name: Upload Insight Controller Chart as Artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: agent-chart
+          path: .deploy/cloudzero-insights-controller-${{ env.NEW_VERSION }}.tgz
+
+      - name: Upload Cloudzero Agent Chart as Artifact
         uses: actions/upload-artifact@v4
         with:
           name: agent-chart
           path: .deploy/cloudzero-agent-${{ env.NEW_VERSION }}.tgz
 
+
       - name: Checkout GH Pages
         run: |
           git checkout -f gh-pages
 
       - name: Move release Tarball
         run: |
+          cp .deploy/cloudzero-insights-controller-${{ env.NEW_VERSION }}.tgz ./
           cp .deploy/cloudzero-agent-${{ env.NEW_VERSION }}.tgz ./
           rm -fr .deploy
 
@@ -125,7 +143,7 @@ jobs:
       - name: Save Index in GH Pages
         run: |
           # copy the new chart and index.yaml
-          git add cloudzero-agent-${{ env.NEW_VERSION }}.tgz index.yaml 
+          git add cloudzero-insights-controller-${{ env.NEW_VERSION }}.tgz cloudzero-agent-${{ env.NEW_VERSION }}.tgz index.yaml 
           git commit -m "Updating ${{ env.NEW_VERSION }} Index"
           git push origin gh-pages
         continue-on-error: true
diff --git a/.github/workflows/test-chart.yml b/.github/workflows/test-chart.yml
index fab253ee..47470911 100644
--- a/.github/workflows/test-chart.yml
+++ b/.github/workflows/test-chart.yml
@@ -36,6 +36,8 @@ jobs:
         env:
           # Agent Chart settings (prom repo is to work around issue with chart-testing tool)
           PROM_CHART_REPO: https://prometheus-community.github.io/helm-charts
+          CZ_CHART_REPO: https://cloudzero.github.io/cloudzero-charts
+          CZ_CHART_BETA_REPO: https://cloudzero.github.io/cloudzero-charts/beta
           CLUSTER_NAME: cz-node-agent-ci
           CLOUD_ACCOUNT_ID: '00000000'
           CZ_API_TOKEN: 'fake-api-token'
@@ -46,6 +48,7 @@ jobs:
           ct lint --debug --charts . \
             --chart-repos=kube-state-metrics=$PROM_CHART_REPO \
             --chart-repos=prometheus-node-exporter=$PROM_CHART_REPO \
+            --chart-repos=cloudzero-insights-controller=$CZ_CHART_BETA_REPO \
             --helm-lint-extra-args "--set=existingSecretName=api-token,clusterName=$CLUSTER_NAME,cloudAccountId=$CLOUD_ACCOUNT_ID,region=$REGION"
 
   # This job tests the chart on a KinD cluster
@@ -100,6 +103,18 @@ jobs:
             echo "SKIP_VALIDATIONS=true" >>${GITHUB_ENV}
           fi
 
+      - name: TEST PREP - Update dependency path for local insights-controller testing
+        run: |
+          full_path=$(pwd)/charts/cloudzero-insights-controller
+          awk -v dir="$full_path" '
+            /name: cloudzero-insights-controller/ {flag=1}
+            flag && /repository:/ {sub(/repository: .*/, "repository: file://" dir); flag=0}
+            {print}
+          ' charts/cloudzero-agent/Chart.yaml > temp.yaml && mv temp.yaml charts/cloudzero-agent/Chart.yaml
+          cd charts/cloudzero-insights-controller
+          helm dependency update
+          cd ../../
+
       # Install the chart using our temporary image
       - name: TEST - Install the chart
         id: test_chart_installation
@@ -108,11 +123,14 @@ jobs:
           NAMESPACE: monitoring
           # Agent Chart settings (prom repo is to work around issue with chart-testing tool)
           PROM_CHART_REPO: https://prometheus-community.github.io/helm-charts
+          CZ_CHART_REPO: https://cloudzero.github.io/cloudzero-charts
+          CZ_CHART_BETA_REPO: https://cloudzero.github.io/cloudzero-charts/beta
           CLUSTER_NAME: cz-node-agent-ci
           CLOUD_ACCOUNT_ID: '00000000'
           CZ_API_TOKEN: ${{ secrets.CZ_API_TOKEN || 'fake-api-token' }}
           REGION: 'us-east-1'
-        run: |  
+        run: |
+          kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.crds.yaml
           kubectl create namespace $NAMESPACE
           kubectl create secret -n $NAMESPACE generic api-token --from-literal=value=$CZ_API_TOKEN
           cd charts/cloudzero-agent
@@ -120,11 +138,17 @@ jobs:
           ct install --charts . \
             --chart-repos=kube-state-metrics=$PROM_CHART_REPO \
             --chart-repos=prometheus-node-exporter=$PROM_CHART_REPO \
+            --chart-repos=cloudzero-insights-controller=$CZ_CHART_BETA_REPO \
             --namespace $NAMESPACE \
             --helm-extra-set-args "\
-              --set=existingSecretName=api-token \
+              --set=global.existingSecretName=api-token \
+              --set=host=dev-api.cloudzero.com \
               --set=clusterName=$CLUSTER_NAME \
               --set=cloudAccountId=$CLOUD_ACCOUNT_ID \
               --set=region=$REGION \
               --set=kube-state-metrics.enabled=true \
-              --set=prometheus-node-exporter.enabled=true"
+              --set=prometheus-node-exporter.enabled=true \
+              --set=tags.enabled=true \
+              --set=tags.labels.enabled=true \
+              --set=tags.labels.patterns[0]='.*' \
+              "
diff --git a/charts/cloudzero-agent/Chart.lock b/charts/cloudzero-agent/Chart.lock
index a2094f70..e171ce42 100644
--- a/charts/cloudzero-agent/Chart.lock
+++ b/charts/cloudzero-agent/Chart.lock
@@ -2,8 +2,8 @@ dependencies:
 - name: kube-state-metrics
   repository: https://prometheus-community.github.io/helm-charts
   version: 5.15.3
-- name: prometheus-node-exporter
-  repository: https://prometheus-community.github.io/helm-charts
-  version: 4.24.0
-digest: sha256:827a33fa07fde17be0bf808e0beba3ca7b23c9fc1960580b2ba6d0ecc0b57a3f
-generated: "2024-03-20T11:42:44.034766-04:00"
+- name: cloudzero-insights-controller
+  repository: https://cloudzero.github.io/cloudzero-charts/beta
+  version: 0.0.1
+digest: sha256:87c19ba797e44296e4ac2433e17f24b73bcda8c9ee8b4286bc31c0d7fd0f0224
+generated: "2024-11-13T12:34:06.381064+01:00"
diff --git a/charts/cloudzero-agent/Chart.yaml b/charts/cloudzero-agent/Chart.yaml
index d110bab5..5585449e 100644
--- a/charts/cloudzero-agent/Chart.yaml
+++ b/charts/cloudzero-agent/Chart.yaml
@@ -12,7 +12,8 @@ dependencies:
     version: "5.15.*"
     repository: https://prometheus-community.github.io/helm-charts
     condition: kube-state-metrics.enabled
-  - name: prometheus-node-exporter
-    version: "4.24.*"
-    repository: https://prometheus-community.github.io/helm-charts
-    condition: prometheus-node-exporter.enabled
+  - name: cloudzero-insights-controller
+    version: "0.0.1"
+    repository: https://cloudzero.github.io/cloudzero-charts/beta
+    condition: cloudzero-insights-controller.enabled
+    alias: tags
diff --git a/charts/cloudzero-agent/README.md b/charts/cloudzero-agent/README.md
index ccb295cc..a544bb51 100644
--- a/charts/cloudzero-agent/README.md
+++ b/charts/cloudzero-agent/README.md
@@ -31,16 +31,56 @@ _See [`helm repo`](https://helm.sh/docs/helm/helm_repo/) for command documentati
 
 The chart can be installed directly with Helm or any other common Kubernetes deployment tools.
 
-If installing with Helm directly, the following command will install the chart:
+If installing with Helm directly, execute the following steps:
 
+1. Ensure that the most recent chart version is available:
+```console
+helm repo update
+```
+
+2. Ensure that required CRDs are installed for certifiacte management. If you have more specific requirements around managing TLS certificates, see the [Certificate Management](https://github.com/Cloudzero/cloudzero-charts/tree/develop/charts/cloudzero-insights-controller#deployment-configurations-and-certificate-management) section in the `cloudzero-insights-controller` subchart.
 ```console
 helm install <RELEASE_NAME> cloudzero/cloudzero-agent \
-    --set existingSecretName=<NAME_OF_SECRET> \
-    --set clusterName=<CLUSTER_NAME> \
-    --set-string cloudAccountId=<CLOUD_ACCOUNT_ID> \
-    --set region=<REGION> \
-    # optionally deploy kube-state-metrics if it doesn't exist in the cluster already
-    --set kube-state-metrics.enabled=<true|false>
+    --set tags.webhook.issuer.enabled=false \
+    --set tags.webhook.certificate.enabled=false \
+    --set tags.cert-manager.installCRDs=true
+```
+
+3. Fill out all required fields in the `configuration.example.yaml` file in this directory. Rename the file as necessary. Below is an example of a completed configuration file:
+```yaml
+# -- Account ID of the account the cluster is running in. This must be a string - even if it is a number in your system.
+cloudAccountId: YOUR_CLOUD_ACCOUNT_ID
+# -- Name of the clusters.
+clusterName: YOUR_CLUSTER_NAME
+# -- Region the cluster is running in.
+region: YOUR_CLOUD_REGION
+global:
+  # -- CloudZero API key. Required if useExistingSecret is false.
+  apiKey: YOUR_CLOUDZERO_API_KEY
+  # -- If set, the agent will use the API key in this Secret to authenticate with CloudZero.
+  existingSecretName: YOUR_EXISTING_API_KEY_K8S_SECRET
+
+# label and annotation configuration (referred together as 'tags'). See the below 'Labels and Annotations' section for more details.
+tags:
+  # -- By default, a ValidatingAdmissionWebhook will be deployed that records all created labels and annotations
+  enabled: true
+  labels:
+    # -- This value MUST be set to either true or false. The installation will fail otherwise
+    enabled: true
+    # -- This value MUST be set to a list of regular expressions which will be used to gather labels from pods, deployments, statefulsets, daemonsets, cronjobs, jobs, nodes, and namespaces
+    patterns:
+      - '^foo' # -- match all labels whose key starts with "foo"
+      - 'bar$' # -- match all labels whose key ends with "bar"
+  annotations:
+    # -- By default, the gathering of annotations is not enabled. To enable, set this field to true
+    enabled: false
+    patterns:
+      - '.*'
+```
+
+4. Install the helm chart using the completed configuration file:
+```console
+helm install <RELEASE_NAME> cloudzero/cloudzero-agent -f configuration.example.yaml
 ```
 
 ### Update Helm Chart
@@ -53,12 +93,7 @@ helm repo update
 Next, upgrade the installation to the latest chart version:
 
 ```console
-helm upgrade <RELEASE_NAME> cloudzero/cloudzero-agent \
-    --set existingSecretName=<NAME_OF_SECRET> \
-    --set clusterName=<CLUSTER_NAME> \
-    --set-string cloudAccountId=<CLOUD_ACCOUNT_ID> \
-    --set region=<REGION> \
-    --set kube-state-metrics.enabled=<true|false>
+helm upgrade --install <RELEASE_NAME> cloudzero/cloudzero-agent -f configuration.example.yaml
 ```
 
 ### Mandatory Values
@@ -70,13 +105,15 @@ There are several mandatory values that must be specified for the chart to insta
 | cloudAccountId    | string | `nil`                 | Account ID in AWS or Subscription ID in Azure or Project Number in GCP where the cluster is running. Must be a string due to Helm limitations.  |
 | clusterName       | string | `nil`                 | Name of the cluster. Must be RFC 1123 compliant.                                                                         |
 | host              | string | `"api.cloudzero.com"` | CloudZero host to send metrics to.                                                                                      |
-| apiKey            | string | `nil`                 | The CloudZero API key to use for exporting metrics. Only used if `existingSecretName` is not set.                       |
-| existingSecretName| string | `nil`                 | Name of the secret that contains the CloudZero API key. Required if not providing the API key via `apiKey`.             |
+| global.apiKey            | string | `nil`                 | The CloudZero API key to use for exporting metrics. Only used if `global.existingSecretName` is not set.                       |
+| global.existingSecretName| string | `nil`                 | Name of the secret that contains the CloudZero API key. Required if not providing the API key via `apiKey`.             |
 | region            | string | `nil`                 | Region where the cluster is running (e.g., `us-east-1`, `eastus`). For more information, see AWS or Azure documentation. |
+| tags.labels.enabled            | string | `nil`                 | If enabled, labels for pods, deployments, statefulsets, daemonsets, cronjobs, jobs, nodes, and namespaces |
+| tags.labels.patterns            | string | `nil`                 | An array of regular expressions, which are used to match specific label keys |
 
 #### Overriding Default Values
 
-Default values are specified in the chart's `values.yaml` file. If you need to change any of these values, it is recommended to create a `values-override.yaml` file for your customizations.
+Default values are specified in the chart's `values.yaml` file. If you need to change any of these values, it is recommended to create a `values-override.yaml` based on the `configuration.example.yaml`  file for your customizations.
 
 ##### Using the `--values` Flag
 
@@ -84,10 +121,6 @@ You can use the `--values` (or short form `-f`) flag in your Helm commands to ov
 
 ```console
 helm install <RELEASE_NAME> cloudzero/cloudzero-agent \
-    --set existingSecretName=<NAME_OF_SECRET> \
-    --set clusterName=<CLUSTER_NAME> \
-    --set-string cloudAccountId=<CLOUD_ACCOUNT_ID> \
-    --set region=<REGION> \
     -f values-override.yaml
 ```
 
@@ -101,7 +134,7 @@ You can use the `--set` flag in Helm commands to directly set or override specif
 
 ```console
 helm install <RELEASE_NAME> cloudzero/cloudzero-agent \
-    --set existingSecretName=<NAME_OF_SECRET> \
+    --set global.existingSecretName=<NAME_OF_SECRET> \
     --set clusterName=<CLUSTER_NAME> \
     --set-string cloudAccountId=<CLOUD_ACCOUNT_ID> \
     --set region=<REGION> \
@@ -109,32 +142,24 @@ helm install <RELEASE_NAME> cloudzero/cloudzero-agent \
     -f values-override.yaml
 ```
 
-### Metric Exporters
-
-This chart depends on metrics from [kube-state-metrics](https://github.com/kubernetes/kube-state-metrics). There are two installation options for providing the `kube-state-metrics` metrics to the cloudzero-agent. If you don't know which option is right for you, use the second option.
-
-#### Option 1 (default): Use existing kube-state-metrics
-
-Using an existing `kube-state-metrics` exporter may be desirable for minimizing cost. By default, the `cloudzero-agent` will attempt to find an existing `kube-state-metrics` K8s Service by searching for a K8s Service with the annotation `prometheus.io/scrape: "true"`. If an existing `kube-state-metrics` Service exists but does not have that annotation and you do not wish to add it, see the **Custom Scrape Configs** section below.
-
-In addition to the above, the existing `kube-state-metrics` Service address should be added in `values-override.yaml` as shown below so that the `cloudzero-agent` can validate the connection:
-
-```yaml
-validator:
-  serviceEndpoints:
-     kubeStateMetrics: <kube-state-metrics>.<example-namespace>.svc.cluster.local:8080
-```
-
-
-#### Option 2: Use kube-state-metrics subchart
-
-Alternatively, deploy the `kube-state-metrics` subchart that comes packaged with this chart. This is done by enabling settings in `values-override.yaml` as shown:
-
-```yaml
-kube-state-metrics:
-  enabled: true
-```
-In this option, no additional configuration is required in the `validator` field.
+### Labels and Annotations
+
+This chart allows the exporting of labels and annotations from the following resources:
+- `Pod`
+- `Deployment`
+- `StatefulSet`
+- `Daemonset`
+- `Job`
+- `CronJob`
+- `Node`
+- `Namespace`
+
+Additional Notes:
+- By default, only labels from pods and namespaces are exported. To enabled more resources, see the `webhooks.configurations` section of the `values.yaml` file.
+- Labels and annotations exports are managed by a subchart, `cloudzero-insights-controller`, which is also maintained in this repository.
+- To disambiguate labels/annotations between resources, a prefix representing the resource type is prepended to the label key in the Explorer page. For example, a `foo=bar` node label would be presented as `node:foo: bar`. The exception is pod labels which do not have resource prefixes for backward compatibility with previous versions.
+- Annotations are not exported by default; see the `tags.annotations.enabled` setting to enable. To disambiguate annotations from labels, an `annotation` prefix is prepended to the annotation key; i.e., an `foo: bar` annotation on a namespace would be represented in the Explorer as `node:annotation:foo: bar`
+- For both labels and annotations, the `enabled` flag applies across all resource types; i.e., setting `true` for `tags.labels.enabled` enabels label exporting for labels on pods, nodes, deployments, statefulsets, namespaces, daemonets, jobs, and cronjobs. Specific resources can be disabled by altering the `webhooks.configurations` configuration.
 
 ### Secret Management
 
@@ -153,7 +178,7 @@ Example of creating a secret:
 kubectl create secret -n example-namespace generic example-secret-name --from-literal=value=<example-api-key-value>
 ```
 
-The secret can then be used with `existingSecretName`.
+The secret can then be used with `global.existingSecretName`.
 
 ### Memory Sizing
 
@@ -174,68 +199,6 @@ kube-state-metrics:
     repository: my-custom-kube-state-metrics/kube-state-metrics
 ```
 
-### Custom Scrape Configs
-
-If running without the default `kube-state-metrics` exporter subchart and your existing `kube-state-metrics` deployment does not have the required `prometheus.io/scrape: "true"`, adjust the Prometheus scrape configs as shown:
-
-`values-override.yaml`
-```yaml
-prometheusConfig:
-  scrapeJobs:
-    kubeStateMetrics:
-      enabled: false # this disables the default kube-state-metrics scrape job, which will be replaced by an entry in additionalScrapeJobs
-    additionalScrapeJobs:
-    - job_name: custom-kube-state-metrics
-      honor_timestamps: true
-      scrape_interval: 1m
-      scrape_timeout: 10s
-      metrics_path: /metrics
-      static_configs:
-        - targets:
-          - 'my-kube-state-metrics-service.default.svc.cluster.local:8080'
-      relabel_configs:
-      - separator: ;
-        regex: __meta_kubernetes_service_label_(.+)
-        replacement: $1
-        action: labelmap
-      - source_labels: [__meta_kubernetes_namespace]
-        separator: ;
-        regex: (.*)
-        target_label: namespace
-        replacement: $1
-        action: replace
-      - source_labels: [__meta_kubernetes_service_name]
-        separator: ;
-        regex: (.*)
-        target_label: service
-        replacement: $1
-        action: replace
-      - source_labels: [__meta_kubernetes_pod_node_name]
-        separator: ;
-        regex: (.*)
-        target_label: node
-        replacement: $1
-        action: replace
-```
-
-### Exporting Pod Labels
-
-Pod labels can be exported as metrics using kube-state-metrics. To customize the labels for export, modify the values-override.yaml file as shown below:
-
-**Example: Exporting only the pod labels named foo and bar:**
-
-```yaml
-kube-state-metrics:
-  extraArgs:
-     - --metric-labels-allowlist=pods=[foo,bar]
-```
-
-> This is preferable to including all labels with `*` because the performance and memory impact is reduced. Regular expression matching is not currently supported. See the `kube-state-metrics` [documentation](https://github.com/kubernetes/kube-state-metrics/blob/main/docs/developer/cli-arguments.md) for more details.
-
-⚠️ Important: If you are running an existing `kube-state-metrics` instance, ensure that the labels you want to use are whitelisted. kube-state-metrics version 2.x and above will **_not_** export the `kube_pod_labels` metrics unless they are explicitly allowed. This prevents the use of those labels for cost allocation and other purposes. Make sure you have configured the labels at the appropriate level using the --metric-labels-allowlist parameter:
-
-> eg:  `- --metric-labels-allowlist=namespaces=[*],pods=[*],deployments=[app.kubernetes.io/*,k8s.*]`
-
 ## Dependencies
 
 | Repository                                         | Name                     | Version |
@@ -251,47 +214,6 @@ To receive a notification when a new version of the chart is [released](https://
 3. Check the **Releases** box.
 4. Select **Apply**.
 
-
-## Troubleshooting
-
-### Issue
-I've deployed the chart, but I don't see Kubernetes data in CloudZero.
-
-## Resolution
-This can happen for a number of reasons; see below for solutions to the most common problems
-
-### Ensure kube-state-metrics is deployed correctly
-
-1. Review the **Metric Exporters** section.
-2. If opting for **Option 1**
-  - Is kube-state-metrics installed?
-  ```bash
-  kubectl get services --all-namespaces | grep kube-state-metrics
-  ```
-  If the above command does not return any services, install a `kube-state-metrics` exporter, or use **Option 2** in the **Metric Exporters** section.
-  
-3. If opting for **Option 2**, ensure that `kube-state-metrics.enabled=true` is set as an annotation on the Service.
-4. Ensure the cloudzero-agent pod can find the `kube-state-metrics` Service.
-   Run the following command:
-   ```
-   kubectl get services -A -o jsonpath='{range .items[?(@.metadata.annotations.prometheus\.io/scrape=="true")]}{.metadata.name}{" in "}{.metadata.namespace}{"\n"}{end}'
-   ```
-   If this does not return a `kube-state-metrics` Service, then either annotate the existing Service found in Step 2 with `prometheus.io/scrape: "true"`, or following the instructions in the **Custom Scrape Configs** section above.
-5. Ensure connectivity between the `cloudzero-agent` pod and the `kube-state-metrics` Service.
-  ```
-  SERVER_POD=$(kubectl get pod -l app.kubernetes.io/name=cloudzero-agent -o jsonpath='{.items[0].metadata.name}')
-  kubectl exec -it -n <NAMESPACE> $SERVER_POD -- wget -qO- <KSM_SERVICE_NAME>.<KSM_NAMESPACE>.svc.cluster.local:8080/metrics
-  ```
-  The request should return a 200 response with a list of metrics prefixed with `kube_`, i.e., `kube_pod_info`. If not, ensure that the `kube-state-metrics` deployment is configured correctly.
-
-### Issue
-I have Kubernetes data in CloudZero, but I don't see Kubernetes labels as Dimensions.
-
-## Resolution
-Note that
-1. Only labels on Pods are currently supported, and
-2. Labels are "opt-in"; see the **Exporting Pod Labels** section for details.
-
 ## Useful References
 
 - [Memory Sizing Guide](./docs/sizing-guide.md)
diff --git a/charts/cloudzero-agent/configuration.example.yaml b/charts/cloudzero-agent/configuration.example.yaml
new file mode 100644
index 00000000..6d9bb8b6
--- /dev/null
+++ b/charts/cloudzero-agent/configuration.example.yaml
@@ -0,0 +1,28 @@
+# -- Account ID of the account the cluster is running in. This must be a string - even if it is a number in your system.
+cloudAccountId: null
+# -- Name of the clusters.
+clusterName: null
+# -- Region the cluster is running in.
+region: null
+
+global:
+  # -- CloudZero API key. Required if useExistingSecret is false.
+  apiKey: null
+  # -- If set, the agent will use the API key in this Secret to authenticate with CloudZero.
+  existingSecretName: null
+
+# label and annotation configuration:
+tags:
+  # -- By default, a ValidatingAdmissionWebhook will be deployed that records all created labels and annotations
+  enabled: true
+  labels:
+    # -- This value MUST be set to either true or false. The installation will fail otherwise
+    enabled: null
+    # -- This value MUST be set to a list of regular expressions which will be used to gather labels from pods, deployments, statefulsets, daemonsets, cronjobs, jobs, nodes, and namespaces
+    patterns:
+      # - '.*' # -- This option enables gathering ALL labels from the above resources. Use with caution, as the number of labels can be large.
+  annotations:
+    # -- By default, the gathering of annotations is not enabled. To enable, set this field to true
+    enabled: false
+    patterns:
+      - '.*'
diff --git a/charts/cloudzero-agent/docs/releases/1.0.0-beta.md b/charts/cloudzero-agent/docs/releases/1.0.0-beta.md
new file mode 100644
index 00000000..cf4321c5
--- /dev/null
+++ b/charts/cloudzero-agent/docs/releases/1.0.0-beta.md
@@ -0,0 +1,53 @@
+## [1.0.0](https://github.com/cloudzero/cloudzero-charts/compare/v0.0.29...v1.0.0) (2024-11-12)
+
+Adds a subchart, `cloudzero-insights-controller`, that allows the chart to gather labels and annotations from a variety of Kubernetes resources.
+
+### Upgrade Steps
+Upgrading to the `1.0.0-beta` version requires the following migration steps of settings from previous versions:
+* Note that this is a beta release; follow the instructions in the [beta-installation](https://github.com/Cloudzero/cloudzero-charts/blob/develop/charts/cloudzero-agent/BETA-INSTALLATION.md#adding-the-beta-helm-repository) document.
+* Set the argument `tags.labels.enabled=true|false` if exporting labels for k8s resources. See [Install Helm Chart](https://github.com/Cloudzero/cloudzero-charts/tree/develop/charts/cloudzero-agent#install-helm-chart) for details.
+* Move `apiKey` or `existingSecretName` arguments to `global.apiKey` or `global.existingSecretName`.
+
+An example `configuration-example.yaml` file:
+```yaml
+# unchaged:
+cloudAccountId: YOUR_CLOUD_ACCOUNT_ID
+clusterName: YOUR_CLUSTER_NAME
+region: YOUR_CLOUD_REGION
+
+# changed
+#   apiKey: YOUR_CLOUDZERO_API_KEY                       <-- No longer set!
+#   existingSecretName: YOUR_EXISTING_API_KEY_K8S_SECRET <-- No longer set!
+global:
+  apiKey: YOUR_CLOUDZERO_API_KEY                        #<-- API key now set here
+  existingSecretName: YOUR_EXISTING_API_KEY_K8S_SECRET  #<-- existing secret name now set here
+
+# kube-state-metrics:
+#   extraArgs:                                          <-- No longer set!
+#      - --metric-labels-allowlist=pods=[foo,bar]
+tags:
+  enabled: true
+  labels:
+    enabled: true
+    patterns:
+      - '^foo$'                                         #<-- Setting to export "foo=bar" label now set here
+```
+* Upgrade with:
+```sh
+helm upgrade --install -n cloudzero-agent cloudzero-beta -f configuration-example.yaml
+```
+
+### Breaking Changes
+* Labels export configuration method has changed
+    * Previously, pod labels were exported using the `kube-state-metrics.extraArgs` field.
+    * Pod labels must now be configured using the `tags.labels` section. See the [Labels and Annotaitons](https://github.com/Cloudzero/cloudzero-charts/tree/develop/charts/cloudzero-agent#labels-and-annotations) section for details
+* API key management arguments have moved to the `global` section.
+    * Previous, an `apiKey` or `existingSecretName` argument could be passed to the chart. This is no longer allowed; those arguments should instead be passed as `global.apiKey` and `global.existingSecretName`, respectively.
+
+### New Features
+* **Labels and Annotations:** A subchart `cloudzero-insights-controller` is added, which deploys one or more `ValidatingWebhookConfiguration` resources
+    * Allows users to export labels/annotations from pods, deployments, daemonsets, statefulsets, jobs, cronjobs, namespaces, and nodes.
+    * Supports filtering labels/anotations by regular expressions using the `tags.labels.patterns` and/or `tags.annotations.patterns` array.
+
+### Other Changes
+* **CloudZero Metrics:** CloudZero State Metrics is enabled/installed by default.
\ No newline at end of file
diff --git a/charts/cloudzero-agent/templates/_helpers.tpl b/charts/cloudzero-agent/templates/_helpers.tpl
index 0cf200b6..c21eb5ec 100644
--- a/charts/cloudzero-agent/templates/_helpers.tpl
+++ b/charts/cloudzero-agent/templates/_helpers.tpl
@@ -14,7 +14,7 @@ Create chart name and version as used by the chart label.
 
 {{/* Define the secret name which holds the CloudZero API key */}}
 {{ define "cloudzero-agent.secretName" -}}
-{{ .Values.existingSecretName | default (printf "%s-api-key" .Release.Name) }}
+{{ .Values.global.existingSecretName | default (printf "%s-api-key" .Release.Name) }}
 {{- end}}
 
 {{/* Define the path and filename on the container filesystem which holds the CloudZero API key */}}
@@ -23,7 +23,11 @@ Create chart name and version as used by the chart label.
 {{- end}}
 
 {{ define "cloudzero-agent.configMapName" -}}
-{{ .Values.prometheusConfig.configMapNameOverride | default (printf "%s-configuration" .Release.Name) }}
+{{ .Values.configMapNameOverride | default (printf "%s-configuration" .Release.Name) }}
+{{- end}}
+
+{{ define "cloudzero-agent.cloudzeroConfigMapName" -}}
+{{ .Values.cloudzeroConfigMapNameOverride | default (printf "%s-cloudzero-configuration" .Release.Name) }}
 {{- end}}
 
 {{ define "cloudzero-agent.validatorConfigMapName" -}}
diff --git a/charts/cloudzero-agent/templates/cm.yaml b/charts/cloudzero-agent/templates/cm.yaml
index e73e816d..d8bd4e85 100644
--- a/charts/cloudzero-agent/templates/cm.yaml
+++ b/charts/cloudzero-agent/templates/cm.yaml
@@ -166,3 +166,23 @@ data:
             action: keep
         metadata_config:
           send: false
+{{- if .Values.tags.enabled }}
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  labels:
+    {{- include "cloudzero-agent.server.labels" . | nindent 4 }}
+  name: {{ include "cloudzero-agent.cloudzeroConfigMapName" . }}
+  namespace: {{ include "cloudzero-agent.namespace" . }}
+  {{- with .Values.prometheusConfig.configMapAnnotations }}
+  annotations:
+  {{- toYaml . | nindent 4 }}
+  {{- end }}
+data:
+  cloudzero-config.yaml: |-
+    cloud_account_id: {{ .Values.cloudAccountId }}
+    region: {{ .Values.region }}
+    cluster_name: {{ .Values.clusterName }}
+    host: {{ .Values.host }}
+{{- end }}
diff --git a/charts/cloudzero-agent/templates/deploy.yaml b/charts/cloudzero-agent/templates/deploy.yaml
index 688305a0..831f491f 100644
--- a/charts/cloudzero-agent/templates/deploy.yaml
+++ b/charts/cloudzero-agent/templates/deploy.yaml
@@ -39,7 +39,7 @@ spec:
             - -c
             - cp -r /app /checks/bin/ && cloudzero-agent-validator d pre-start -f /checks/config/validator.yml
           volumeMounts:
-          {{- if or .Values.existingSecretName .Values.apiKey }}
+          {{- if or .Values.global.existingSecretName .Values.global.apiKey }}
             - name: cloudzero-api-key
               mountPath: {{ .Values.server.containerSecretFilePath }}
               subPath: ""
@@ -132,7 +132,7 @@ spec:
               mountPath: /check/
             - name: validator-config-volume
               mountPath: /check/app/config/
-          {{- if or .Values.existingSecretName .Values.apiKey }}
+          {{- if or .Values.global.existingSecretName .Values.global.apiKey }}
             - name: cloudzero-api-key
               mountPath: {{ .Values.server.containerSecretFilePath }}
               subPath: ""
@@ -174,7 +174,10 @@ spec:
             name: {{ template "cloudzero-agent.validatorConfigMapName" . }}
         - name: lifecycle-volume
           emptyDir: {}
-        {{- if or .Values.existingSecretName .Values.apiKey }}
+        {{- if or .Values.existingSecretName .Values.apiKey}}
+        {{- fail "The `existingSecretName` or `apiKey` setting should be set as `globals.existingSecretName` or `globals.apiKey`, not at the top level." }}
+        {{- end }}
+        {{- if or .Values.global.existingSecretName .Values.global.apiKey }}
         - name: cloudzero-api-key
           secret:
             secretName: {{ include "cloudzero-agent.secretName" . }}
diff --git a/charts/cloudzero-agent/templates/secret.yaml b/charts/cloudzero-agent/templates/secret.yaml
index e01029db..b506b81f 100644
--- a/charts/cloudzero-agent/templates/secret.yaml
+++ b/charts/cloudzero-agent/templates/secret.yaml
@@ -1,4 +1,4 @@
-{{- if .Values.apiKey }}
+{{- if .Values.global.apiKey }}
 apiVersion: v1
 kind: Secret
 metadata:
@@ -11,7 +11,7 @@ metadata:
   {{- toYaml . | nindent 4 }}
   {{- end }}
 data:
-  {{ .Values.server.containerSecretFileName }}: {{- $apiKey := .Values.apiKey | toString }}
+  {{ .Values.server.containerSecretFileName }}: {{- $apiKey := .Values.global.apiKey | toString }}
           {{- if not (regexMatch "^[a-zA-Z0-9-_.~!*'();]+$" $apiKey) }}
           {{- fail "The provided apiKey is invalid. Check that the provided value from apiKey matches exactly what is found in the CloudZero admin page." }}
           {{- end }}
diff --git a/charts/cloudzero-agent/templates/validatorcm.yaml b/charts/cloudzero-agent/templates/validatorcm.yaml
index 213f5df1..80b33a93 100644
--- a/charts/cloudzero-agent/templates/validatorcm.yaml
+++ b/charts/cloudzero-agent/templates/validatorcm.yaml
@@ -33,7 +33,7 @@ data:
       {{- if .Values.validator.serviceEndpoints.kubeStateMetrics }}
       kube_state_metrics_service_endpoint: http://{{ .Values.validator.serviceEndpoints.kubeStateMetrics }}/
       {{- else }}
-      kube_state_metrics_service_endpoint: http://{{- if .Release.Name }}{{.Release.Name}}-{{- end }}kube-state-metrics:8080/
+      kube_state_metrics_service_endpoint: http://{{- if .Release.Name }}{{.Release.Name}}-{{- end }}state-metrics:8080/
       {{- end }}
       {{- if .Values.validator.serviceEndpoints.prometheusNodeExporter }}
       prometheus_node_exporter_service_endpoint: http://{{ .Values.validator.serviceEndpoints.prometheusNodeExporter }}/
diff --git a/charts/cloudzero-agent/values.yaml b/charts/cloudzero-agent/values.yaml
index 4b92db4a..99d78130 100644
--- a/charts/cloudzero-agent/values.yaml
+++ b/charts/cloudzero-agent/values.yaml
@@ -6,10 +6,12 @@ cloudAccountId: null
 clusterName: null
 # -- Region the cluster is running in.
 region: null
-# -- CloudZero API key. Required if useExistingSecret is false.
-apiKey: null
-# -- If set, the agent will use the API key in this Secret to authenticate with CloudZero.
-existingSecretName: null
+
+global:
+  # -- CloudZero API key. Required if useExistingSecret is false.
+  apiKey: null
+  # -- If set, the agent will use the API key in this Secret to authenticate with CloudZero.
+  existingSecretName: null
 
 # -- The following lists of metrics are required for CloudZero to function.
 # -- Modifications made to these lists may cause issues with the processing of cluster data
@@ -47,12 +49,26 @@ prometheusConfig:
     # -- Any items added to this list will be added to the Prometheus scrape configuration.
     additionalScrapeJobs: []
 
+tags:
+  enabled: true
+  labels:
+    enabled:
+    patterns:
+      # - '.*'
+  annotations:
+    enabled: false
+    patterns:
+      - '.*'
+
 kube-state-metrics:
-  enabled: false
-  extraArgs:
-    - --metric-labels-allowlist=pods=[app.kubernetes.io/component]
-prometheus-node-exporter:
-  enabled: false
+  enabled: true
+  fullnameOverride: "cloudzero-state-metrics"
+  # Disable CloudZero KSM as a Scrape Target since the service endpoint is explicity defined
+  # by the Validators config file.
+  prometheusScrape: false
+  # Set a default port other than 8080 to avoid collisions with any existing KSM services.
+  service:
+    port: 8080
 
 # -- Annotations to be added to the Secret, if the chart is configured to create one
 secretAnnotations: {}
@@ -89,7 +105,6 @@ server:
       memory: 1024Mi
   deploymentAnnotations: {}
   podAnnotations: {}
-  configMapOverrideName: configuration
   args:
   - --config.file=/etc/config/prometheus/configmaps/prometheus.yml
   - --web.enable-lifecycle
diff --git a/charts/cloudzero-insights-controller/Chart.lock b/charts/cloudzero-insights-controller/Chart.lock
new file mode 100644
index 00000000..346e17dc
--- /dev/null
+++ b/charts/cloudzero-insights-controller/Chart.lock
@@ -0,0 +1,6 @@
+dependencies:
+- name: cert-manager
+  repository: https://charts.jetstack.io
+  version: v1.15.3
+digest: sha256:9027951628db45ef674f00e5baeca157f95755de9818a9d1e78396b86971f527
+generated: "2024-08-29T11:00:51.842705-04:00"
diff --git a/charts/cloudzero-insights-controller/Chart.yaml b/charts/cloudzero-insights-controller/Chart.yaml
new file mode 100644
index 00000000..484e7001
--- /dev/null
+++ b/charts/cloudzero-insights-controller/Chart.yaml
@@ -0,0 +1,12 @@
+apiVersion: v2
+name: cloudzero-insights-controller
+description: Provides telemetry to the CloudZero platform to enabling complex cost allocation and analysis.
+type: application
+version: 0.0.1
+appVersion: "0.0.1"
+dependencies:
+  - name: cert-manager
+    version: v1.15.3
+    repository: https://charts.jetstack.io
+    alias: cert-manager
+    condition: cert-manager.enabled
diff --git a/charts/cloudzero-insights-controller/README.md b/charts/cloudzero-insights-controller/README.md
new file mode 100644
index 00000000..dd083dfe
--- /dev/null
+++ b/charts/cloudzero-insights-controller/README.md
@@ -0,0 +1,145 @@
+# Cloudzero Insights Controller Helm Chart
+
+[![Contributor Covenant](https://img.shields.io/badge/Contributor%20Covenant-2.1-4baaaa.svg)](CODE-OF-CONDUCT.md)
+[![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](LICENSE)
+![GitHub release](https://img.shields.io/github/release/Cloudzero/cloudzero-charts.svg)
+
+A Helm chart for a validating admission webhook to send cluster metrics to the CloudZero platform.
+
+## Overview
+
+This Validating Admission Webhook monitors and intercepts `CREATE` and `UPDATE` operations on the following Kubernetes resources:
+
+- `Pod`
+- `Deployment`
+- `StatefulSet`
+- `Daemonset`
+- `Job`
+- `CronJob`
+- `Node`
+- `Namespace`
+
+The webhook captures the labels from these resources and uploads them to the CloudZero API endpoint. For both `CREATE` and `UPDATE` operations, the full set of labels is sent to the API, ensuring that the most up-to-date labels are always uploaded. For `Deployment` and `Statefulset` resources, annotations are also uploaded.
+
+
+## Prerequisites
+
+- Kubernetes 1.23+
+- Helm 3+
+- A CloudZero API key
+
+## Installation
+
+This helm chart is best used alongside the [cloudzero-agent](https://github.com/Cloudzero/cloudzero-charts/tree/develop/charts/cloudzero-agent) chart. In this case, the same API key can be used for both installations.
+
+### Get Helm Repository Info
+
+```console
+helm repo add cloudzero https://cloudzero.github.io/cloudzero-charts
+helm repo update
+```
+
+_See [`helm repo`](https://helm.sh/docs/helm/helm_repo/) for command documentation._
+
+The chart can be installed directly with Helm or any other common Kubernetes deployment tools. See the next section for different deployment configurations.
+
+### Deployment Configurations and Certificate Management
+
+This chart contains a `ValidatingWebhookConfiguration` resource, which uses a certificate in order validate requests to the webhook server. See related Kubernetes documentation [here](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#configure-admission-webhooks-on-the-fly).
+
+
+**There are two ways to install the chart as it relates to certificate management:**
+
+1. (Default) Manage certificates using [cert-manager](https://github.com/cert-manager/cert-manager/tree/master).
+By default, the chart installs [cert-manager](https://github.com/cert-manager/cert-manager/tree/master) as a subchart. `cert-manager` handles the creation of the certificate and injects the CA bundle into the `ValidatingWebhookConfiguration` resource. For details on how cert-manager does this, see [here](https://cert-manager.io/docs/concepts/ca-injector/).
+
+To install the chart with this configuration, install the chart with the following helm command. The default configuration uses cert-manager to create the certificate:
+
+```console
+helm install <RELEASE_NAME> cloudzero/insights-controller \
+    --set existingSecretName=<NAME_OF_SECRET> \
+    --set clusterName=<CLUSTER_NAME> \
+    --set-string cloudAccountId=<CLOUD_ACCOUNT_ID> \
+    --set region=<REGION>
+```
+
+If `cert-manager` CRDs are not already installed, the installation may fail with the error message that contains:
+```console
+no matches for kind "Certificate" in version "cert-manager.io/v1"
+```
+
+If this happens, run the following:
+
+```bash
+helm install <RELEASE_NAME> cloudzero/insights-controller \
+    --set webhook.issuer.enabled=false \
+    --set webhook.certificate.enabled=false \
+    --set cert-manager.installCRDs=true
+```
+Or, alternatively, [install the cert-manager CRDs yourself](https://cert-manager.io/docs/installation/helm/).
+Then rerun the original command:
+```console
+helm install <RELEASE_NAME> cloudzero/insights-controller \
+    --set existingSecretName=<NAME_OF_SECRET> \
+    --set clusterName=<CLUSTER_NAME> \
+    --set-string cloudAccountId=<CLOUD_ACCOUNT_ID> \
+    --set region=<REGION>
+```
+
+2. The second option is to bring your own certificate. In this case, the tls information must be mounted to the server Deployment at the `/etc/certs/` path in a file formatted as:
+```
+ca.crt: <CA_CRT>
+tls.crt: <TLS_CERT>
+tls.key: <TLS_KEY>
+```
+An example command would be:
+```bash
+helm install <RELEASE_NAME> cloudzero/insights-controller \
+    --set existingSecretName=<NAME_OF_SECRET> \
+    --set clusterName=<CLUSTER_NAME> \
+    --set-string cloudAccountId=<CLOUD_ACCOUNT_ID> \
+    --set region=<REGION> \
+    -f config.yaml
+```
+where `config.yaml` is:
+```
+server:
+  tls:
+    useManagedSecret: false
+  volumeMounts:
+    - name: your-tls-volume
+      mountPath: /etc/certs
+      readOnly: true
+  volumes:
+    - name: tls-certs
+      secret:
+        secretName: your-tls-secret-name
+webhook:
+  issuer:
+    enabled: false
+  certificate:
+    enabled: false
+  caBundle: '<YOUR_CA_BUNDLE>'
+
+cert-manager:
+  enabled: false
+```
+
+## Troubleshooting
+
+### `<RELEASE-NAME>-server` pod stuck in `Pending` state
+  The server pod, which handles incoming webhook requests, may be stuck in this state if the TLS secret is not available. Confirm this is the case by describing the server pod:
+  ```console
+  kubectl describe pod -l app.kubernetes.io/name=insights-controller
+  ```
+  If the event log shows that the pod cannot be created due to a missing volume, check that the TLS secret has been created successfully:
+  ```console
+  kubectl get secret -l app.kubernetes.io/name=insights-controller
+  ```
+  If no secrets are returned by that command, then cert-manager did not provision a certificate. Consult the `cert-manager` pod logs and/or the cert-manager CRDs for more infomration:
+  ```console
+  kubectl get certificaterequests
+  kubectl get certificates
+  kubectl get certificatesigningrequests
+  kubectl get issuers
+  ```
diff --git a/charts/cloudzero-insights-controller/templates/_helpers.tpl b/charts/cloudzero-insights-controller/templates/_helpers.tpl
new file mode 100644
index 00000000..7b0da7fe
--- /dev/null
+++ b/charts/cloudzero-insights-controller/templates/_helpers.tpl
@@ -0,0 +1,206 @@
+{{/*
+Expand the name of the chart.
+*/}}
+{{- define "insights-controller.name" -}}
+{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{/*
+Create a default fully qualified app name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+If release name contains chart name it will be used as a full name.
+*/}}
+{{- define "insights-controller.fullname" -}}
+{{- if .Values.fullnameOverride }}
+{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- $name := default .Chart.Name .Values.nameOverride }}
+{{- if contains $name .Release.Name }}
+{{- .Release.Name | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }}
+{{- end }}
+{{- end }}
+{{- end }}
+
+{{/*
+Generate certificates for the webhook server
+*/}}
+{{- define "insights-controller.generate-certs" -}}
+{{- $altNames := list ( printf "DNS:webhook-server.%s" ( .Release.Namespace )) -}}
+{{- $ca := genCA "insights-controller-ca" 365 -}}
+{{- $cert := genSignedCert ( include "insights-controller.name" . ) nil $altNames 365 $ca -}}
+tls.crt: {{ $cert.Cert | b64enc }}
+tls.key: {{ $cert.Key | b64enc }}
+{{- end -}}
+
+
+{{/*
+Create chart name and version as used by the chart label.
+*/}}
+{{- define "insights-controller.chart" -}}
+{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{/*
+Common labels
+*/}}
+{{- define "insights-controller.labels" -}}
+helm.sh/chart: {{ include "insights-controller.chart" . }}
+{{ include "insights-controller.selectorLabels" . }}
+{{- if .Chart.AppVersion }}
+app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
+{{- end }}
+app.kubernetes.io/managed-by: {{ .Release.Service }}
+{{- end }}
+
+{{/*
+Selector labels
+*/}}
+{{- define "insights-controller.selectorLabels" -}}
+app.kubernetes.io/name: {{ include "insights-controller.name" . }}
+app.kubernetes.io/instance: {{ .Release.Name }}
+{{- end }}
+
+{{/*
+Selector labels
+*/}}
+{{- define "insights-controller.annotations" -}}
+{{- if .Values.webhook.annotations }}
+{{ toYaml .Values.webhook.annotations }}
+{{- end }}
+{{- if and .Values.webhook.certificate.enabled .Values.webhook.issuer.enabled }}
+cert-manager.io/inject-ca-from: {{ .Values.webhook.caInjection | default (printf "%s/%s" .Release.Namespace (include "insights-controller.certificateName" .)) }}
+{{- end }}
+{{- end }}
+
+{{/*
+Create the name of the service account to use
+*/}}
+{{- define "insights-controller.serviceAccountName" -}}
+{{- if .Values.serviceAccount.create }}
+{{- default (include "insights-controller.fullname" .) .Values.serviceAccount.name }}
+{{- else }}
+{{- default "default" .Values.serviceAccount.name }}
+{{- end }}
+{{- end }}
+
+{{/*
+Name for the webhook server deployment
+*/}}
+{{- define "insights-controller.deploymentName" -}}
+{{- printf "%s-server" (include "insights-controller.fullname" .) }}
+{{- end }}
+
+{{/*
+Name for the webhook server service
+*/}}
+{{- define "insights-controller.serviceName" -}}
+{{- printf "%s-svc" (include "insights-controller.fullname" .) }}
+{{- end }}
+
+{{/*
+Name for the validating webhook configuration resource
+*/}}
+{{- define "insights-controller.validatingWebhookConfigName" -}}
+{{- printf "%s-webhook" (include "insights-controller.fullname" .) }}
+{{- end }}
+
+{{/*
+Name for the validating webhook
+*/}}
+{{- define "insights-controller.validatingWebhookName" -}}
+{{- printf "%s.%s.svc" (include "insights-controller.validatingWebhookConfigName" .) .Release.Namespace }}
+{{- end }}
+
+{{/*
+Name for the certificate resource
+*/}}
+{{- define "insights-controller.certificateName" -}}
+{{- printf "%s-certificate" (include "insights-controller.fullname" .) }}
+{{- end }}
+
+{{/*
+Name for the certificate secret
+*/}}
+{{- define "insights-controller.tlsSecretName" -}}
+{{- printf "%s-tls" (include "insights-controller.fullname" .) }}
+{{- end }}
+
+{{/*
+Name for the cloudzero API key secret name
+*/}}
+{{- define "insights-controller.cloudzeroSecretName" -}}
+{{- if and .Values.global .Values.global.existingSecretName }}
+{{- .Values.global.existingSecretName }}
+{{- else}}
+{{- .Values.existingSecretName | default (printf "%s-api-key" .Release.Name) }}
+{{- end }}
+{{- end }}
+
+{{/*
+Name for the cloudzero specific configuration file
+*/}}
+{{- define "insights-controller.cloudzeroConfigMapName" -}}
+{{ .Values.cloudzeroConfigMapNameOverride | default (printf "%s-cloudzero-configuration" .Release.Name) }}
+{{- end }}
+
+{{/*
+Name for the webhook server configuration file
+*/}}
+{{- define "insights-controller.configMapName" -}}
+{{- printf "%s-configuration" (include "insights-controller.fullname" .) }}
+{{- end }}
+
+{{/*
+Mount path for the cloudzero configuration file
+*/}}
+{{- define "insights-controller.cloudzeroConfigurationMountPath" -}}
+{{- printf "/etc/%s-cloudzero-config" .Chart.Name  }}
+{{- end }}
+
+{{/*
+Mount path for the insights server configuration file
+*/}}
+{{- define "insights-controller.serverConfigurationMountPath" -}}
+{{- printf "/etc/%s-insights-server-config" .Chart.Name  }}
+{{- end }}
+
+{{/*
+Name for the issuer resource
+*/}}
+{{- define "insights-controller.issuerName" -}}
+{{- printf "%s-issuer" (include "insights-controller.fullname" .) }}
+{{- end }}
+
+{{/*
+Name for the job resource
+*/}}
+{{- define "insights-controller.initJobName" -}}
+{{- printf "%s-init" (include "insights-controller.fullname" .) }}
+{{- end }}
+
+{{/*
+Name for the clusterrole resource
+*/}}
+{{- define "insights-controller.clusterRoleName" -}}
+{{- printf "%s" (include "insights-controller.fullname" .) }}
+{{- end }}
+
+{{/*
+Name for the clusterrolebinding resource
+*/}}
+{{- define "insights-controller.clusterRoleBindingName" -}}
+{{- printf "%s" (include "insights-controller.fullname" .) }}
+{{- end }}
+
+{{/*
+Name for the clusterrolebinding resource
+*/}}
+{{- define "insights-controller.cloudAccountId" -}}
+{{- if .Values.global.cloudAccountId }}
+{{ tpl .Values.global.cloudAccountId . }}
+{{- else}}
+{{ .Values.cloudAccountId . }}
+{{- end }}
+{{- end }}
diff --git a/charts/cloudzero-insights-controller/templates/certificate.yaml b/charts/cloudzero-insights-controller/templates/certificate.yaml
new file mode 100644
index 00000000..20f47f69
--- /dev/null
+++ b/charts/cloudzero-insights-controller/templates/certificate.yaml
@@ -0,0 +1,28 @@
+{{ if .Values.webhook.certificate.enabled }}
+---
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: {{ include "insights-controller.certificateName" . }}
+  namespace: {{ .Release.Namespace }}
+spec:
+  secretName: {{ include "insights-controller.tlsSecretName" .}}
+  secretTemplate:
+    {{- with .Values.secretAnnotations }}
+    annotations:
+    {{- toYaml . | nindent 4 }}
+    {{- end }}
+    labels:
+      {{- include "insights-controller.labels" . | nindent 6 }}
+  privateKey:
+    algorithm: RSA
+    encoding: PKCS1
+    size: 2048
+  duration: 2160h # 90d
+  renewBefore: 360h # 15d
+  dnsNames:
+    - {{ include "insights-controller.serviceName" . }}.{{ .Release.Namespace }}.svc
+  issuerRef:
+    name: {{ include "insights-controller.issuerName" . }}
+    kind: Issuer
+{{ end }}
diff --git a/charts/cloudzero-insights-controller/templates/clusterrole.yaml b/charts/cloudzero-insights-controller/templates/clusterrole.yaml
new file mode 100644
index 00000000..c893f76b
--- /dev/null
+++ b/charts/cloudzero-insights-controller/templates/clusterrole.yaml
@@ -0,0 +1,14 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: {{ include "insights-controller.clusterRoleName" . }}
+rules:
+  - apiGroups: [""]
+    resources: ["namespaces", "nodes", "pods"]
+    verbs: ["get", "list"]
+  - apiGroups: ["apps"]
+    resources: ["deployments", "statefulsets", "daemonsets"]
+    verbs: ["get", "list"]
+  - apiGroups: ["batch"]
+    resources: ["jobs", "cronjobs"]
+    verbs: ["get", "list"]
diff --git a/charts/cloudzero-insights-controller/templates/clusterrolebinding.yaml b/charts/cloudzero-insights-controller/templates/clusterrolebinding.yaml
new file mode 100644
index 00000000..85611453
--- /dev/null
+++ b/charts/cloudzero-insights-controller/templates/clusterrolebinding.yaml
@@ -0,0 +1,14 @@
+{{- if .Values.rbac.create -}}
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: {{ include "insights-controller.clusterRoleBindingName" . }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: {{ include "insights-controller.clusterRoleName" . }}
+subjects:
+  - kind: ServiceAccount
+    name: {{ include "insights-controller.serviceAccountName" . }}
+    namespace: {{ .Release.Namespace }}
+{{- end }}
diff --git a/charts/cloudzero-insights-controller/templates/cm.yaml b/charts/cloudzero-insights-controller/templates/cm.yaml
new file mode 100644
index 00000000..5a8477c1
--- /dev/null
+++ b/charts/cloudzero-insights-controller/templates/cm.yaml
@@ -0,0 +1,64 @@
+{{- if not (and .Values.labels.enabled .Values.labels.patterns) }}
+{{- $msg := "\n\nThe required field(s) 'labels.enabled' and/or 'labels.patterns' is not set! See the README.md for more information." }}
+{{- $enabledMsg:=""}}
+{{- $patternMsg:=""}}
+{{- if not .Values.labels.enabled  }}
+{{- $enabledMsg = "Ensure that 'labels.enabled' is a boolean (true or false). Set 'true' to enable exporting labels."}}
+{{- end }}
+{{- if not .Values.labels.patterns }}
+{{- $patternMsg = "The required field 'labels.patterns' is not set or set incorrectly. It must be an array of regular expressions that match label keys to be exported."}}
+{{- end }}
+{{- fail (printf "\n %s \n %s \n %s" $msg $enabledMsg $patternMsg) }}
+{{- end }}
+
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  labels:
+    {{- include "insights-controller.labels" . | nindent 4 }}
+  name: {{ include "insights-controller.configMapName" . }}
+  namespace: {{ .Release.Namespace }}
+  {{- with .Values.server.configMap.annotations }}
+  annotations:
+  {{- toYaml . | nindent 4 }}
+  {{- end }}
+data:
+  server-config.yaml: |-
+    {{- if .Values.cloudAccountId }}
+    cloud_account_id: {{ .Values.cloudAccountId }}
+    {{- end }}
+    {{- if .Values.region }}
+    region: {{ .Values.region }}
+    {{- end }}
+    {{- if .Values.clusterName }}
+    cluster_name: {{ .Values.clusterName }}
+    {{- end }}
+    {{- if .Values.host }}
+    host: {{ .Values.host }}
+    {{- end }}
+    remote_write:
+      send_interval: 1m
+      max_bytes_per_send: 10000000
+      send_timeout: 10s
+    k8s_client:
+      timeout: 30s
+    database:
+      retention_time: 24h
+      cleanup_interval: 3h
+    {{- with .Values.server.tls }}
+    certificate:
+      key: {{ .mountPath }}/tls.key
+      cert: {{ .mountPath }}/tls.crt
+    {{- end }}
+    api_key_path: {{ .Values.server.serverConfig.containerSecretFilePath }}/{{ .Values.server.serverConfig.containerSecretFileName }}
+    server:
+      port: {{ .Values.server.serverConfig.port }}
+      read_timeout: {{ .Values.server.serverConfig.read_timeout }}
+      write_timeout: {{ .Values.server.serverConfig.write_timeout }}
+      idle_timeout: {{ .Values.server.serverConfig.idle_timeout }}
+    filters:
+      labels:
+        {{- .Values.labels | toYaml | nindent 8 }}
+      annotations:
+        {{- .Values.labels | toYaml | nindent 8 }}
+      
diff --git a/charts/cloudzero-insights-controller/templates/deploy.yaml b/charts/cloudzero-insights-controller/templates/deploy.yaml
new file mode 100644
index 00000000..3be72577
--- /dev/null
+++ b/charts/cloudzero-insights-controller/templates/deploy.yaml
@@ -0,0 +1,107 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ include "insights-controller.deploymentName" . }}
+  namespace: {{ .Release.Namespace }}
+  labels:
+    {{- include "insights-controller.labels" . | nindent 4 }}
+spec:
+  replicas: {{ .Values.server.replicaCount }}
+  selector:
+    matchLabels:
+      {{- include "insights-controller.selectorLabels" . | nindent 6 }}
+  template:
+    metadata:
+      {{- with .Values.server.podAnnotations }}
+      annotations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      labels:
+        {{- include "insights-controller.labels" . | nindent 8 }}
+        {{- with .Values.server.podLabels }}
+        {{- toYaml . | nindent 8 }}
+        {{- end }}
+    spec:
+      serviceAccountName: {{ include "insights-controller.serviceAccountName" . }}
+      {{- with .Values.server.imagePullSecrets }}
+      imagePullSecrets:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      serviceAccountName: {{ include "insights-controller.serviceAccountName" . }}
+      securityContext:
+        runAsUser: 65534
+        runAsNonRoot: true
+        runAsGroup: 65534
+        fsGroup: 65534
+      containers:
+        - name: webhook-server
+          image: "{{ .Values.server.image.repository }}:{{ .Values.server.image.tag | default .Chart.AppVersion }}"
+          imagePullPolicy: {{ .Values.server.image.pullPolicy }}
+          command:
+            - /app/controller
+          args:
+            - -config
+            - "{{ include "insights-controller.serverConfigurationMountPath" . }}/server-config.yaml"
+            {{-  if (include "insights-controller.cloudzeroConfigMapName" .) }}
+            - -config
+            - "{{include "insights-controller.cloudzeroConfigurationMountPath" . }}/cloudzero-config.yaml"
+            {{- end }}
+          ports:
+            - containerPort: 8443
+          resources:
+            {{- toYaml .Values.server.resources | nindent 12 }}
+          {{- if or .Values.server.volumeMounts .Values.server.tls.useManagedSecret }}
+          volumeMounts:
+            {{-  if (include "insights-controller.cloudzeroConfigMapName" .) }}
+            - name: cloudzero-server-config
+              mountPath: {{ include "insights-controller.cloudzeroConfigurationMountPath" . }}
+            {{- end }}
+            - name: insights-server-config
+              mountPath: {{ include "insights-controller.serverConfigurationMountPath" . }}
+            {{- if .Values.server.tls.useManagedSecret }}
+            - name: tls-certs
+              mountPath: {{ .Values.server.tls.mountPath }}
+              readOnly: true
+            {{- end }}
+            - name: cloudzero-api-key
+              mountPath: {{ .Values.server.serverConfig.containerSecretFilePath }}
+              subPath: ""
+              readOnly: true
+            {{- with .Values.server.volumeMounts }}
+              {{- toYaml . | nindent 12 }}
+            {{- end }}
+          {{- end }}
+      {{- if or .Values.server.volumes .Values.server.tls.useManagedSecret }}
+      volumes:
+      {{-  if (include "insights-controller.cloudzeroConfigMapName" .) }}
+        - name: cloudzero-server-config
+          configMap:
+            name: {{ include "insights-controller.cloudzeroConfigMapName" . }}
+      {{- end }}
+        - name: insights-server-config
+          configMap:
+            name: {{ include "insights-controller.configMapName" . }}
+        {{- if .Values.server.tls.useManagedSecret }}
+        - name: tls-certs
+          secret:
+            secretName: {{ include "insights-controller.tlsSecretName" . }}
+        {{- end }}
+        - name: cloudzero-api-key
+          secret:
+            secretName: {{ include "insights-controller.cloudzeroSecretName" . }}
+        {{- with .Values.server.volumes }}
+          {{- toYaml . | nindent 8 }}
+        {{- end }}
+      {{- end }}
+      {{- with .Values.server.nodeSelector }}
+      nodeSelector:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.server.affinity }}
+      affinity:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.server.tolerations }}
+      tolerations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
diff --git a/charts/cloudzero-insights-controller/templates/init-job.yaml b/charts/cloudzero-insights-controller/templates/init-job.yaml
new file mode 100644
index 00000000..ed2276b1
--- /dev/null
+++ b/charts/cloudzero-insights-controller/templates/init-job.yaml
@@ -0,0 +1,27 @@
+{{- if .Values.initJob.enabled }}
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: {{ include "insights-controller.initJobName" . }}
+  namespace: {{ .Release.Namespace }}
+  labels:
+    {{- include "insights-controller.labels" . | nindent 4 }}
+spec:
+  template:
+    metadata:
+      name: {{ include "insights-controller.initJobName" . }}
+      labels:
+        {{- include "insights-controller.labels" . | nindent 8 }}
+    spec:
+      restartPolicy: Never
+      containers:
+        - name: start-scrape
+          image: {{ .Values.initJob.image.repository }}:{{ .Values.initJob.image.tag }}
+          command: ["sh", "-c"]
+          args:
+            - |
+              until curl -X POST -k https://{{ include "insights-controller.serviceName" . }}.{{ .Release.Namespace }}.svc.cluster.local:{{ .Values.server.service.port }}/scrape; do
+                echo "Waiting for service..."
+                sleep 5
+              done
+{{- end }}
diff --git a/charts/cloudzero-insights-controller/templates/issuer.yaml b/charts/cloudzero-insights-controller/templates/issuer.yaml
new file mode 100644
index 00000000..3f2f81de
--- /dev/null
+++ b/charts/cloudzero-insights-controller/templates/issuer.yaml
@@ -0,0 +1,10 @@
+{{ if .Values.webhook.issuer.enabled }}
+---
+apiVersion: cert-manager.io/v1
+kind: Issuer
+metadata:
+  name: {{ include "insights-controller.issuerName" . }}
+  namespace: {{ .Release.Namespace }}
+spec:
+  {{- toYaml .Values.webhook.issuer.spec | nindent 2 }}
+{{- end }}
diff --git a/charts/cloudzero-insights-controller/templates/service.yaml b/charts/cloudzero-insights-controller/templates/service.yaml
new file mode 100644
index 00000000..42f69659
--- /dev/null
+++ b/charts/cloudzero-insights-controller/templates/service.yaml
@@ -0,0 +1,15 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ include "insights-controller.serviceName" . }}
+  labels:
+    {{- include "insights-controller.labels" . | nindent 4 }}
+  namespace: {{ .Release.Namespace }}
+spec:
+  type: ClusterIP
+  ports:
+    - port: {{ .Values.server.service.port }}
+      targetPort: 8443
+      name: http
+  selector:
+    {{- include "insights-controller.selectorLabels" . | nindent 4 }}
diff --git a/charts/cloudzero-insights-controller/templates/serviceaccount.yaml b/charts/cloudzero-insights-controller/templates/serviceaccount.yaml
new file mode 100644
index 00000000..9f5647cc
--- /dev/null
+++ b/charts/cloudzero-insights-controller/templates/serviceaccount.yaml
@@ -0,0 +1,13 @@
+{{- if .Values.serviceAccount.create -}}
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: {{ include "insights-controller.serviceAccountName" . }}
+  labels:
+    {{- include "insights-controller.labels" . | nindent 4 }}
+  {{- with .Values.serviceAccount.annotations }}
+  annotations:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+automountServiceAccountToken: true
+{{- end }}
diff --git a/charts/cloudzero-insights-controller/templates/webhooks.yaml b/charts/cloudzero-insights-controller/templates/webhooks.yaml
new file mode 100644
index 00000000..71c19f98
--- /dev/null
+++ b/charts/cloudzero-insights-controller/templates/webhooks.yaml
@@ -0,0 +1,32 @@
+{{- range $configType, $configs := .Values.webhook.configurations }}
+---
+apiVersion: admissionregistration.k8s.io/v1
+kind: ValidatingWebhookConfiguration
+metadata:
+  name: {{ include "insights-controller.validatingWebhookConfigName" $ }}-{{ $configType }}
+  namespace: {{ $.Release.Namespace }}
+  annotations:
+    {{- include "insights-controller.annotations" $ | nindent 4 }}
+webhooks:
+  - name: {{ include "insights-controller.validatingWebhookName" $ }}
+    namespaceSelector: {{ toYaml $.Values.webhook.namespaceSelector }}
+    failurePolicy: Ignore
+    rules:
+      - operations: [ "CREATE", "UPDATE" ]
+        apiGroups: {{ $configs.apiGroups }}
+        apiVersions: [ "v1" ]
+        resources: [ {{ $configType }} ]
+        scope: "*"
+    clientConfig:
+      service:
+        namespace: {{ $.Release.Namespace }}
+        name: {{ include "insights-controller.serviceName" $ }}
+        path: "{{ $configs.path }}"
+        port: {{ $.Values.server.service.port }}
+      {{- if and $.Values.webhook.certificate.enabled $.Values.webhook.issuer.enabled }}
+      caBundle: {{ $.Values.webhook.caBundle }}
+      {{- end }}
+    admissionReviewVersions: ["v1"]
+    sideEffects: None
+    timeoutSeconds: 5
+{{- end }}
diff --git a/charts/cloudzero-insights-controller/values.yaml b/charts/cloudzero-insights-controller/values.yaml
new file mode 100644
index 00000000..209f3753
--- /dev/null
+++ b/charts/cloudzero-insights-controller/values.yaml
@@ -0,0 +1,112 @@
+# -- CloudZero host to send metrics to.
+host: null
+# -- Account ID of the account the cluster is running in. This must be a string - even if it is a number in your system.
+cloudAccountId: null
+# -- Name of the clusters.
+clusterName: null
+# -- Region the cluster is running in.
+region: null
+# -- CloudZero API key. Required if useExistingSecret is false.
+apiKey: null
+# -- If set, the agent will use the API key in this Secret to authenticate with CloudZero.
+existingSecretName: null
+
+replicaCount: 3
+
+nameOverride: ""
+fullnameOverride: ""
+
+labels:
+  enabled: null
+  patterns:
+    # - '.*'
+annotations:
+  enabled: false
+  patterns:
+    - '.*'
+
+server:
+  service:
+    port: 443
+  image:
+    repository: ghcr.io/cloudzero/cloudzero-insights-controller/cloudzero-insights-controller
+    pullPolicy: Always
+    # Overrides the image tag whose default is the chart appVersion.
+    # tag: "develop"
+  resources: {}
+  nodeSelector: {}
+  tolerations: []
+  affinity: {}
+  imagePullSecrets: []
+  podAnnotations: {}
+  podLabels: {}
+  configMap:
+    annotations: {}
+  serverConfig:
+    containerSecretFilePath: /etc/config/cloudzero/secrets
+    containerSecretFileName: value
+    port: 8443
+    read_timeout: 10s
+    write_timeout: 10s
+    idle_timeout: 120s
+  tls:
+    enabled: true
+    useManagedSecret: true
+    mountPath: /etc/certs
+  volumeMounts: []
+  volumes: []
+
+
+webhook:
+  annotations: {}
+  namespaceSelector: {} # This denotes no specific selection, applies to all namespaces
+  issuer:
+    enabled: true
+    spec:
+      selfSigned: {}
+  certificate:
+    enabled: true
+  caBundle: '' # by default, this is empty, and the value is populated by cert-manager's ca-injector if cert-manager is used
+  configurations:
+    pods:
+      path: /validate/pod
+      apiGroups: [ '""' ]
+    namespaces:
+      path: /validate/namespace
+      apiGroups: [ '""' ]
+  # -- Uncomment the following to enable exporting labels/annotations for the following resources
+    # deployments:
+    #   path: /validate/deployment
+    #   apiGroups: [ "apps" ]
+    # statefulsets:
+    #   path: /validate/statefulset
+    #   apiGroups: [ "apps" ]
+    # nodes:
+    #   path: /validate/node
+    #   apiGroups: [ '""' ]
+    # jobs:
+    #   path: /validate/job
+    #   apiGroups: [ "batch" ]
+    # cronjobs:
+    #   path: /validate/cronjob
+    #   apiGroups: [ "batch" ]
+    # daemonsets:
+    #   path: /validate/daemonset
+    #   apiGroups: [ "apps" ]
+
+serviceAccount:
+  create: true
+  annotations: {}
+
+rbac:
+  create: true
+
+cert-manager:
+  enabled: true
+
+initJob:
+  enabled: true
+  image:
+    repository: curlimages/curl
+    pullPolicy: Always
+    tag: "8.10.1"

From fe55e8d7e563c57fd8349f153da91295b33cba6b Mon Sep 17 00:00:00 2001
From: Daniel Mepham <daniel.mepham@cloudzero.com>
Date: Thu, 14 Nov 2024 14:32:27 +0100
Subject: [PATCH 02/26] CP-22730: use correct pattern list in config

---
 charts/cloudzero-insights-controller/templates/cm.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/charts/cloudzero-insights-controller/templates/cm.yaml b/charts/cloudzero-insights-controller/templates/cm.yaml
index 5a8477c1..0fa001e8 100644
--- a/charts/cloudzero-insights-controller/templates/cm.yaml
+++ b/charts/cloudzero-insights-controller/templates/cm.yaml
@@ -60,5 +60,5 @@ data:
       labels:
         {{- .Values.labels | toYaml | nindent 8 }}
       annotations:
-        {{- .Values.labels | toYaml | nindent 8 }}
+        {{- .Values.annotations | toYaml | nindent 8 }}
       

From 551339d5f86a63edadf2462ec163fe317e39cb81 Mon Sep 17 00:00:00 2001
From: Daniel Mepham <daniel.mepham@cloudzero.com>
Date: Thu, 14 Nov 2024 08:53:54 -0500
Subject: [PATCH 03/26] CP-22730: update doc check location to match normal
 release path (#100)

---
 .github/workflows/build-and-publish-beta-chart.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/build-and-publish-beta-chart.yml b/.github/workflows/build-and-publish-beta-chart.yml
index ba54b316..8243a206 100644
--- a/.github/workflows/build-and-publish-beta-chart.yml
+++ b/.github/workflows/build-and-publish-beta-chart.yml
@@ -79,7 +79,7 @@ jobs:
 
       - name: Validate Release Notes are Present
         run: |
-          if [ ! -f "docs/releases/${{ env.NEW_VERSION }}.md" ]; then
+          if [ ! -f "charts/cloudzero-agent/docs/releases/${{ env.NEW_VERSION }}.md" ]; then
             echo "Release notes for ${{ env.NEW_VERSION }} are missing. Please create a release notes file at docs/releases/${{ env.NEW_VERSION }}.md"
             exit 1
           fi

From efe79179af7f3ee78d9a6bc34455d144788c0178 Mon Sep 17 00:00:00 2001
From: Automated CZ Release <ops@cloudzero.com>
Date: Thu, 14 Nov 2024 14:01:29 +0000
Subject: [PATCH 04/26] Update Chart.yaml to version 1.0.0-beta

---
 charts/cloudzero-agent/Chart.yaml               | 2 +-
 charts/cloudzero-insights-controller/Chart.yaml | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/charts/cloudzero-agent/Chart.yaml b/charts/cloudzero-agent/Chart.yaml
index 5585449e..d604a2b0 100644
--- a/charts/cloudzero-agent/Chart.yaml
+++ b/charts/cloudzero-agent/Chart.yaml
@@ -2,7 +2,7 @@ apiVersion: v2
 name: cloudzero-agent
 description: A chart for using Prometheus in agent mode to send cluster metrics to the CloudZero platform.
 type: application
-version: 0.0.0-dev
+version: 1.0.0-beta
 maintainers:
   - name: CloudZero
     email: support@cloudzero.com
diff --git a/charts/cloudzero-insights-controller/Chart.yaml b/charts/cloudzero-insights-controller/Chart.yaml
index 484e7001..1f2a9085 100644
--- a/charts/cloudzero-insights-controller/Chart.yaml
+++ b/charts/cloudzero-insights-controller/Chart.yaml
@@ -2,7 +2,7 @@ apiVersion: v2
 name: cloudzero-insights-controller
 description: Provides telemetry to the CloudZero platform to enabling complex cost allocation and analysis.
 type: application
-version: 0.0.1
+version: 1.0.0-beta
 appVersion: "0.0.1"
 dependencies:
   - name: cert-manager

From 68b1dcfb361f540a9a717c745204b373d625f1ba Mon Sep 17 00:00:00 2001
From: Daniel Mepham <daniel.mepham@cloudzero.com>
Date: Thu, 14 Nov 2024 16:17:50 +0100
Subject: [PATCH 05/26] use latest insights-controller

---
 charts/cloudzero-agent/Chart.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/charts/cloudzero-agent/Chart.yaml b/charts/cloudzero-agent/Chart.yaml
index d604a2b0..746e0f62 100644
--- a/charts/cloudzero-agent/Chart.yaml
+++ b/charts/cloudzero-agent/Chart.yaml
@@ -13,7 +13,7 @@ dependencies:
     repository: https://prometheus-community.github.io/helm-charts
     condition: kube-state-metrics.enabled
   - name: cloudzero-insights-controller
-    version: "0.0.1"
+    version: "1.0.0-beta"
     repository: https://cloudzero.github.io/cloudzero-charts/beta
     condition: cloudzero-insights-controller.enabled
     alias: tags

From 49d41c24a2f2034f71f6307d0f8bb775c09b78b0 Mon Sep 17 00:00:00 2001
From: Daniel Mepham <daniel.mepham@cloudzero.com>
Date: Fri, 15 Nov 2024 13:55:05 -0500
Subject: [PATCH 06/26] CP-23435: remove duplicate service account name in
 insights-controller chart (#103)

---
 charts/cloudzero-insights-controller/templates/deploy.yaml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/charts/cloudzero-insights-controller/templates/deploy.yaml b/charts/cloudzero-insights-controller/templates/deploy.yaml
index 3be72577..164d0d9a 100644
--- a/charts/cloudzero-insights-controller/templates/deploy.yaml
+++ b/charts/cloudzero-insights-controller/templates/deploy.yaml
@@ -27,7 +27,6 @@ spec:
       imagePullSecrets:
         {{- toYaml . | nindent 8 }}
       {{- end }}
-      serviceAccountName: {{ include "insights-controller.serviceAccountName" . }}
       securityContext:
         runAsUser: 65534
         runAsNonRoot: true

From 3ea1e9bea0d36ddd901adc9a830b8c19710c9303 Mon Sep 17 00:00:00 2001
From: Daniel Mepham <daniel.mepham@cloudzero.com>
Date: Sun, 17 Nov 2024 03:23:46 -0500
Subject: [PATCH 07/26] CP-23426: use insights-controller service account for
 init job (#104)

---
 charts/cloudzero-insights-controller/templates/init-job.yaml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/charts/cloudzero-insights-controller/templates/init-job.yaml b/charts/cloudzero-insights-controller/templates/init-job.yaml
index ed2276b1..e81dffbf 100644
--- a/charts/cloudzero-insights-controller/templates/init-job.yaml
+++ b/charts/cloudzero-insights-controller/templates/init-job.yaml
@@ -13,6 +13,7 @@ spec:
       labels:
         {{- include "insights-controller.labels" . | nindent 8 }}
     spec:
+      serviceAccountName: {{ include "insights-controller.serviceAccountName" . }}
       restartPolicy: Never
       containers:
         - name: start-scrape

From 0e0d5f866fddcda9a8a3135158187864f5fb8063 Mon Sep 17 00:00:00 2001
From: Daniel Mepham <daniel.mepham@cloudzero.com>
Date: Mon, 18 Nov 2024 11:53:42 -0500
Subject: [PATCH 08/26] CP-23465: increase default replica count for insights
 controller (#106)

---
 charts/cloudzero-insights-controller/values.yaml | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/charts/cloudzero-insights-controller/values.yaml b/charts/cloudzero-insights-controller/values.yaml
index 209f3753..8a0ba5e2 100644
--- a/charts/cloudzero-insights-controller/values.yaml
+++ b/charts/cloudzero-insights-controller/values.yaml
@@ -11,8 +11,6 @@ apiKey: null
 # -- If set, the agent will use the API key in this Secret to authenticate with CloudZero.
 existingSecretName: null
 
-replicaCount: 3
-
 nameOverride: ""
 fullnameOverride: ""
 
@@ -26,6 +24,7 @@ annotations:
     - '.*'
 
 server:
+  replicaCount: 3
   service:
     port: 443
   image:

From 3f85beab64fbe37c5a76af4611f8fefc585fdbe6 Mon Sep 17 00:00:00 2001
From: Daniel Mepham <daniel.mepham@cloudzero.com>
Date: Mon, 18 Nov 2024 11:57:39 -0500
Subject: [PATCH 09/26] CP-23423: add release doc for 1.0.1-beta release (#107)

---
 .../cloudzero-agent/docs/releases/1.0.1-beta.md | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)
 create mode 100644 charts/cloudzero-agent/docs/releases/1.0.1-beta.md

diff --git a/charts/cloudzero-agent/docs/releases/1.0.1-beta.md b/charts/cloudzero-agent/docs/releases/1.0.1-beta.md
new file mode 100644
index 00000000..a3fa540e
--- /dev/null
+++ b/charts/cloudzero-agent/docs/releases/1.0.1-beta.md
@@ -0,0 +1,17 @@
+## [1.0.1-beta](https://github.com/Cloudzero/cloudzero-insights-controller/compare/v1.0.0-beta...v1.0.1-beta) (2024-11-17)
+
+Bug fixes and improvements, including an upgrade to version `0.0.2` of the `cloudzero-insights-controller` image. 
+
+### Upgrade Steps
+* Upgrade with:
+```sh
+helm upgrade --install -n cloudzero-agent cloudzero-beta -f configuration-example.yaml
+```
+See the [beta installation instructions](https://github.com/Cloudzero/cloudzero-charts/blob/develop/charts/cloudzero-agent/BETA-INSTALLATION.md) for further detail
+
+### Bug Fixes
+* **Remove Duplicate Service Account Name:** Removes a duplicate entry for Service Account used by the server Deployment.
+
+### Improvements
+* **Increase default replica count for insights-controller server:** Increases the default replica count from 1 to 3 for high availability.
+* **Initialization Job Uses Chart Service Account:** The `init-job` now uses the Service Account created in this chart instead of the default Service Account, which is required by some security policies.

From 3e074c29216188bbffcd86855aa505dedbe8560b Mon Sep 17 00:00:00 2001
From: Daniel Mepham <daniel.mepham@cloudzero.com>
Date: Mon, 18 Nov 2024 12:03:43 -0500
Subject: [PATCH 10/26] [CP-23425] add default remote write retries (#108)

* CP-23425: set default max retries

* update init job to work with long running scrapes

* increase wait time for scrape endpoint

* default batch size added

* increase wait time for init job

* adjust remote write threshold, add default resource values
---
 .../cloudzero-insights-controller/Chart.yaml   |  2 +-
 .../templates/cm.yaml                          |  4 +++-
 .../templates/init-job.yaml                    | 13 ++++++++++---
 .../cloudzero-insights-controller/values.yaml  | 18 ++++++++++++++++++
 4 files changed, 32 insertions(+), 5 deletions(-)

diff --git a/charts/cloudzero-insights-controller/Chart.yaml b/charts/cloudzero-insights-controller/Chart.yaml
index 1f2a9085..f3b2b744 100644
--- a/charts/cloudzero-insights-controller/Chart.yaml
+++ b/charts/cloudzero-insights-controller/Chart.yaml
@@ -2,7 +2,7 @@ apiVersion: v2
 name: cloudzero-insights-controller
 description: Provides telemetry to the CloudZero platform to enabling complex cost allocation and analysis.
 type: application
-version: 1.0.0-beta
+version: 1.0.1-beta
 appVersion: "0.0.1"
 dependencies:
   - name: cert-manager
diff --git a/charts/cloudzero-insights-controller/templates/cm.yaml b/charts/cloudzero-insights-controller/templates/cm.yaml
index 0fa001e8..bf9928d5 100644
--- a/charts/cloudzero-insights-controller/templates/cm.yaml
+++ b/charts/cloudzero-insights-controller/templates/cm.yaml
@@ -38,13 +38,15 @@ data:
     {{- end }}
     remote_write:
       send_interval: 1m
-      max_bytes_per_send: 10000000
+      max_bytes_per_send: 500000
       send_timeout: 10s
+      max_retries: 3
     k8s_client:
       timeout: 30s
     database:
       retention_time: 24h
       cleanup_interval: 3h
+      batch_update_size: 500
     {{- with .Values.server.tls }}
     certificate:
       key: {{ .mountPath }}/tls.key
diff --git a/charts/cloudzero-insights-controller/templates/init-job.yaml b/charts/cloudzero-insights-controller/templates/init-job.yaml
index e81dffbf..d9644fac 100644
--- a/charts/cloudzero-insights-controller/templates/init-job.yaml
+++ b/charts/cloudzero-insights-controller/templates/init-job.yaml
@@ -21,8 +21,15 @@ spec:
           command: ["sh", "-c"]
           args:
             - |
-              until curl -X POST -k https://{{ include "insights-controller.serviceName" . }}.{{ .Release.Namespace }}.svc.cluster.local:{{ .Values.server.service.port }}/scrape; do
-                echo "Waiting for service..."
-                sleep 5
+              while true; do
+                echo "Waiting for the tags server to be ready...";
+                if curl -s -o /dev/null -w "%{http_code}" -k https://{{ include "insights-controller.serviceName" . }}.{{ .Release.Namespace }}.svc.cluster.local:{{ .Values.server.service.port }}/healthz | grep -q 200; then
+                  echo "Server is ready, starting scrape job...";
+                  curl -X POST -k https://{{ include "insights-controller.serviceName" . }}.{{ .Release.Namespace }}.svc.cluster.local:{{ .Values.server.service.port }}/scrape;
+                  echo "Scrape process started.";
+                  break;
+                fi;
+                echo "No 200 response from health endpoint. Retrying in 30 seconds...";
+                sleep 30;
               done
 {{- end }}
diff --git a/charts/cloudzero-insights-controller/values.yaml b/charts/cloudzero-insights-controller/values.yaml
index 8a0ba5e2..ba2ac7ec 100644
--- a/charts/cloudzero-insights-controller/values.yaml
+++ b/charts/cloudzero-insights-controller/values.yaml
@@ -18,10 +18,28 @@ labels:
   enabled: null
   patterns:
     # - '.*'
+  resources:
+    pods: true
+    namespaces: true
+    deployments: false
+    statefulsets: false
+    nodes: false
+    jobs: false
+    cronjobs: false
+    daemonsets: false
 annotations:
   enabled: false
   patterns:
     - '.*'
+  resources:
+    pods: true
+    namespaces: true
+    deployments: false
+    statefulsets: false
+    nodes: false
+    jobs: false
+    cronjobs: false
+    daemonsets: false
 
 server:
   replicaCount: 3

From 6d91d105e7923c2a83b945e3cf6558fb29c567d8 Mon Sep 17 00:00:00 2001
From: Daniel Mepham <daniel.mepham@cloudzero.com>
Date: Wed, 20 Nov 2024 17:46:18 -0500
Subject: [PATCH 11/26] Release 1.0.2-beta (#109)

* CP-23051: Change default kube-state-metrics behavior to use Cloudzero subchart (#91)

* override KSM name

* enable ksm by default

* CP-23388: Define Static KubeStateMetrics Target Endpoint (#99)

* add 1.0.2 release doc file

---------

Co-authored-by: bdrennz <146774453+bdrennz@users.noreply.github.com>
---
 .github/workflows/test-chart.yml              | 12 -----
 charts/cloudzero-agent/BETA-INSTALLATION.md   | 20 ++------
 charts/cloudzero-agent/Chart.lock             |  6 +--
 charts/cloudzero-agent/Chart.yaml             |  5 +-
 charts/cloudzero-agent/README.md              |  3 +-
 charts/cloudzero-agent/templates/cm.yaml      | 51 ++++---------------
 charts/cloudzero-agent/values.yaml            |  3 +-
 .../cloudzero-insights-controller/Chart.yaml  |  2 +-
 docs/releases/1.0.2-beta.md                   | 13 +++++
 9 files changed, 35 insertions(+), 80 deletions(-)
 create mode 100644 docs/releases/1.0.2-beta.md

diff --git a/.github/workflows/test-chart.yml b/.github/workflows/test-chart.yml
index 47470911..fc8bbef5 100644
--- a/.github/workflows/test-chart.yml
+++ b/.github/workflows/test-chart.yml
@@ -103,18 +103,6 @@ jobs:
             echo "SKIP_VALIDATIONS=true" >>${GITHUB_ENV}
           fi
 
-      - name: TEST PREP - Update dependency path for local insights-controller testing
-        run: |
-          full_path=$(pwd)/charts/cloudzero-insights-controller
-          awk -v dir="$full_path" '
-            /name: cloudzero-insights-controller/ {flag=1}
-            flag && /repository:/ {sub(/repository: .*/, "repository: file://" dir); flag=0}
-            {print}
-          ' charts/cloudzero-agent/Chart.yaml > temp.yaml && mv temp.yaml charts/cloudzero-agent/Chart.yaml
-          cd charts/cloudzero-insights-controller
-          helm dependency update
-          cd ../../
-
       # Install the chart using our temporary image
       - name: TEST - Install the chart
         id: test_chart_installation
diff --git a/charts/cloudzero-agent/BETA-INSTALLATION.md b/charts/cloudzero-agent/BETA-INSTALLATION.md
index cf4f222a..08a09fc7 100644
--- a/charts/cloudzero-agent/BETA-INSTALLATION.md
+++ b/charts/cloudzero-agent/BETA-INSTALLATION.md
@@ -39,14 +39,7 @@ There are two ways to install a beta version of the chart:
 This method installs the latest beta version available.
 
 ```sh
-helm install <RELEASE_NAME> cloudzero-beta/cloudzero-agent \
-    --devel \
-    --set existingSecretName=<NAME_OF_SECRET> \
-    --set clusterName=<CLUSTER_NAME> \
-    --set-string cloudAccountId=<CLOUD_ACCOUNT_ID> \
-    --set region=<REGION> \
-    --set kube-state-metrics.enabled=<true|false> \
-    --create-namespace
+helm install <RELEASE_NAME> cloudzero-beta/cloudzero-agent -n <NAMESPACE> --create-namespace -f configuration.example.yaml --devel
 ```
 
 - The `--devel` flag allows Helm to consider beta versions when resolving the chart version.
@@ -57,17 +50,10 @@ helm install <RELEASE_NAME> cloudzero-beta/cloudzero-agent \
 If you want to install a specific beta version, specify it using the `--version` flag:
 
 ```sh
-helm install <RELEASE_NAME> cloudzero-beta/cloudzero-agent \
-    --version <CHART_VERSION> \
-    --set existingSecretName=<NAME_OF_SECRET> \
-    --set clusterName=<CLUSTER_NAME> \
-    --set-string cloudAccountId=<CLOUD_ACCOUNT_ID> \
-    --set region=<REGION> \
-    --set kube-state-metrics.enabled=<true|false> \
-    --create-namespace
+helm install <RELEASE_NAME> cloudzero-beta/cloudzero-agent -n <NAMESPACE> --create-namespace -f configuration.example.yaml --version <CHART_VERSION>
 ```
 
-- Replace `<CHART_VERSION>` with the specific beta version (e.g., `0.0.29-beta`).
+- Replace `<CHART_VERSION>` with the specific beta version (e.g., `1.0.0-beta`).
 - This method does not require the `--devel` flag since you are explicitly specifying the version.
 
 ## Listing All Available Versions
diff --git a/charts/cloudzero-agent/Chart.lock b/charts/cloudzero-agent/Chart.lock
index e171ce42..4fe263d1 100644
--- a/charts/cloudzero-agent/Chart.lock
+++ b/charts/cloudzero-agent/Chart.lock
@@ -4,6 +4,6 @@ dependencies:
   version: 5.15.3
 - name: cloudzero-insights-controller
   repository: https://cloudzero.github.io/cloudzero-charts/beta
-  version: 0.0.1
-digest: sha256:87c19ba797e44296e4ac2433e17f24b73bcda8c9ee8b4286bc31c0d7fd0f0224
-generated: "2024-11-13T12:34:06.381064+01:00"
+  version: 1.0.1-beta
+digest: sha256:467fc1a751a584f61a9264a7f071209db408036baf45a62f061e34a444137e0f
+generated: "2024-11-18T12:45:12.292096-05:00"
diff --git a/charts/cloudzero-agent/Chart.yaml b/charts/cloudzero-agent/Chart.yaml
index 746e0f62..64988dcd 100644
--- a/charts/cloudzero-agent/Chart.yaml
+++ b/charts/cloudzero-agent/Chart.yaml
@@ -11,9 +11,10 @@ dependencies:
   - name: kube-state-metrics
     version: "5.15.*"
     repository: https://prometheus-community.github.io/helm-charts
-    condition: kube-state-metrics.enabled
+    condition: kubeStateMetrics.enabled
+    alias: kubeStateMetrics
   - name: cloudzero-insights-controller
-    version: "1.0.0-beta"
+    version: "1.0.1-beta"
     repository: https://cloudzero.github.io/cloudzero-charts/beta
     condition: cloudzero-insights-controller.enabled
     alias: tags
diff --git a/charts/cloudzero-agent/README.md b/charts/cloudzero-agent/README.md
index a544bb51..05f26a8f 100644
--- a/charts/cloudzero-agent/README.md
+++ b/charts/cloudzero-agent/README.md
@@ -14,7 +14,6 @@ For the latest release, see [Releases](https://github.com/Cloudzero/cloudzero-ch
 - Helm 3+
 - A CloudZero API key
 - Each Kubernetes cluster must have a route to the internet and a rule that allows egress from the agent to the CloudZero collector endpoint at https://api.cloudzero.com on port 443
-- A kube-state-metrics exporter running in the cluster, available via Kubernetes Service (see below for details)
 
 ## Installation
 
@@ -192,7 +191,7 @@ A common addition may be to pull the container images from custom image registri
 
 `values-override.yaml`
 ```yaml
-kube-state-metrics:
+kubeStateMetrics:
   enabled: true
   image:
     registry: my-custom-registry.io
diff --git a/charts/cloudzero-agent/templates/cm.yaml b/charts/cloudzero-agent/templates/cm.yaml
index d8bd4e85..c4d271c6 100644
--- a/charts/cloudzero-agent/templates/cm.yaml
+++ b/charts/cloudzero-agent/templates/cm.yaml
@@ -18,11 +18,10 @@ data:
       scrape_interval: {{ .Values.prometheusConfig.globalScrapeInterval }}
     scrape_configs:
       {{- if .Values.prometheusConfig.scrapeJobs.kubeStateMetrics.enabled }}
-      - job_name: cloudzero-service-endpoints # kube_*, node_* metrics
-        honor_labels: true
+      - job_name: static-kube-state-metrics
         honor_timestamps: true
         track_timestamps_staleness: false
-        scrape_interval: {{ .Values.prometheusConfig.scrapeJobs.kubeStateMetrics.scrapeInterval }}
+        scrape_interval: 1m
         scrape_timeout: 10s
         scrape_protocols:
         - OpenMetricsText1.0.0
@@ -34,38 +33,6 @@ data:
         follow_redirects: true
         enable_http2: true
         relabel_configs:
-        - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scrape]
-          separator: ;
-          regex: "true"
-          replacement: $1
-          action: keep
-        - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scrape_slow]
-          separator: ;
-          regex: "true"
-          replacement: $1
-          action: drop
-        - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scheme]
-          separator: ;
-          regex: (https?)
-          target_label: __scheme__
-          replacement: $1
-          action: replace
-        - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_path]
-          separator: ;
-          regex: (.+)
-          target_label: __metrics_path__
-          replacement: $1
-          action: replace
-        - source_labels: [__address__, __meta_kubernetes_service_annotation_prometheus_io_port]
-          separator: ;
-          regex: (.+?)(?::\d+)?;(\d+)
-          target_label: __address__
-          replacement: $1:$2
-          action: replace
-        - separator: ;
-          regex: __meta_kubernetes_service_annotation_prometheus_io_param_(.+)
-          replacement: __param_$1
-          action: labelmap
         - separator: ;
           regex: __meta_kubernetes_service_label_(.+)
           replacement: $1
@@ -92,13 +59,13 @@ data:
         - source_labels: [__name__]
           regex: "^({{ join "|" .Values.kubeMetrics }})$"
           action: keep
-        - action: labelkeep
-          regex: "^({{ include "cloudzero-agent.requiredMetricLabels" . }})$"
-        kubernetes_sd_configs:
-        - role: endpoints
-          kubeconfig_file: ""
-          follow_redirects: true
-          enable_http2: true
+        - separator: ;
+          regex: ^(board_asset_tag|container|created_by_kind|created_by_name|image|instance|name|namespace|node|node_kubernetes_io_instance_type|pod|product_name|provider_id|resource|unit|uid|_.*|label_.*|app.kubernetes.io/*|k8s.*)$
+          replacement: $1
+          action: labelkeep
+        static_configs:
+        - targets:
+            - {{ printf "%s.%s.svc.cluster.local:%d" .Values.kubeStateMetrics.fullnameOverride .Release.Namespace (int .Values.kubeStateMetrics.service.port) }}
       {{- end }}
       {{- if .Values.prometheusConfig.scrapeJobs.cadvisor.enabled }}
       - job_name: cloudzero-nodes-cadvisor # container_* metrics
diff --git a/charts/cloudzero-agent/values.yaml b/charts/cloudzero-agent/values.yaml
index 99d78130..f6444ed6 100644
--- a/charts/cloudzero-agent/values.yaml
+++ b/charts/cloudzero-agent/values.yaml
@@ -60,9 +60,10 @@ tags:
     patterns:
       - '.*'
 
-kube-state-metrics:
+kubeStateMetrics:
   enabled: true
   fullnameOverride: "cloudzero-state-metrics"
+  nameOverride: "cloudzero-state-metrics"
   # Disable CloudZero KSM as a Scrape Target since the service endpoint is explicity defined
   # by the Validators config file.
   prometheusScrape: false
diff --git a/charts/cloudzero-insights-controller/Chart.yaml b/charts/cloudzero-insights-controller/Chart.yaml
index f3b2b744..d46ee99c 100644
--- a/charts/cloudzero-insights-controller/Chart.yaml
+++ b/charts/cloudzero-insights-controller/Chart.yaml
@@ -3,7 +3,7 @@ name: cloudzero-insights-controller
 description: Provides telemetry to the CloudZero platform to enabling complex cost allocation and analysis.
 type: application
 version: 1.0.1-beta
-appVersion: "0.0.1"
+appVersion: "0.0.2"
 dependencies:
   - name: cert-manager
     version: v1.15.3
diff --git a/docs/releases/1.0.2-beta.md b/docs/releases/1.0.2-beta.md
new file mode 100644
index 00000000..c6402b31
--- /dev/null
+++ b/docs/releases/1.0.2-beta.md
@@ -0,0 +1,13 @@
+## [1.0.2-beta](https://github.com/Cloudzero/cloudzero-agent/compare/v1.0.1-beta...v1.0.2-beta) (2024-11-20)
+
+The internal `kube-state-metrics` is now renamed to `cloudzero-state-metrics`. It is enabled by default and set as a static target for the agent.
+
+### Upgrade Steps
+* Upgrade with:
+```sh
+helm upgrade --install <RELEASE_NAME> cloudzero-beta/cloudzero-agent -n <NAMESPACE> --create-namespace -f configuration.example.yaml --version 1.0.2-beta
+```
+See the [beta installation instructions](https://github.com/Cloudzero/cloudzero-charts/blob/develop/charts/cloudzero-agent/BETA-INSTALLATION.md) for further detail
+
+### Improvements
+* **CloudZero Metrics:** The `cloudzero-state-metrics` deployment is enabled/installed by default and set as a static target, improving reliability and performance.

From b039ffc2b462e903428067ede2f017a18ead17f9 Mon Sep 17 00:00:00 2001
From: Daniel Mepham <daniel.mepham@cloudzero.com>
Date: Wed, 20 Nov 2024 19:02:42 -0500
Subject: [PATCH 12/26] move release doc to correct location

---
 {docs => charts/cloudzero-agent/docs}/releases/1.0.2-beta.md | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename {docs => charts/cloudzero-agent/docs}/releases/1.0.2-beta.md (100%)

diff --git a/docs/releases/1.0.2-beta.md b/charts/cloudzero-agent/docs/releases/1.0.2-beta.md
similarity index 100%
rename from docs/releases/1.0.2-beta.md
rename to charts/cloudzero-agent/docs/releases/1.0.2-beta.md

From 58c4546c992f7bfc7e44cef1a4fb1b2dedee6370 Mon Sep 17 00:00:00 2001
From: Automated CZ Release <ops@cloudzero.com>
Date: Thu, 21 Nov 2024 00:05:14 +0000
Subject: [PATCH 13/26] Update Chart.yaml to version 1.0.2-beta

---
 charts/cloudzero-agent/Chart.lock               | 4 ++--
 charts/cloudzero-agent/Chart.yaml               | 2 +-
 charts/cloudzero-insights-controller/Chart.yaml | 2 +-
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/charts/cloudzero-agent/Chart.lock b/charts/cloudzero-agent/Chart.lock
index 4fe263d1..208554b2 100644
--- a/charts/cloudzero-agent/Chart.lock
+++ b/charts/cloudzero-agent/Chart.lock
@@ -5,5 +5,5 @@ dependencies:
 - name: cloudzero-insights-controller
   repository: https://cloudzero.github.io/cloudzero-charts/beta
   version: 1.0.1-beta
-digest: sha256:467fc1a751a584f61a9264a7f071209db408036baf45a62f061e34a444137e0f
-generated: "2024-11-18T12:45:12.292096-05:00"
+digest: sha256:3518ffb88382f3e613cc9b21aac6dc474d634b316aa69cfe7f4e85474844e54b
+generated: "2024-11-21T00:05:12.882895449Z"
diff --git a/charts/cloudzero-agent/Chart.yaml b/charts/cloudzero-agent/Chart.yaml
index 64988dcd..da47c433 100644
--- a/charts/cloudzero-agent/Chart.yaml
+++ b/charts/cloudzero-agent/Chart.yaml
@@ -2,7 +2,7 @@ apiVersion: v2
 name: cloudzero-agent
 description: A chart for using Prometheus in agent mode to send cluster metrics to the CloudZero platform.
 type: application
-version: 1.0.0-beta
+version: 1.0.2-beta
 maintainers:
   - name: CloudZero
     email: support@cloudzero.com
diff --git a/charts/cloudzero-insights-controller/Chart.yaml b/charts/cloudzero-insights-controller/Chart.yaml
index d46ee99c..8898645f 100644
--- a/charts/cloudzero-insights-controller/Chart.yaml
+++ b/charts/cloudzero-insights-controller/Chart.yaml
@@ -2,7 +2,7 @@ apiVersion: v2
 name: cloudzero-insights-controller
 description: Provides telemetry to the CloudZero platform to enabling complex cost allocation and analysis.
 type: application
-version: 1.0.1-beta
+version: 1.0.2-beta
 appVersion: "0.0.2"
 dependencies:
   - name: cert-manager

From 12b204ac44711d3f9043f2d91cf4821e4c1b8ef1 Mon Sep 17 00:00:00 2001
From: Daniel Mepham <daniel.mepham@cloudzero.com>
Date: Wed, 20 Nov 2024 19:24:12 -0500
Subject: [PATCH 14/26] CP-22730: package both charts in beta release (#110)

---
 .github/workflows/build-and-publish-beta-chart.yml | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/build-and-publish-beta-chart.yml b/.github/workflows/build-and-publish-beta-chart.yml
index 8243a206..b381ebc8 100644
--- a/.github/workflows/build-and-publish-beta-chart.yml
+++ b/.github/workflows/build-and-publish-beta-chart.yml
@@ -86,7 +86,9 @@ jobs:
 
       # Step 4: Package and Commit Chart
       - name: Package Chart
-        run: helm package charts/cloudzero-agent/ --destination .deploy
+        run: |
+          helm package charts/cloudzero-agent/ --destination .deploy
+          helm package charts/cloudzero-insights-controller/ --destination .deploy
 
       - name: Commit updated Chart.yaml
         run: |

From 7aa96dfcf5376e788796a0ee5702f234b07cb2aa Mon Sep 17 00:00:00 2001
From: Daniel Mepham <daniel.mepham@cloudzero.com>
Date: Wed, 20 Nov 2024 19:36:59 -0500
Subject: [PATCH 15/26] CP-22730: fix artficat name (#111)

---
 .github/workflows/build-and-publish-beta-chart.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/build-and-publish-beta-chart.yml b/.github/workflows/build-and-publish-beta-chart.yml
index b381ebc8..56783e32 100644
--- a/.github/workflows/build-and-publish-beta-chart.yml
+++ b/.github/workflows/build-and-publish-beta-chart.yml
@@ -103,7 +103,7 @@ jobs:
       - name: Upload Insight Controller Chart as Artifact
         uses: actions/upload-artifact@v4
         with:
-          name: agent-chart
+          name: insights-controller-chart
           path: .deploy/cloudzero-insights-controller-${{ env.NEW_VERSION }}.tgz
 
       - name: Upload Cloudzero Agent Chart as Artifact

From 10045f28b13cfc79b36d8229319afa09adb6d62a Mon Sep 17 00:00:00 2001
From: Daniel Mepham <daniel.mepham@cloudzero.com>
Date: Wed, 20 Nov 2024 20:33:24 -0500
Subject: [PATCH 16/26] CP-22730: fix doc path for github release publish
 (#112)

---
 .github/workflows/build-and-publish-beta-chart.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/build-and-publish-beta-chart.yml b/.github/workflows/build-and-publish-beta-chart.yml
index 56783e32..f7833384 100644
--- a/.github/workflows/build-and-publish-beta-chart.yml
+++ b/.github/workflows/build-and-publish-beta-chart.yml
@@ -156,4 +156,4 @@ jobs:
           files: .deploy/cloudzero-agent-${{ env.NEW_VERSION }}.tgz
           make_latest: false
           target_commitish: ${{ env.COMMIT_HASH }}
-          body_path: ${{ github.workspace }}/docs/releases/${{ env.NEW_VERSION }}.md
+          body_path: ${{ github.workspace }}/charts/cloudzero-agent/docs/releases/${{ env.NEW_VERSION }}.md

From 1e6f62cc76b3fe35f98f4d480c6c6771fc2270a1 Mon Sep 17 00:00:00 2001
From: bdrennz <146774453+bdrennz@users.noreply.github.com>
Date: Thu, 5 Dec 2024 11:43:07 -0800
Subject: [PATCH 17/26] CP-23740 (Feature/1.0.3 beta release): Validate KSM
 Metrics at Install (#116)

* remove unused metric

* add kubemetrics

* bump chart version for beta

* use dev tag for validator

* fix endpoint var name

* allow github to bump version

* simplify metric logic

* update tag

* use dev tag for chart
---
 charts/cloudzero-agent/Chart.yaml                  |  2 +-
 charts/cloudzero-agent/docs/releases/1.0.3-beta.md | 13 +++++++++++++
 charts/cloudzero-agent/templates/validatorcm.yaml  |  6 ++++--
 charts/cloudzero-agent/values.yaml                 |  3 +--
 charts/cloudzero-insights-controller/Chart.yaml    |  2 +-
 5 files changed, 20 insertions(+), 6 deletions(-)
 create mode 100644 charts/cloudzero-agent/docs/releases/1.0.3-beta.md

diff --git a/charts/cloudzero-agent/Chart.yaml b/charts/cloudzero-agent/Chart.yaml
index da47c433..37ce8f5b 100644
--- a/charts/cloudzero-agent/Chart.yaml
+++ b/charts/cloudzero-agent/Chart.yaml
@@ -2,7 +2,7 @@ apiVersion: v2
 name: cloudzero-agent
 description: A chart for using Prometheus in agent mode to send cluster metrics to the CloudZero platform.
 type: application
-version: 1.0.2-beta
+version: 0.0.0-dev
 maintainers:
   - name: CloudZero
     email: support@cloudzero.com
diff --git a/charts/cloudzero-agent/docs/releases/1.0.3-beta.md b/charts/cloudzero-agent/docs/releases/1.0.3-beta.md
new file mode 100644
index 00000000..e82381ef
--- /dev/null
+++ b/charts/cloudzero-agent/docs/releases/1.0.3-beta.md
@@ -0,0 +1,13 @@
+## [1.0.3-beta](https://github.com/Cloudzero/cloudzero-agent/compare/v1.0.1-beta...v1.0.3-beta) (2024-11-20)
+
+The Agent now validates the existence of all required KSM metrics during the `post-start` phase.
+
+### Upgrade Steps
+* Upgrade with:
+```sh
+helm upgrade --install <RELEASE_NAME> cloudzero-beta/cloudzero-agent -n <NAMESPACE> --create-namespace -f configuration.example.yaml --version 1.0.3-beta
+```
+See the [beta installation instructions](https://github.com/Cloudzero/cloudzero-charts/blob/develop/charts/cloudzero-agent/BETA-INSTALLATION.md) for further detail
+
+### Improvements
+* The Validator check (kube_state_metrics_reachable) now validates the existence of all required KSM metrics.
diff --git a/charts/cloudzero-agent/templates/validatorcm.yaml b/charts/cloudzero-agent/templates/validatorcm.yaml
index 80b33a93..cd9b8259 100644
--- a/charts/cloudzero-agent/templates/validatorcm.yaml
+++ b/charts/cloudzero-agent/templates/validatorcm.yaml
@@ -33,7 +33,7 @@ data:
       {{- if .Values.validator.serviceEndpoints.kubeStateMetrics }}
       kube_state_metrics_service_endpoint: http://{{ .Values.validator.serviceEndpoints.kubeStateMetrics }}/
       {{- else }}
-      kube_state_metrics_service_endpoint: http://{{- if .Release.Name }}{{.Release.Name}}-{{- end }}state-metrics:8080/
+      kube_state_metrics_service_endpoint: http://{{ .Values.kubeStateMetrics.nameOverride }}.{{ .Release.Namespace }}.svc.cluster.local:{{ .Values.kubeStateMetrics.service.port }}
       {{- end }}
       {{- if .Values.validator.serviceEndpoints.prometheusNodeExporter }}
       prometheus_node_exporter_service_endpoint: http://{{ .Values.validator.serviceEndpoints.prometheusNodeExporter }}/
@@ -41,6 +41,8 @@ data:
       prometheus_node_exporter_service_endpoint: http://{{- if .Release.Name }}{{.Release.Name}}-{{- end }}prometheus-node-exporter:9100/
       {{- end }}
       executable: /bin/prometheus
+      kube_metrics:
+        {{- toYaml .Values.kubeMetrics | nindent 8 }}
       configurations:
         - /etc/prometheus/prometheus.yml
         - /etc/config/prometheus/configmaps/prometheus.yml
@@ -57,7 +59,7 @@ data:
           checks:
             - k8s_version
             - kube_state_metrics_reachable
-            - node_exporter_reachable
+            #- node_exporter_reachable
             - prometheus_version
             - scrape_cfg
         - name: pre-stop
diff --git a/charts/cloudzero-agent/values.yaml b/charts/cloudzero-agent/values.yaml
index f6444ed6..4e06e542 100644
--- a/charts/cloudzero-agent/values.yaml
+++ b/charts/cloudzero-agent/values.yaml
@@ -22,7 +22,6 @@ kubeMetrics:
   - kube_pod_container_resource_requests
   - kube_pod_labels
   - kube_pod_info
-  - node_dmi_info
 containerMetrics:
   - container_cpu_usage_seconds_total
   - container_memory_working_set_bytes
@@ -84,7 +83,7 @@ validator:
   name: env-validator
   image:
     repository: ghcr.io/cloudzero/cloudzero-agent-validator/cloudzero-agent-validator
-    tag: 0.4.1
+    tag: 0.9.0
     digest:
     pullPolicy: Always
 
diff --git a/charts/cloudzero-insights-controller/Chart.yaml b/charts/cloudzero-insights-controller/Chart.yaml
index 8898645f..548dc81a 100644
--- a/charts/cloudzero-insights-controller/Chart.yaml
+++ b/charts/cloudzero-insights-controller/Chart.yaml
@@ -2,7 +2,7 @@ apiVersion: v2
 name: cloudzero-insights-controller
 description: Provides telemetry to the CloudZero platform to enabling complex cost allocation and analysis.
 type: application
-version: 1.0.2-beta
+version: 0.0.0-dev
 appVersion: "0.0.2"
 dependencies:
   - name: cert-manager

From 45f242d86ff5c6d9fb90142933354e7131b1b5a4 Mon Sep 17 00:00:00 2001
From: Daniel Mepham <daniel.mepham@cloudzero.com>
Date: Tue, 10 Dec 2024 13:58:34 -0500
Subject: [PATCH 18/26] [CP-23429] merge insights-controller into main chart
 (#117)

* insights-controller added to agent chart
---
 .github/workflows/test-chart.yml              |  18 +--
 charts/cloudzero-agent/Chart.lock             |  10 +-
 charts/cloudzero-agent/Chart.yaml             |   9 +-
 charts/cloudzero-agent/README.md              |  54 ++++---
 .../configuration.example.yaml                |   2 +-
 charts/cloudzero-agent/templates/_helpers.tpl | 143 ++++++++++++++++--
 .../templates/certificate.yaml                |  28 ++++
 .../templates/clusterrole.yaml                |  18 +++
 .../templates/clusterrolebinding.yaml         |   4 +-
 charts/cloudzero-agent/templates/cm.yaml      |  47 +++++-
 charts/cloudzero-agent/templates/deploy.yaml  |  15 +-
 .../cloudzero-agent/templates/init-job.yaml   |  35 +++++
 .../templates/insights-deploy.yaml            | 103 +++++++++++++
 charts/cloudzero-agent/templates/issuer.yaml  |  10 ++
 charts/cloudzero-agent/templates/secret.yaml  |   4 +-
 charts/cloudzero-agent/templates/service.yaml |  15 ++
 .../templates/serviceaccount.yaml             |  10 +-
 .../cloudzero-agent/templates/webhooks.yaml   |  38 +++++
 charts/cloudzero-agent/values.yaml            | 132 +++++++++++++---
 .../templates/init-job.yaml                   |   2 +-
 20 files changed, 598 insertions(+), 99 deletions(-)
 create mode 100644 charts/cloudzero-agent/templates/certificate.yaml
 create mode 100644 charts/cloudzero-agent/templates/init-job.yaml
 create mode 100644 charts/cloudzero-agent/templates/insights-deploy.yaml
 create mode 100644 charts/cloudzero-agent/templates/issuer.yaml
 create mode 100644 charts/cloudzero-agent/templates/service.yaml
 create mode 100644 charts/cloudzero-agent/templates/webhooks.yaml

diff --git a/.github/workflows/test-chart.yml b/.github/workflows/test-chart.yml
index fc8bbef5..cff2b8d1 100644
--- a/.github/workflows/test-chart.yml
+++ b/.github/workflows/test-chart.yml
@@ -36,6 +36,7 @@ jobs:
         env:
           # Agent Chart settings (prom repo is to work around issue with chart-testing tool)
           PROM_CHART_REPO: https://prometheus-community.github.io/helm-charts
+          JETSTACK_CHART_REPO: https://charts.jetstack.io
           CZ_CHART_REPO: https://cloudzero.github.io/cloudzero-charts
           CZ_CHART_BETA_REPO: https://cloudzero.github.io/cloudzero-charts/beta
           CLUSTER_NAME: cz-node-agent-ci
@@ -47,8 +48,7 @@ jobs:
           helm dependency update
           ct lint --debug --charts . \
             --chart-repos=kube-state-metrics=$PROM_CHART_REPO \
-            --chart-repos=prometheus-node-exporter=$PROM_CHART_REPO \
-            --chart-repos=cloudzero-insights-controller=$CZ_CHART_BETA_REPO \
+            --chart-repos=cert-manager=$JETSTACK_CHART_REPO \
             --helm-lint-extra-args "--set=existingSecretName=api-token,clusterName=$CLUSTER_NAME,cloudAccountId=$CLOUD_ACCOUNT_ID,region=$REGION"
 
   # This job tests the chart on a KinD cluster
@@ -111,6 +111,7 @@ jobs:
           NAMESPACE: monitoring
           # Agent Chart settings (prom repo is to work around issue with chart-testing tool)
           PROM_CHART_REPO: https://prometheus-community.github.io/helm-charts
+          JETSTACK_CHART_REPO: https://charts.jetstack.io
           CZ_CHART_REPO: https://cloudzero.github.io/cloudzero-charts
           CZ_CHART_BETA_REPO: https://cloudzero.github.io/cloudzero-charts/beta
           CLUSTER_NAME: cz-node-agent-ci
@@ -125,18 +126,15 @@ jobs:
           helm dependency update
           ct install --charts . \
             --chart-repos=kube-state-metrics=$PROM_CHART_REPO \
-            --chart-repos=prometheus-node-exporter=$PROM_CHART_REPO \
-            --chart-repos=cloudzero-insights-controller=$CZ_CHART_BETA_REPO \
+            --chart-repos=cert-manager=$JETSTACK_CHART_REPO \
             --namespace $NAMESPACE \
             --helm-extra-set-args "\
-              --set=global.existingSecretName=api-token \
+              --set=existingSecretName=api-token \
               --set=host=dev-api.cloudzero.com \
               --set=clusterName=$CLUSTER_NAME \
               --set=cloudAccountId=$CLOUD_ACCOUNT_ID \
               --set=region=$REGION \
-              --set=kube-state-metrics.enabled=true \
-              --set=prometheus-node-exporter.enabled=true \
-              --set=tags.enabled=true \
-              --set=tags.labels.enabled=true \
-              --set=tags.labels.patterns[0]='.*' \
+              --set=insightsController.enabled=true \
+              --set=insightsController.labels.enabled=true \
+              --set=insightsController.labels.patterns[0]='.*' \
               "
diff --git a/charts/cloudzero-agent/Chart.lock b/charts/cloudzero-agent/Chart.lock
index 208554b2..e49b79bc 100644
--- a/charts/cloudzero-agent/Chart.lock
+++ b/charts/cloudzero-agent/Chart.lock
@@ -2,8 +2,8 @@ dependencies:
 - name: kube-state-metrics
   repository: https://prometheus-community.github.io/helm-charts
   version: 5.15.3
-- name: cloudzero-insights-controller
-  repository: https://cloudzero.github.io/cloudzero-charts/beta
-  version: 1.0.1-beta
-digest: sha256:3518ffb88382f3e613cc9b21aac6dc474d634b316aa69cfe7f4e85474844e54b
-generated: "2024-11-21T00:05:12.882895449Z"
+- name: cert-manager
+  repository: https://charts.jetstack.io
+  version: v1.15.3
+digest: sha256:6135d395ad03a38895bd8f1d71fb84856d76b89978b5b4250f6d154c6e33b128
+generated: "2024-12-06T14:53:41.361206-05:00"
diff --git a/charts/cloudzero-agent/Chart.yaml b/charts/cloudzero-agent/Chart.yaml
index 37ce8f5b..daaaf820 100644
--- a/charts/cloudzero-agent/Chart.yaml
+++ b/charts/cloudzero-agent/Chart.yaml
@@ -13,8 +13,7 @@ dependencies:
     repository: https://prometheus-community.github.io/helm-charts
     condition: kubeStateMetrics.enabled
     alias: kubeStateMetrics
-  - name: cloudzero-insights-controller
-    version: "1.0.1-beta"
-    repository: https://cloudzero.github.io/cloudzero-charts/beta
-    condition: cloudzero-insights-controller.enabled
-    alias: tags
+  - name: cert-manager
+    version: v1.15.3
+    repository: https://charts.jetstack.io
+    condition: cert-manager.enabled
diff --git a/charts/cloudzero-agent/README.md b/charts/cloudzero-agent/README.md
index 05f26a8f..9d146404 100644
--- a/charts/cloudzero-agent/README.md
+++ b/charts/cloudzero-agent/README.md
@@ -37,13 +37,14 @@ If installing with Helm directly, execute the following steps:
 helm repo update
 ```
 
-2. Ensure that required CRDs are installed for certifiacte management. If you have more specific requirements around managing TLS certificates, see the [Certificate Management](https://github.com/Cloudzero/cloudzero-charts/tree/develop/charts/cloudzero-insights-controller#deployment-configurations-and-certificate-management) section in the `cloudzero-insights-controller` subchart.
+2. Ensure that required CRDs are installed for certificate management. If you have more specific requirements around managing TLS certificates, see the [Certificate Management](#certificate-management) section.
 ```console
 helm install <RELEASE_NAME> cloudzero/cloudzero-agent \
-    --set tags.webhook.issuer.enabled=false \
-    --set tags.webhook.certificate.enabled=false \
-    --set tags.cert-manager.installCRDs=true
+    --set insightsController.webhook.issuer.enabled=false \
+    --set insightsController.webhook.certificate.enabled=false \
+    --set insightsController.cert-manager.installCRDs=true
 ```
+Alternatively, [install the cert-manager CRDs directly](https://cert-manager.io/docs/installation/helm/).
 
 3. Fill out all required fields in the `configuration.example.yaml` file in this directory. Rename the file as necessary. Below is an example of a completed configuration file:
 ```yaml
@@ -53,14 +54,13 @@ cloudAccountId: YOUR_CLOUD_ACCOUNT_ID
 clusterName: YOUR_CLUSTER_NAME
 # -- Region the cluster is running in.
 region: YOUR_CLOUD_REGION
-global:
-  # -- CloudZero API key. Required if useExistingSecret is false.
-  apiKey: YOUR_CLOUDZERO_API_KEY
-  # -- If set, the agent will use the API key in this Secret to authenticate with CloudZero.
-  existingSecretName: YOUR_EXISTING_API_KEY_K8S_SECRET
-
-# label and annotation configuration (referred together as 'tags'). See the below 'Labels and Annotations' section for more details.
-tags:
+# -- CloudZero API key. Required if useExistingSecret is false.
+apiKey: YOUR_CLOUDZERO_API_KEY
+# -- If set, the agent will use the API key in this Secret to authenticate with CloudZero.
+existingSecretName: YOUR_EXISTING_API_KEY_K8S_SECRET
+
+# label and annotation configuration (managed in the 'insightsController' section). See the below 'Labels and Annotations' section for more details.
+insightsController:
   # -- By default, a ValidatingAdmissionWebhook will be deployed that records all created labels and annotations
   enabled: true
   labels:
@@ -68,13 +68,13 @@ tags:
     enabled: true
     # -- This value MUST be set to a list of regular expressions which will be used to gather labels from pods, deployments, statefulsets, daemonsets, cronjobs, jobs, nodes, and namespaces
     patterns:
-      - '^foo' # -- match all labels whose key starts with "foo"
-      - 'bar$' # -- match all labels whose key ends with "bar"
+      - '^foo' # -- Match all labels whose key starts with "foo"
+      - 'bar$' # -- Match all labels whose key ends with "bar"
   annotations:
     # -- By default, the gathering of annotations is not enabled. To enable, set this field to true
     enabled: false
     patterns:
-      - '.*'
+      - '.*' # -- match all annotations. This is not recommended.
 ```
 
 4. Install the helm chart using the completed configuration file:
@@ -104,11 +104,11 @@ There are several mandatory values that must be specified for the chart to insta
 | cloudAccountId    | string | `nil`                 | Account ID in AWS or Subscription ID in Azure or Project Number in GCP where the cluster is running. Must be a string due to Helm limitations.  |
 | clusterName       | string | `nil`                 | Name of the cluster. Must be RFC 1123 compliant.                                                                         |
 | host              | string | `"api.cloudzero.com"` | CloudZero host to send metrics to.                                                                                      |
-| global.apiKey            | string | `nil`                 | The CloudZero API key to use for exporting metrics. Only used if `global.existingSecretName` is not set.                       |
-| global.existingSecretName| string | `nil`                 | Name of the secret that contains the CloudZero API key. Required if not providing the API key via `apiKey`.             |
+| apiKey            | string | `nil`                 | The CloudZero API key to use for exporting metrics. Only used if `global.existingSecretName` is not set.                       |
+| existingSecretName| string | `nil`                 | Name of the secret that contains the CloudZero API key. Required if not providing the API key via `apiKey`.             |
 | region            | string | `nil`                 | Region where the cluster is running (e.g., `us-east-1`, `eastus`). For more information, see AWS or Azure documentation. |
-| tags.labels.enabled            | string | `nil`                 | If enabled, labels for pods, deployments, statefulsets, daemonsets, cronjobs, jobs, nodes, and namespaces |
-| tags.labels.patterns            | string | `nil`                 | An array of regular expressions, which are used to match specific label keys |
+| insightsController.labels.enabled            | string | `nil`                 | If enabled, labels for pods, deployments, statefulsets, daemonsets, cronjobs, jobs, nodes, and namespaces |
+| insightsController.labels.patterns            | string | `nil`                 | An array of regular expressions, which are used to match specific label keys |
 
 #### Overriding Default Values
 
@@ -133,7 +133,7 @@ You can use the `--set` flag in Helm commands to directly set or override specif
 
 ```console
 helm install <RELEASE_NAME> cloudzero/cloudzero-agent \
-    --set global.existingSecretName=<NAME_OF_SECRET> \
+    --set existingSecretName=<NAME_OF_SECRET> \
     --set clusterName=<CLUSTER_NAME> \
     --set-string cloudAccountId=<CLOUD_ACCOUNT_ID> \
     --set region=<REGION> \
@@ -154,11 +154,17 @@ This chart allows the exporting of labels and annotations from the following res
 - `Namespace`
 
 Additional Notes:
-- By default, only labels from pods and namespaces are exported. To enabled more resources, see the `webhooks.configurations` section of the `values.yaml` file.
-- Labels and annotations exports are managed by a subchart, `cloudzero-insights-controller`, which is also maintained in this repository.
+- By default, only labels from pods and namespaces are exported. To enable more resources, see the `insightsController.labels.resources` and `insightsController.annotations.resources` section of the `values.yaml` file.
+- Labels and annotations exports are managed in the `insightsController` section of the `values.yaml` file.
 - To disambiguate labels/annotations between resources, a prefix representing the resource type is prepended to the label key in the Explorer page. For example, a `foo=bar` node label would be presented as `node:foo: bar`. The exception is pod labels which do not have resource prefixes for backward compatibility with previous versions.
-- Annotations are not exported by default; see the `tags.annotations.enabled` setting to enable. To disambiguate annotations from labels, an `annotation` prefix is prepended to the annotation key; i.e., an `foo: bar` annotation on a namespace would be represented in the Explorer as `node:annotation:foo: bar`
-- For both labels and annotations, the `enabled` flag applies across all resource types; i.e., setting `true` for `tags.labels.enabled` enabels label exporting for labels on pods, nodes, deployments, statefulsets, namespaces, daemonets, jobs, and cronjobs. Specific resources can be disabled by altering the `webhooks.configurations` configuration.
+- Annotations are not exported by default; see the `insightsController.annotations.enabled` setting to enable. To disambiguate annotations from labels, an `annotation` prefix is prepended to the annotation key; i.e., an `foo: bar` annotation on a namespace would be represented in the Explorer as `node:annotation:foo: bar`
+- For both labels and annotations, the `patterns` array applies across all resource types; i.e., setting `['^foo']` for `insightsController.labels.patterns` will match label keys that start with `foo` for all resource types set to `true` in `insightsController.labels.resources`.
+
+### Certificate Management
+
+This chart contains a `ValidatingWebhookConfiguration` resource, which requires a certificate in order validate requests to the webhook server. See related Kubernetes documentation [here](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#configure-admission-webhooks-on-the-fly).
+
+The default behavior of the chart is to deploy [cert-manager](https://github.com/cert-manager/cert-manager/tree/master) to create and manage the certificate.
 
 ### Secret Management
 
diff --git a/charts/cloudzero-agent/configuration.example.yaml b/charts/cloudzero-agent/configuration.example.yaml
index 6d9bb8b6..bfbaccad 100644
--- a/charts/cloudzero-agent/configuration.example.yaml
+++ b/charts/cloudzero-agent/configuration.example.yaml
@@ -12,7 +12,7 @@ global:
   existingSecretName: null
 
 # label and annotation configuration:
-tags:
+insightsController:
   # -- By default, a ValidatingAdmissionWebhook will be deployed that records all created labels and annotations
   enabled: true
   labels:
diff --git a/charts/cloudzero-agent/templates/_helpers.tpl b/charts/cloudzero-agent/templates/_helpers.tpl
index c21eb5ec..97858908 100644
--- a/charts/cloudzero-agent/templates/_helpers.tpl
+++ b/charts/cloudzero-agent/templates/_helpers.tpl
@@ -14,22 +14,25 @@ Create chart name and version as used by the chart label.
 
 {{/* Define the secret name which holds the CloudZero API key */}}
 {{ define "cloudzero-agent.secretName" -}}
-{{ .Values.global.existingSecretName | default (printf "%s-api-key" .Release.Name) }}
+{{ .Values.existingSecretName | default (printf "%s-api-key" .Release.Name) }}
 {{- end}}
 
 {{/* Define the path and filename on the container filesystem which holds the CloudZero API key */}}
 {{ define "cloudzero-agent.secretFileFullPath" -}}
-{{ printf "%s%s" .Values.server.containerSecretFilePath .Values.server.containerSecretFileName }}
+{{ printf "%s%s" .Values.serverConfig.containerSecretFilePath .Values.serverConfig.containerSecretFileName }}
 {{- end}}
 
+{{/*
+Name for the validating webhook
+*/}}
+{{- define "cloudzero-agent.validatingWebhookName" -}}
+{{- printf "%s.%s.svc" (include "cloudzero-agent.validatingWebhookConfigName" .) .Release.Namespace }}
+{{- end }}
+
 {{ define "cloudzero-agent.configMapName" -}}
 {{ .Values.configMapNameOverride | default (printf "%s-configuration" .Release.Name) }}
 {{- end}}
 
-{{ define "cloudzero-agent.cloudzeroConfigMapName" -}}
-{{ .Values.cloudzeroConfigMapNameOverride | default (printf "%s-cloudzero-configuration" .Release.Name) }}
-{{- end}}
-
 {{ define "cloudzero-agent.validatorConfigMapName" -}}
 {{- printf "%s-validator-configuration" .Release.Name -}}
 {{- end}}
@@ -75,7 +78,6 @@ app.kubernetes.io/part-of: {{ include "cloudzero-agent.name" . }}
 {{ include "cloudzero-agent.common.metaLabels" . }}
 {{- end -}}
 
-
 {{/*
 Define the cloudzero-agent.namespace template if set with forceNamespace or .Release.Namespace is set
 */}}
@@ -86,11 +88,11 @@ Define the cloudzero-agent.namespace template if set with forceNamespace or .Rel
 {{/*
 Create the name of the service account to use for the server component
 */}}
-{{- define "cloudzero-agent.serviceAccountName.server" -}}
-{{- if .Values.serviceAccounts.server.create -}}
-    {{ default (include "cloudzero-agent.server.fullname" .) .Values.serviceAccounts.server.name }}
+{{- define "cloudzero-agent.serviceAccountName" -}}
+{{- if .Values.serviceAccount.create -}}
+    {{ default (include "cloudzero-agent.server.fullname" .) .Values.serviceAccount.name }}
 {{- else -}}
-    {{ default "default" .Values.serviceAccounts.server.name }}
+    {{ default "default" .Values.server.serviceAccount.name }}
 {{- end -}}
 {{- end -}}
 
@@ -153,3 +155,122 @@ Required metric labels
 {{- $result := join "|" $total -}}
 {{- $result -}}
 {{- end -}}
+
+
+{{/*
+Insights Controller
+*/}}
+
+{{/*
+Create common matchLabels for webhook server
+*/}}
+{{- define "cloudzero-agent.insightsController.common.matchLabels" -}}
+app.kubernetes.io/name: {{ include "cloudzero-agent.name" . }}
+app.kubernetes.io/instance: {{ .Release.Name }}
+{{- end -}}
+
+{{- define "cloudzero-agent.insightsController.server.matchLabels" -}}
+app.kubernetes.io/component: {{ .Values.insightsController.server.name }}
+{{ include "cloudzero-agent.common.matchLabels" . }}
+{{- end -}}
+
+{{- define "cloudzero-agent.insightsController.initJob.matchLabels" -}}
+app.kubernetes.io/component: {{ include "cloudzero-agent.initJobName" . }}
+{{ include "cloudzero-agent.common.matchLabels" . }}
+{{- end -}}
+
+{{/*
+Service selector labels
+*/}}
+{{- define "cloudzero-agent.selectorLabels" -}}
+{{ include "cloudzero-agent.common.matchLabels" . }}
+{{ include "cloudzero-agent.insightsController.server.matchLabels" . }}
+{{- end }}
+
+{{- define "cloudzero-agent.insightsController.labels" -}}
+{{ include "cloudzero-agent.insightsController.server.matchLabels" . }}
+{{ include "cloudzero-agent.common.metaLabels" . }}
+{{- end -}}
+
+{{/*
+Create a fully qualified webhook server name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+*/}}
+{{- define "cloudzero-agent.insightsController.server.webhookFullname" -}}
+{{- if .Values.server.fullnameOverride -}}
+{{- .Values.server.fullnameOverride | trunc 63 | trimSuffix "-" -}}-webhook
+{{- else -}}
+{{- $name := default .Chart.Name .Values.nameOverride -}}
+{{- if contains $name .Release.Name -}}
+{{- printf "%s-%s" .Release.Name .Values.insightsController.server.name | trunc 63 | trimSuffix "-" -}}
+{{- else -}}
+{{- printf "%s-%s-%s" .Release.Name $name .Values.insightsController.server.name | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+
+{{/*
+Name for the webhook server service
+*/}}
+{{- define "cloudzero-agent.serviceName" -}}
+{{- printf "%s-svc" (include "cloudzero-agent.insightsController.server.webhookFullname" .) }}
+{{- end }}
+
+{{/*
+Name for the validating webhook configuration resource
+*/}}
+{{- define "cloudzero-agent.validatingWebhookConfigName" -}}
+{{- printf "%s-webhook" (include "cloudzero-agent.insightsController.server.webhookFullname" .) }}
+{{- end }}
+
+{{/*
+Name for the certificate secret
+*/}}
+{{- define "cloudzero-agent.tlsSecretName" -}}
+{{- printf "%s-tls" (include "cloudzero-agent.insightsController.server.webhookFullname" .) }}
+{{- end }}
+
+
+{{ define "cloudzero-agent.webhookConfigMapName" -}}
+{{ .Values.insightsController.ConfigMapNameOverride | default (printf "%s-webhook-configuration" .Release.Name) }}
+{{- end}}
+
+{{/*
+Mount path for the insights server configuration file
+*/}}
+{{- define "cloudzero-agent.insightsController.configurationMountPath" -}}
+{{- default .Values.insightsController.configurationMountPath (printf "/etc/%s-insights" .Chart.Name)  }}
+{{- end }}
+
+{{/*
+Name for the issuer resource
+*/}}
+{{- define "cloudzero-agent.issuerName" -}}
+{{- printf "%s-issuer" (include "cloudzero-agent.insightsController.server.webhookFullname" .) }}
+{{- end }}
+
+{{/*
+Name for the job resource
+*/}}
+{{- define "cloudzero-agent.initJobName" -}}
+{{- printf "%s-init" (include "cloudzero-agent.insightsController.server.webhookFullname" .) }}
+{{- end }}
+
+{{/*
+Annotations for the webhooks
+*/}}
+{{- define "cloudzero-agent.webhooks.annotations" -}}
+{{- if .Values.insightsController.webhooks.annotations }}
+{{ toYaml .Values.insightsController.webhook.annotations }}
+{{- end }}
+{{- if and .Values.insightsController.webhooks.certificate.enabled .Values.insightsController.webhooks.issuer.enabled }}
+cert-manager.io/inject-ca-from: {{ .Values.insightsController.webhooks.caInjection | default (printf "%s/%s" .Release.Namespace (include "cloudzero-agent.certificateName" .)) }}
+{{- end }}
+{{- end }}
+
+{{/*
+Name for the certificate resource
+*/}}
+{{- define "cloudzero-agent.certificateName" -}}
+{{- printf "%s-certificate" (include "cloudzero-agent.insightsController.server.webhookFullname" .) }}
+{{- end }}
diff --git a/charts/cloudzero-agent/templates/certificate.yaml b/charts/cloudzero-agent/templates/certificate.yaml
new file mode 100644
index 00000000..97b92643
--- /dev/null
+++ b/charts/cloudzero-agent/templates/certificate.yaml
@@ -0,0 +1,28 @@
+{{ if and .Values.insightsController.webhooks.certificate.enabled .Values.insightsController.enabled }}
+---
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: {{ include "cloudzero-agent.certificateName" . }}
+  namespace: {{ .Release.Namespace }}
+spec:
+  secretName: {{ include "cloudzero-agent.tlsSecretName" .}}
+  secretTemplate:
+    {{- with .Values.secretAnnotations }}
+    annotations:
+    {{- toYaml . | nindent 4 }}
+    {{- end }}
+    labels:
+      {{- include "cloudzero-agent.insightsController.labels" . | nindent 6 }}
+  privateKey:
+    algorithm: RSA
+    encoding: PKCS1
+    size: 2048
+  duration: 2160h # 90d
+  renewBefore: 360h # 15d
+  dnsNames:
+    - {{ include "cloudzero-agent.serviceName" . }}.{{ .Release.Namespace }}.svc
+  issuerRef:
+    name: {{ include "cloudzero-agent.issuerName" . }}
+    kind: Issuer
+{{ end }}
diff --git a/charts/cloudzero-agent/templates/clusterrole.yaml b/charts/cloudzero-agent/templates/clusterrole.yaml
index baee8c70..52806d6d 100644
--- a/charts/cloudzero-agent/templates/clusterrole.yaml
+++ b/charts/cloudzero-agent/templates/clusterrole.yaml
@@ -6,10 +6,28 @@ metadata:
     {{- include "cloudzero-agent.server.labels" . | nindent 4 }}
   name: {{ include "cloudzero-agent.clusterRoleName" . }}
 rules:
+  - apiGroups:
+      - "apps"
+    resources:
+      - "deployments"
+      - "statefulsets"
+      - "daemonsets"
+    verbs:
+      - "get"
+      - "list"
+  - apiGroups:
+      - "batch"
+    resources:
+      - "jobs"
+      - "cronjobs"
+    verbs:
+      - "get"
+      - "list"
   - apiGroups:
       - ""
     resources:
       - endpoints
+      - namespaces
       - nodes
       - nodes/proxy
       - nodes/metrics
diff --git a/charts/cloudzero-agent/templates/clusterrolebinding.yaml b/charts/cloudzero-agent/templates/clusterrolebinding.yaml
index 081a31c8..330fd930 100644
--- a/charts/cloudzero-agent/templates/clusterrolebinding.yaml
+++ b/charts/cloudzero-agent/templates/clusterrolebinding.yaml
@@ -1,4 +1,4 @@
-{{- if and .Values.rbac.create (empty .Values.server.namespaces) (empty .Values.server.useExistingClusterRoleName) -}}
+{{- if and .Values.rbac.create -}}
 apiVersion: {{ template "cloudzero-agent.rbac.apiVersion" . }}
 kind: ClusterRoleBinding
 metadata:
@@ -7,7 +7,7 @@ metadata:
   name: {{ include "cloudzero-agent.clusterRoleName" . }}
 subjects:
   - kind: ServiceAccount
-    name: {{ template "cloudzero-agent.serviceAccountName.server" . }}
+    name: {{ template "cloudzero-agent.serviceAccountName" . }}
     namespace: {{ include "cloudzero-agent.namespace" . }}
 roleRef:
   apiGroup: rbac.authorization.k8s.io
diff --git a/charts/cloudzero-agent/templates/cm.yaml b/charts/cloudzero-agent/templates/cm.yaml
index c4d271c6..3b063962 100644
--- a/charts/cloudzero-agent/templates/cm.yaml
+++ b/charts/cloudzero-agent/templates/cm.yaml
@@ -133,23 +133,64 @@ data:
             action: keep
         metadata_config:
           send: false
-{{- if .Values.tags.enabled }}
+{{- if .Values.insightsController.enabled }}
 ---
+{{- with .Values.insightsController }}
+{{- if not (and .labels.enabled .labels.patterns) }}
+{{- $msg := "\n\nThe required field(s) 'insightsController.labels.enabled' and/or 'insightsController.labels.patterns' is not set! See the README.md for more information." }}
+{{- $enabledMsg:=""}}
+{{- $patternMsg:=""}}
+{{- if not .labels.enabled  }}
+{{- $enabledMsg = "Ensure that 'insightsController.labels.enabled' is a boolean (true or false). Set 'true' to enable exporting labels."}}
+{{- end }}
+{{- if not .labels.patterns }}
+{{- $patternMsg = "The required field 'labels.patterns' is not set or set incorrectly. It must be an array of regular expressions that match label keys to be exported."}}
+{{- end }}
+{{- fail (printf "\n %s \n %s \n %s" $msg $enabledMsg $patternMsg) }}
+{{- end }}
+{{- end }}
 apiVersion: v1
 kind: ConfigMap
 metadata:
   labels:
     {{- include "cloudzero-agent.server.labels" . | nindent 4 }}
-  name: {{ include "cloudzero-agent.cloudzeroConfigMapName" . }}
+  name: {{ include "cloudzero-agent.webhookConfigMapName" . }}
   namespace: {{ include "cloudzero-agent.namespace" . }}
   {{- with .Values.prometheusConfig.configMapAnnotations }}
   annotations:
   {{- toYaml . | nindent 4 }}
   {{- end }}
 data:
-  cloudzero-config.yaml: |-
+  server-config.yaml: |-
     cloud_account_id: {{ .Values.cloudAccountId }}
     region: {{ .Values.region }}
     cluster_name: {{ .Values.clusterName }}
     host: {{ .Values.host }}
+    remote_write:
+      send_interval: 1m
+      max_bytes_per_send: 500000
+      send_timeout: 10s
+      max_retries: 3
+    k8s_client:
+      timeout: 30s
+    database:
+      retention_time: 24h
+      cleanup_interval: 3h
+      batch_update_size: 500
+    api_key_path: {{ include "cloudzero-agent.secretFileFullPath" . }}
+    {{- with .Values.insightsController.server }}
+    certificate:
+      key: {{ .tls.mountPath }}/tls.key
+      cert: {{ .tls.mountPath }}/tls.crt
+    server:
+      port: {{ .port }}
+      read_timeout: {{ .read_timeout }}
+      write_timeout: {{ .write_timeout }}
+      idle_timeout: {{ .idle_timeout }}
+    {{- end }}
+    filters:
+      labels:
+        {{- .Values.insightsController.labels | toYaml | nindent 8 }}
+      annotations:
+        {{- .Values.insightsController.annotations | toYaml | nindent 8 }}
 {{- end }}
diff --git a/charts/cloudzero-agent/templates/deploy.yaml b/charts/cloudzero-agent/templates/deploy.yaml
index 831f491f..60064b8e 100644
--- a/charts/cloudzero-agent/templates/deploy.yaml
+++ b/charts/cloudzero-agent/templates/deploy.yaml
@@ -29,7 +29,7 @@ spec:
 {{- if .Values.server.priorityClassName }}
       priorityClassName: "{{ .Values.server.priorityClassName }}"
 {{- end }}
-      serviceAccountName: {{ template "cloudzero-agent.serviceAccountName.server" . }}
+      serviceAccountName: {{ template "cloudzero-agent.serviceAccountName" . }}
       initContainers:
         - name: {{ .Values.validator.name }}
           image: "{{ .Values.validator.image.repository }}:{{ .Values.validator.image.tag }}"
@@ -39,9 +39,9 @@ spec:
             - -c
             - cp -r /app /checks/bin/ && cloudzero-agent-validator d pre-start -f /checks/config/validator.yml
           volumeMounts:
-          {{- if or .Values.global.existingSecretName .Values.global.apiKey }}
+          {{- if or .Values.existingSecretName .Values.apiKey }}
             - name: cloudzero-api-key
-              mountPath: {{ .Values.server.containerSecretFilePath }}
+              mountPath: {{ .Values.serverConfig.containerSecretFilePath }}
               subPath: ""
               readOnly: true
           {{- end }}
@@ -132,9 +132,9 @@ spec:
               mountPath: /check/
             - name: validator-config-volume
               mountPath: /check/app/config/
-          {{- if or .Values.global.existingSecretName .Values.global.apiKey }}
+          {{- if or .Values.existingSecretName .Values.apiKey }}
             - name: cloudzero-api-key
-              mountPath: {{ .Values.server.containerSecretFilePath }}
+              mountPath: {{ .Values.serverConfig.containerSecretFilePath }}
               subPath: ""
               readOnly: true
           {{- end }}
@@ -174,10 +174,7 @@ spec:
             name: {{ template "cloudzero-agent.validatorConfigMapName" . }}
         - name: lifecycle-volume
           emptyDir: {}
-        {{- if or .Values.existingSecretName .Values.apiKey}}
-        {{- fail "The `existingSecretName` or `apiKey` setting should be set as `globals.existingSecretName` or `globals.apiKey`, not at the top level." }}
-        {{- end }}
-        {{- if or .Values.global.existingSecretName .Values.global.apiKey }}
+        {{- if or .Values.existingSecretName .Values.apiKey }}
         - name: cloudzero-api-key
           secret:
             secretName: {{ include "cloudzero-agent.secretName" . }}
diff --git a/charts/cloudzero-agent/templates/init-job.yaml b/charts/cloudzero-agent/templates/init-job.yaml
new file mode 100644
index 00000000..fe51882c
--- /dev/null
+++ b/charts/cloudzero-agent/templates/init-job.yaml
@@ -0,0 +1,35 @@
+{{- if and .Values.initJob.enabled .Values.insightsController.enabled }}
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: {{ include "cloudzero-agent.initJobName" . }}
+  namespace: {{ .Release.Namespace }}
+  labels:
+    {{- include "cloudzero-agent.insightsController.labels" . | nindent 4 }}
+spec:
+  template:
+    metadata:
+      name: {{ include "cloudzero-agent.initJobName" . }}
+      labels:
+        {{- include "cloudzero-agent.insightsController.initJob.matchLabels" . | nindent 8 }}
+    spec:
+      serviceAccountName: {{ include "cloudzero-agent.serviceAccountName" . }}
+      restartPolicy: Never
+      containers:
+        - name: start-scrape
+          image: {{ .Values.initJob.image.repository }}:{{ .Values.initJob.image.tag }}
+          command: ["sh", "-c"]
+          args:
+            - |
+              while true; do
+                echo "Waiting for the insightsController server to be ready...";
+                if curl -s -o /dev/null -w "%{http_code}" -k https://{{ include "cloudzero-agent.serviceName" . }}.{{ .Release.Namespace }}.svc.cluster.local:{{ .Values.insightsController.service.port }}/healthz | grep -q 200; then
+                  echo "Server is ready, starting scrape job...";
+                  curl -X POST -k https://{{ include "cloudzero-agent.serviceName" . }}.{{ .Release.Namespace }}.svc.cluster.local:{{ .Values.insightsController.service.port }}/scrape;
+                  echo "Scrape process started.";
+                  break;
+                fi;
+                echo "No 200 response from health endpoint. Retrying in 30 seconds...";
+                sleep 30;
+              done
+{{- end }}
diff --git a/charts/cloudzero-agent/templates/insights-deploy.yaml b/charts/cloudzero-agent/templates/insights-deploy.yaml
new file mode 100644
index 00000000..79236aca
--- /dev/null
+++ b/charts/cloudzero-agent/templates/insights-deploy.yaml
@@ -0,0 +1,103 @@
+{{- if .Values.insightsController.enabled }}
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+{{- if .Values.insightsController.server.deploymentAnnotations }}
+  annotations:
+    {{- toYaml .Values.insightsController.server.deploymentAnnotations | nindent 4 }}
+{{- end }}
+  name: {{ include "cloudzero-agent.insightsController.server.webhookFullname" . }}
+  namespace: {{ .Release.Namespace }}
+  labels:
+    {{- include "cloudzero-agent.insightsController.labels" . | nindent 4 }}
+spec:
+  replicas: {{ .Values.insightsController.server.replicaCount }}
+  selector:
+    matchLabels:
+      {{- include "cloudzero-agent.insightsController.server.matchLabels" . | nindent 6 }}
+  template:
+    metadata:
+      {{- with .Values.insightsController.server.podAnnotations }}
+      annotations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      labels:
+        {{- include "cloudzero-agent.insightsController.labels" . | nindent 8 }}
+        {{- with .Values.insightsController.server.podLabels }}
+        {{- toYaml . | nindent 8 }}
+        {{- end }}
+    spec:
+      serviceAccountName: {{ include "cloudzero-agent.serviceAccountName" . }}
+      {{- with .Values.insightsController.server.imagePullSecrets }}
+      imagePullSecrets:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      securityContext:
+        runAsUser: 65534
+        runAsNonRoot: true
+        runAsGroup: 65534
+        fsGroup: 65534
+      containers:
+        - name: webhook-server
+          image: "{{ .Values.insightsController.server.image.repository }}:{{ .Values.insightsController.server.image.tag | default .Chart.AppVersion }}"
+          imagePullPolicy: {{ .Values.insightsController.server.image.pullPolicy }}
+          command:
+            - /app/controller
+          args:
+            - -config
+            - "{{ include "cloudzero-agent.insightsController.configurationMountPath" . }}/server-config.yaml"
+          ports:
+            - containerPort: 8443
+          resources:
+            {{- toYaml .Values.insightsController.resources | nindent 12 }}
+          volumeMounts:
+            - name: insights-server-config
+              mountPath: {{ include "cloudzero-agent.insightsController.configurationMountPath" . }}
+          {{- if or .Values.insightsController.volumeMounts .Values.insightsController.server.tls.useManagedCertificate }}
+            {{- if .Values.insightsController.server.tls.useManagedCertificate }}
+            - name: tls-certs
+              mountPath: {{ .Values.insightsController.server.tls.mountPath }}
+              readOnly: true
+            {{- end }}
+          {{- if or .Values.existingSecretName .Values.apiKey }}
+            - name: cloudzero-api-key
+              mountPath: {{ .Values.serverConfig.containerSecretFilePath }}
+              subPath: ""
+              readOnly: true
+          {{- end }}
+            {{- with .Values.insightsController.volumeMounts }}
+              {{- toYaml . | nindent 12 }}
+            {{- end }}
+          {{- end }}
+      {{- if or .Values.insightsController.volumes .Values.insightsController.server.tls.useManagedCertificate }}
+      volumes:
+        - name: insights-server-config
+          configMap:
+            name: {{ include "cloudzero-agent.webhookConfigMapName" . }}
+        {{- if .Values.insightsController.server.tls.useManagedCertificate }}
+        - name: tls-certs
+          secret:
+            secretName: {{ include "cloudzero-agent.tlsSecretName" . }}
+        {{- end }}
+        {{- if or .Values.existingSecretName .Values.apiKey }}
+        - name: cloudzero-api-key
+          secret:
+            secretName: {{ include "cloudzero-agent.secretName" . }}
+        {{- end }}
+        {{- with .Values.insightsController.volumes }}
+          {{- toYaml . | nindent 8 }}
+        {{- end }}
+      {{- end }}
+      {{- with .Values.insightsController.server.nodeSelector }}
+      nodeSelector:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.insightsController.server.affinity }}
+      affinity:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.insightsController.server.tolerations }}
+      tolerations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+{{- end }}
\ No newline at end of file
diff --git a/charts/cloudzero-agent/templates/issuer.yaml b/charts/cloudzero-agent/templates/issuer.yaml
new file mode 100644
index 00000000..e80ed6e0
--- /dev/null
+++ b/charts/cloudzero-agent/templates/issuer.yaml
@@ -0,0 +1,10 @@
+{{ if and .Values.insightsController.webhooks.issuer.enabled .Values.insightsController.enabled }}
+---
+apiVersion: cert-manager.io/v1
+kind: Issuer
+metadata:
+  name: {{ include "cloudzero-agent.issuerName" . }}
+  namespace: {{ .Release.Namespace }}
+spec:
+  {{- toYaml .Values.insightsController.webhooks.issuer.spec | nindent 2 }}
+{{- end }}
diff --git a/charts/cloudzero-agent/templates/secret.yaml b/charts/cloudzero-agent/templates/secret.yaml
index b506b81f..21862f72 100644
--- a/charts/cloudzero-agent/templates/secret.yaml
+++ b/charts/cloudzero-agent/templates/secret.yaml
@@ -1,4 +1,4 @@
-{{- if .Values.global.apiKey }}
+{{- if .Values.apiKey }}
 apiVersion: v1
 kind: Secret
 metadata:
@@ -11,7 +11,7 @@ metadata:
   {{- toYaml . | nindent 4 }}
   {{- end }}
 data:
-  {{ .Values.server.containerSecretFileName }}: {{- $apiKey := .Values.global.apiKey | toString }}
+  {{ .Values.serverConfig.containerSecretFileName }}: {{- $apiKey := .Values.apiKey | toString }}
           {{- if not (regexMatch "^[a-zA-Z0-9-_.~!*'();]+$" $apiKey) }}
           {{- fail "The provided apiKey is invalid. Check that the provided value from apiKey matches exactly what is found in the CloudZero admin page." }}
           {{- end }}
diff --git a/charts/cloudzero-agent/templates/service.yaml b/charts/cloudzero-agent/templates/service.yaml
new file mode 100644
index 00000000..d8289f1d
--- /dev/null
+++ b/charts/cloudzero-agent/templates/service.yaml
@@ -0,0 +1,15 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ include "cloudzero-agent.serviceName" . }}
+  labels:
+    {{- include "cloudzero-agent.insightsController.labels" . | nindent 4 }}
+  namespace: {{ .Release.Namespace }}
+spec:
+  type: ClusterIP
+  ports:
+    - port: {{ .Values.insightsController.service.port }}
+      targetPort: 8443
+      name: http
+  selector:
+    {{- include "cloudzero-agent.insightsController.server.matchLabels" . | nindent 4 }}
diff --git a/charts/cloudzero-agent/templates/serviceaccount.yaml b/charts/cloudzero-agent/templates/serviceaccount.yaml
index 206919d9..7bff4768 100644
--- a/charts/cloudzero-agent/templates/serviceaccount.yaml
+++ b/charts/cloudzero-agent/templates/serviceaccount.yaml
@@ -1,17 +1,17 @@
-{{- if .Values.serviceAccounts.server.create }}
+{{- if .Values.serviceAccount.create }}
 apiVersion: v1
 kind: ServiceAccount
 metadata:
   labels:
     {{- include "cloudzero-agent.server.labels" . | nindent 4 }}
-  name: {{ template "cloudzero-agent.serviceAccountName.server" . }}
+  name: {{ template "cloudzero-agent.serviceAccountName" . }}
   namespace: {{ include "cloudzero-agent.namespace" . }}
   annotations:
-{{ toYaml .Values.serviceAccounts.server.annotations | indent 4 }}
+{{ toYaml .Values.serviceAccount.annotations | indent 4 }}
 {{- if kindIs "bool" .Values.server.automountServiceAccountToken }}
 automountServiceAccountToken: {{ .Values.server.automountServiceAccountToken }}
-{{- else if kindIs "bool" .Values.serviceAccounts.server.automountServiceAccountToken }}
-automountServiceAccountToken: {{ .Values.serviceAccounts.server.automountServiceAccountToken }}
+{{- else if kindIs "bool" .Values.serviceAccount.automountServiceAccountToken }}
+automountServiceAccountToken: {{ .Values.serviceAccount.automountServiceAccountToken }}
 {{- end }}
 {{- end }}
 
diff --git a/charts/cloudzero-agent/templates/webhooks.yaml b/charts/cloudzero-agent/templates/webhooks.yaml
new file mode 100644
index 00000000..747450b8
--- /dev/null
+++ b/charts/cloudzero-agent/templates/webhooks.yaml
@@ -0,0 +1,38 @@
+{{- if .Values.insightsController.enabled }}
+{{- range $configType, $configs := .Values.insightsController.webhooks.configurations }}
+{{- if or (index $.Values.insightsController.labels.resources $configType) (index $.Values.insightsController.annotations.resources $configType) }}
+---
+apiVersion: admissionregistration.k8s.io/v1
+kind: ValidatingWebhookConfiguration
+metadata:
+  name: {{ include "cloudzero-agent.validatingWebhookConfigName" $ }}-{{ $configType }}
+  namespace: {{ $.Release.Namespace }}
+  labels:
+    {{- include "cloudzero-agent.insightsController.labels" $ | nindent 4 }}
+  annotations:
+    {{- include "cloudzero-agent.webhooks.annotations" $ | nindent 4 }}
+webhooks:
+  - name: {{ include "cloudzero-agent.validatingWebhookName" $ }}
+    namespaceSelector: {{ toYaml $.Values.insightsController.webhooks.namespaceSelector }}
+    failurePolicy: Ignore
+    rules:
+      - operations: [ "CREATE", "UPDATE" ]
+        apiGroups: {{ $configs.apiGroups }}
+        apiVersions: [ "v1" ]
+        resources: [ {{ $configType }} ]
+        scope: "*"
+    clientConfig:
+      service:
+        namespace: {{ $.Release.Namespace }}
+        name: {{ include "cloudzero-agent.serviceName" $ }}
+        path: "{{ $configs.path }}"
+        port: {{ $.Values.insightsController.service.port }}
+      {{- if and $.Values.insightsController.webhooks.certificate.enabled $.Values.insightsController.webhooks.issuer.enabled }}
+      caBundle: {{ $.Values.insightsController.webhooks.caBundle }}
+      {{- end }}
+    admissionReviewVersions: ["v1"]
+    sideEffects: None
+    timeoutSeconds: 5
+{{- end }}
+{{- end }}
+{{- end }}
\ No newline at end of file
diff --git a/charts/cloudzero-agent/values.yaml b/charts/cloudzero-agent/values.yaml
index 4e06e542..a3f7d584 100644
--- a/charts/cloudzero-agent/values.yaml
+++ b/charts/cloudzero-agent/values.yaml
@@ -7,11 +7,10 @@ clusterName: null
 # -- Region the cluster is running in.
 region: null
 
-global:
-  # -- CloudZero API key. Required if useExistingSecret is false.
-  apiKey: null
-  # -- If set, the agent will use the API key in this Secret to authenticate with CloudZero.
-  existingSecretName: null
+# -- CloudZero API key. Required if useExistingSecret is false.
+apiKey: null
+# -- If set, the agent will use the API key in this Secret to authenticate with CloudZero.
+existingSecretName: null
 
 # -- The following lists of metrics are required for CloudZero to function.
 # -- Modifications made to these lists may cause issues with the processing of cluster data
@@ -48,16 +47,19 @@ prometheusConfig:
     # -- Any items added to this list will be added to the Prometheus scrape configuration.
     additionalScrapeJobs: []
 
-tags:
+# General server settings that apply to both the prometheus agent server and the webhook server
+serverConfig:
+  # -- The agent will use this file path on the container filesystem to get the CZ API key.
+  containerSecretFilePath: /etc/config/secrets/
+  # -- The agent will look for a file with this name to get the CZ API key.
+  containerSecretFileName: value
+
+initJob:
   enabled: true
-  labels:
-    enabled:
-    patterns:
-      # - '.*'
-  annotations:
-    enabled: false
-    patterns:
-      - '.*'
+  image:
+    repository: curlimages/curl
+    pullPolicy: Always
+    tag: "8.10.1"
 
 kubeStateMetrics:
   enabled: true
@@ -111,10 +113,6 @@ server:
   - --web.console.libraries=/etc/prometheus/console_libraries
   - --web.console.templates=/etc/prometheus/consoles
   - --enable-feature=agent
-  # -- The agent will use this file path on the container filesystem to get the CZ API key.
-  containerSecretFilePath: /etc/config/prometheus/secrets/
-  # -- The agent will look for a file with this name to get the CZ API key.
-  containerSecretFileName: value
   persistentVolume:
     existingClaim: ""
     enabled: false
@@ -128,11 +126,103 @@ server:
   emptyDir:
     sizeLimit: 8Gi
 
-serviceAccounts:
+insightsController:
+  enabled: true
+  labels:
+    enabled:
+    patterns:
+      # - '.*'
+    resources:
+      pods: true
+      namespaces: true
+      deployments: false
+      statefulsets: false
+      nodes: false
+      jobs: false
+      cronjobs: false
+      daemonsets: false
+  annotations:
+    enabled: false
+    patterns:
+      - '.*'
+    resources:
+      pods: true
+      namespaces: true
+      deployments: false
+      statefulsets: false
+      nodes: false
+      jobs: false
+      cronjobs: false
+      daemonsets: false
   server:
-    create: true
-    name: ""
+    name: webhook-server
+    replicaCount: 3
+    image:
+      repository: ghcr.io/cloudzero/cloudzero-insights-controller/cloudzero-insights-controller
+      tag: 0.0.2
+      pullPolicy: Always
+    tls:
+      enabled: true
+      useManagedCertificate: true
+      mountPath: /etc/certs
+    port: 8443
+    read_timeout: 10s
+    write_timeout: 10s
+    idle_timeout: 120s
+  volumeMounts: []
+  volumes: []
+  resources: {}
+  nodeSelector: {}
+  tolerations: []
+  affinity: {}
+  imagePullSecrets: []
+  podAnnotations: {}
+  podLabels: {}
+  service:
+    port: 443
+  webhooks:
     annotations: {}
+    namespaceSelector: {}  # This denotes no specific selection, applies to all namespaces
+    issuer:
+      enabled: true
+      spec:
+        selfSigned: {}
+    certificate:
+      enabled: true
+    caBundle: ''  # by default, this is empty, and the value is populated by cert-manager's ca-injector if cert-manager is used
+    configurations:
+      pods:
+        path: /validate/pod
+        apiGroups: ['""']
+      namespaces:
+        path: /validate/namespace
+        apiGroups: ['""']
+      deployments:
+        path: /validate/deployment
+        apiGroups: ["apps"]
+      statefulsets:
+        path: /validate/statefulset
+        apiGroups: ["apps"]
+      nodes:
+        path: /validate/node
+        apiGroups: ['""']
+      jobs:
+        path: /validate/job
+        apiGroups: ["batch"]
+      cronjobs:
+        path: /validate/cronjob
+        apiGroups: ["batch"]
+      daemonsets:
+        path: /validate/daemonset
+        apiGroups: ["apps"]
+
+serviceAccount:
+  create: true
+  name: ""
+  annotations: {}
+
+cert-manager:
+  enabled: true
 
 rbac:
   create: true
diff --git a/charts/cloudzero-insights-controller/templates/init-job.yaml b/charts/cloudzero-insights-controller/templates/init-job.yaml
index d9644fac..97b2b6ba 100644
--- a/charts/cloudzero-insights-controller/templates/init-job.yaml
+++ b/charts/cloudzero-insights-controller/templates/init-job.yaml
@@ -22,7 +22,7 @@ spec:
           args:
             - |
               while true; do
-                echo "Waiting for the tags server to be ready...";
+                echo "Waiting for the webhook server to be ready...";
                 if curl -s -o /dev/null -w "%{http_code}" -k https://{{ include "insights-controller.serviceName" . }}.{{ .Release.Namespace }}.svc.cluster.local:{{ .Values.server.service.port }}/healthz | grep -q 200; then
                   echo "Server is ready, starting scrape job...";
                   curl -X POST -k https://{{ include "insights-controller.serviceName" . }}.{{ .Release.Namespace }}.svc.cluster.local:{{ .Values.server.service.port }}/scrape;

From 8b6f730924339420da7674bea9ccccfd4f641f61 Mon Sep 17 00:00:00 2001
From: Daniel Mepham <daniel.mepham@cloudzero.com>
Date: Tue, 10 Dec 2024 19:03:50 -0500
Subject: [PATCH 19/26] [CP-23428] add helm chart for creating cert (#118)

* CP-23428: add certificate helm chart

* update with documentation comments

* Update charts/cloudzero-agent/README.md

Co-authored-by: Becki Lee <becki.lee@cloudzero.com>

* Update charts/cloudzero-agent/README.md

remove duplicate entry

Co-authored-by: Becki Lee <becki.lee@cloudzero.com>

* Update charts/cloudzero-agent/README.md

add period to end of sentence in readme

Co-authored-by: Becki Lee <becki.lee@cloudzero.com>

* PR suggestion for readme

* update config example

---------

Co-authored-by: Becki Lee <becki.lee@cloudzero.com>
---
 charts/cloudzero-agent/README.md              | 124 ++++++++++++++----
 .../configuration.example.yaml                |  10 +-
 charts/cloudzero-agent/templates/_helpers.tpl |   4 +-
 .../templates/certificate.yaml                |   2 +-
 charts/cloudzero-agent/templates/issuer.yaml  |   4 +-
 .../cloudzero-agent/templates/webhooks.yaml   |   2 +-
 charts/cloudzero-agent/values.yaml            |  15 ++-
 charts/cloudzero-certificate/.helmignore      |  23 ++++
 charts/cloudzero-certificate/Chart.yaml       |   5 +
 .../cloudzero-certificate/templates/NOTES.txt |  11 ++
 .../templates/_helpers.tpl                    |  71 ++++++++++
 .../templates/secret.yaml                     |   9 ++
 charts/cloudzero-certificate/values.yaml      |  13 ++
 13 files changed, 250 insertions(+), 43 deletions(-)
 create mode 100644 charts/cloudzero-certificate/.helmignore
 create mode 100644 charts/cloudzero-certificate/Chart.yaml
 create mode 100644 charts/cloudzero-certificate/templates/NOTES.txt
 create mode 100644 charts/cloudzero-certificate/templates/_helpers.tpl
 create mode 100644 charts/cloudzero-certificate/templates/secret.yaml
 create mode 100644 charts/cloudzero-certificate/values.yaml

diff --git a/charts/cloudzero-agent/README.md b/charts/cloudzero-agent/README.md
index 9d146404..63084c53 100644
--- a/charts/cloudzero-agent/README.md
+++ b/charts/cloudzero-agent/README.md
@@ -37,13 +37,22 @@ If installing with Helm directly, execute the following steps:
 helm repo update
 ```
 
-2. Ensure that required CRDs are installed for certificate management. If you have more specific requirements around managing TLS certificates, see the [Certificate Management](#certificate-management) section.
+2. Provision a TLS certificate; by default, this chart deploys a `ValidatingWebhookConfiguration` resource, which requires a certificate in order validate requests to the webhook server. See related Kubernetes documentation [here](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#configure-admission-webhooks-on-the-fly).
+
+There are several options for provisioning this certificate. The default is to use a third party tool, [cert-manager](https://cert-manager.io/). If you would prefer not to use cert-manager, see the [Certificate Management](#certificate-management) section for other options.
+
+To use `cert-manager` for certificate management, first install the `cert-manager` CRDs with the following:
 ```console
 helm install <RELEASE_NAME> cloudzero/cloudzero-agent \
     --set insightsController.webhook.issuer.enabled=false \
     --set insightsController.webhook.certificate.enabled=false \
     --set insightsController.cert-manager.installCRDs=true
 ```
+Alternatively, [install the cert-manager CRDs directly](https://cert-manager.io/docs/installation/helm/) with:
+
+```console
+kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.16.2/cert-manager.yaml
+```
 Alternatively, [install the cert-manager CRDs directly](https://cert-manager.io/docs/installation/helm/).
 
 3. Fill out all required fields in the `configuration.example.yaml` file in this directory. Rename the file as necessary. Below is an example of a completed configuration file:
@@ -54,7 +63,7 @@ cloudAccountId: YOUR_CLOUD_ACCOUNT_ID
 clusterName: YOUR_CLUSTER_NAME
 # -- Region the cluster is running in.
 region: YOUR_CLOUD_REGION
-# -- CloudZero API key. Required if useExistingSecret is false.
+# -- CloudZero API key. Required if existingSecretName is null.
 apiKey: YOUR_CLOUDZERO_API_KEY
 # -- If set, the agent will use the API key in this Secret to authenticate with CloudZero.
 existingSecretName: YOUR_EXISTING_API_KEY_K8S_SECRET
@@ -75,6 +84,9 @@ insightsController:
     enabled: false
     patterns:
       - '.*' # -- match all annotations. This is not recommended.
+cert-manager:
+  # -- Your cluster may already have cert-manager running, in which case this value can be set to false.
+  enabled: true
 ```
 
 4. Install the helm chart using the completed configuration file:
@@ -82,19 +94,6 @@ insightsController:
 helm install <RELEASE_NAME> cloudzero/cloudzero-agent -f configuration.example.yaml
 ```
 
-### Update Helm Chart
-Alternatively, if you are updating an existing installation, pull the latest chart information first:
-
-```console
-helm repo update
-```
-
-Next, upgrade the installation to the latest chart version:
-
-```console
-helm upgrade --install <RELEASE_NAME> cloudzero/cloudzero-agent -f configuration.example.yaml
-```
-
 ### Mandatory Values
 
 There are several mandatory values that must be specified for the chart to install properly. Below are the required settings along with strategies for providing custom values during installation:
@@ -104,7 +103,7 @@ There are several mandatory values that must be specified for the chart to insta
 | cloudAccountId    | string | `nil`                 | Account ID in AWS or Subscription ID in Azure or Project Number in GCP where the cluster is running. Must be a string due to Helm limitations.  |
 | clusterName       | string | `nil`                 | Name of the cluster. Must be RFC 1123 compliant.                                                                         |
 | host              | string | `"api.cloudzero.com"` | CloudZero host to send metrics to.                                                                                      |
-| apiKey            | string | `nil`                 | The CloudZero API key to use for exporting metrics. Only used if `global.existingSecretName` is not set.                       |
+| apiKey            | string | `nil`                 | The CloudZero API key to use for exporting metrics. Only used if `existingSecretName` is not set.                       |
 | existingSecretName| string | `nil`                 | Name of the secret that contains the CloudZero API key. Required if not providing the API key via `apiKey`.             |
 | region            | string | `nil`                 | Region where the cluster is running (e.g., `us-east-1`, `eastus`). For more information, see AWS or Azure documentation. |
 | insightsController.labels.enabled            | string | `nil`                 | If enabled, labels for pods, deployments, statefulsets, daemonsets, cronjobs, jobs, nodes, and namespaces |
@@ -125,7 +124,7 @@ helm install <RELEASE_NAME> cloudzero/cloudzero-agent \
 
 Ensure `values-override.yaml` contains only the values you wish to override from `values.yaml`.
 
-> Note it is possible to save values for different environments, or based on other criteria into seperate values files and multiple files using the `-f` helm parameters.
+> Note it is possible to save values for different environments, or based on other criteria into separate values files and multiple files using the `-f` helm parameters.
 
 ##### Using the `--set` Flag
 
@@ -154,25 +153,87 @@ This chart allows the exporting of labels and annotations from the following res
 - `Namespace`
 
 Additional Notes:
-- By default, only labels from pods and namespaces are exported. To enable more resources, see the `insightsController.labels.resources` and `insightsController.annotations.resources` section of the `values.yaml` file.
 - Labels and annotations exports are managed in the `insightsController` section of the `values.yaml` file.
-- To disambiguate labels/annotations between resources, a prefix representing the resource type is prepended to the label key in the Explorer page. For example, a `foo=bar` node label would be presented as `node:foo: bar`. The exception is pod labels which do not have resource prefixes for backward compatibility with previous versions.
+- By default, only labels from pods and namespaces are exported. To enable more resources, see the `insightsController.labels.resources` and `insightsController.annotations.resources` section of the `values.yaml` file.
+- To disambiguate labels/annotations between resources, a prefix representing the resource type is prepended to the label key in the [CloudZero Explorer](https://app.cloudzero.com/explorer). For example, a `foo=bar` node label would be presented as `node:foo: bar`. The exception is pod labels which do not have resource prefixes for backward compatibility with previous versions.
 - Annotations are not exported by default; see the `insightsController.annotations.enabled` setting to enable. To disambiguate annotations from labels, an `annotation` prefix is prepended to the annotation key; i.e., an `foo: bar` annotation on a namespace would be represented in the Explorer as `node:annotation:foo: bar`
 - For both labels and annotations, the `patterns` array applies across all resource types; i.e., setting `['^foo']` for `insightsController.labels.patterns` will match label keys that start with `foo` for all resource types set to `true` in `insightsController.labels.resources`.
 
 ### Certificate Management
 
-This chart contains a `ValidatingWebhookConfiguration` resource, which requires a certificate in order validate requests to the webhook server. See related Kubernetes documentation [here](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#configure-admission-webhooks-on-the-fly).
+The default behavior of the chart is to deploy [cert-manager](https://github.com/cert-manager/cert-manager/tree/master) to create and manage the certificate, but there are several alternate options if using `cert-manager` is not possible:
+
+#### Option 1: use the `cloudzero-certificate` chart
+
+The `cloudzero-certificate` chart, which is maintained in this repo, creates a certificate and stores it in a Secret. The `cloudzero-agent` resources can then access this Secret. To install this helm chart, first decide what the release name of the `cloudzero-agent` will be (shown as `EXAMPLE_CLOUDZERO_AGENT_RELEASE_NAME` here), as the release name will be used as the DNS name in the certificate. Then, do the following:
+
+1. Create a Secret using the `cloudzero-certificate` chart and get the CA bundle value:
+```console
+helm repo update
+
+# Install the chart, which creates a Secret with a TLS certificate
+helm upgrade --install --namespace <YOUR_NAMESPACE> <YOUR_RELEASE_NAME> cloudzero --set cloudzeroAgentReleaseName=<EXAMPLE_CLOUDZERO_AGENT_RELEASE_NAME>
+
+# Get the CA bundle value by running:
+CA_BUNDLE=$(kubectl get secret -n <YOUR_NAMESPACE> <YOUR_RELEASE_NAME>-cloudzero-certificate -o jsonpath='{.data.ca\.crt}')
+
+# Confirm that CA_BUNDLE is set:
+echo $CA_BUNDLE
+```
+
+2. Next, set the following in your `configuration.example.yaml` file in addition to the existing values:
+```yaml
+insightsController:
+  server:
+    tls:
+      nameOverride: <YOUR_RELEASE_NAME>-cloudzero-certificate # This should be the name of the secret created in the previous step
+  webhooks:
+    caBundle: $CA_BUNDLE # This should be the value of the CA_BUNDLE variable in the previous step
+  certificate:
+    enabled: false
+  issuer:
+    enabled: false
+cert-manager:
+  enabled: false
+```
+
+3. Finally, continue on with the rest of the installation in the [Installation](#installation) section. The only new requirement is that when installing the `cloudzero-agent` helm chart, you must use the same value for the release name as was set in `cloudzeroAgentReleaseName` in step #1. For example:
 
-The default behavior of the chart is to deploy [cert-manager](https://github.com/cert-manager/cert-manager/tree/master) to create and manage the certificate.
+```console
+helm install <EXAMPLE_CLOUDZERO_AGENT_RELEASE_NAME> cloudzero/cloudzero-agent -f configuration.example.yaml
+```
+
+#### Option 2: bring your own certificate
+
+The `cloudzero-agent` chart can also use any certificate provided by an external source. The common name of the certificate should be `<RELEASE_NAME>.<RELEASE_NAMESPACE>.cluster.local`. A Kubernetes Secret should be created with the keys:
+```
+ca.crt: <base64 encoded caBundle>
+tls.crt: <base64 encoded certificate>
+tls.key: <base64 encoded key>
+```
+Finally, set the following in your `configuration.example.yaml` file in addition to the existing values:
+```yaml
+insightsController:
+  server:
+    tls:
+      nameOverride: <YOUR_TLS_SECRET_NAME>-cloudzero-certificate # This should be the name of the secret created in the previous step
+  webhooks:
+    caBundle: $CA_BUNDLE # This should be the value of the `ca.crt` value created in the Secret
+  certificate:
+    enabled: false
+  issuer:
+    enabled: false
+cert-manager:
+  enabled: false
+```
 
 ### Secret Management
 
 The chart requires a CloudZero API key to send metric data. Admins can retrieve API keys [here](https://app.cloudzero.com/organization/api-keys).
 
-The API key can be supplied as an existing secret (default) or created by the chart. Ensure the Secret is in the same namespace as the chart and follows this format:
+The API key is typically stored in a Secret in the cluster. The `cloudzero-agent` chart will create a Secret if the API key is provided via the `apiKey` argument. Alternatively, the API key can be stored in a Secret external to the chart; this Secret name would then be set as the `existingSecretName` argument. If creating a Secret external to the chart, ensure the Secret is in the same namespace as the chart and follows this format:
 
-**values-override.yaml**
+**Example User-Created Secret Content**
 ```yaml
 data:
   value: <API_KEY>
@@ -183,12 +244,25 @@ Example of creating a secret:
 kubectl create secret -n example-namespace generic example-secret-name --from-literal=value=<example-api-key-value>
 ```
 
-The secret can then be used with `global.existingSecretName`.
+The secret can then be used with `existingSecretName`.
 
 ### Memory Sizing
 
 Please see the [sizing guide](./docs/sizing-guide.md) in the docs directory.
 
+### Update Helm Chart
+If you are updating an existing installation, pull the latest chart information:
+
+```console
+helm repo update
+```
+
+Next, upgrade the installation to the latest chart version:
+
+```console
+helm upgrade --install <RELEASE_NAME> cloudzero/cloudzero-agent -f configuration.example.yaml
+```
+
 #### Passing Values to Subcharts
 
 Values can be passed to subcharts like [kube-state-metrics](https://github.com/prometheus-community/helm-charts/blob/main/charts/kube-state-metrics/values.yaml) by adding entries in `values-override.yaml` as per their specifications.
@@ -210,6 +284,8 @@ kubeStateMetrics:
 |----------------------------------------------------|--------------------------|---------|
 | https://prometheus-community.github.io/helm-charts | kube-state-metrics       | 5.15.*  |
 
+Note that while `kube-state-metrics` is listed as a dependency, it is referred to as `cloudzero-state-metrics` within the helm chart. This is to enforce the idea that this KSM deployment is used exclusively by the `cloudzero-agent`.
+
 ## Enabling Release Notifications
 
 To receive a notification when a new version of the chart is [released](https://github.com/Cloudzero/cloudzero-charts/releases), you can [watch the repository](https://docs.github.com/en/account-and-profile/managing-subscriptions-and-notifications-on-github/setting-up-notifications/configuring-notifications#configuring-your-watch-settings-for-an-individual-repository):
diff --git a/charts/cloudzero-agent/configuration.example.yaml b/charts/cloudzero-agent/configuration.example.yaml
index bfbaccad..114cdf5f 100644
--- a/charts/cloudzero-agent/configuration.example.yaml
+++ b/charts/cloudzero-agent/configuration.example.yaml
@@ -4,12 +4,10 @@ cloudAccountId: null
 clusterName: null
 # -- Region the cluster is running in.
 region: null
-
-global:
-  # -- CloudZero API key. Required if useExistingSecret is false.
-  apiKey: null
-  # -- If set, the agent will use the API key in this Secret to authenticate with CloudZero.
-  existingSecretName: null
+# -- CloudZero API key. Required if existingSecretName is null.
+apiKey: null
+# -- If set, the agent will use the API key in this Secret to authenticate with CloudZero.
+existingSecretName: null
 
 # label and annotation configuration:
 insightsController:
diff --git a/charts/cloudzero-agent/templates/_helpers.tpl b/charts/cloudzero-agent/templates/_helpers.tpl
index 97858908..e75d7c64 100644
--- a/charts/cloudzero-agent/templates/_helpers.tpl
+++ b/charts/cloudzero-agent/templates/_helpers.tpl
@@ -227,7 +227,7 @@ Name for the validating webhook configuration resource
 Name for the certificate secret
 */}}
 {{- define "cloudzero-agent.tlsSecretName" -}}
-{{- printf "%s-tls" (include "cloudzero-agent.insightsController.server.webhookFullname" .) }}
+{{- default (printf "%s-tls" (include "cloudzero-agent.insightsController.server.webhookFullname" .)) .Values.insightsController.server.tls.nameOverride }}
 {{- end }}
 
 
@@ -263,7 +263,7 @@ Annotations for the webhooks
 {{- if .Values.insightsController.webhooks.annotations }}
 {{ toYaml .Values.insightsController.webhook.annotations }}
 {{- end }}
-{{- if and .Values.insightsController.webhooks.certificate.enabled .Values.insightsController.webhooks.issuer.enabled }}
+{{- if and .Values.insightsController.certificate.enabled .Values.insightsController.issuer.enabled }}
 cert-manager.io/inject-ca-from: {{ .Values.insightsController.webhooks.caInjection | default (printf "%s/%s" .Release.Namespace (include "cloudzero-agent.certificateName" .)) }}
 {{- end }}
 {{- end }}
diff --git a/charts/cloudzero-agent/templates/certificate.yaml b/charts/cloudzero-agent/templates/certificate.yaml
index 97b92643..6d0e6b4a 100644
--- a/charts/cloudzero-agent/templates/certificate.yaml
+++ b/charts/cloudzero-agent/templates/certificate.yaml
@@ -1,4 +1,4 @@
-{{ if and .Values.insightsController.webhooks.certificate.enabled .Values.insightsController.enabled }}
+{{ if and .Values.insightsController.certificate.enabled .Values.insightsController.enabled }}
 ---
 apiVersion: cert-manager.io/v1
 kind: Certificate
diff --git a/charts/cloudzero-agent/templates/issuer.yaml b/charts/cloudzero-agent/templates/issuer.yaml
index e80ed6e0..a66c1c26 100644
--- a/charts/cloudzero-agent/templates/issuer.yaml
+++ b/charts/cloudzero-agent/templates/issuer.yaml
@@ -1,4 +1,4 @@
-{{ if and .Values.insightsController.webhooks.issuer.enabled .Values.insightsController.enabled }}
+{{ if and .Values.insightsController.issuer.enabled .Values.insightsController.enabled }}
 ---
 apiVersion: cert-manager.io/v1
 kind: Issuer
@@ -6,5 +6,5 @@ metadata:
   name: {{ include "cloudzero-agent.issuerName" . }}
   namespace: {{ .Release.Namespace }}
 spec:
-  {{- toYaml .Values.insightsController.webhooks.issuer.spec | nindent 2 }}
+  {{- toYaml .Values.insightsController.issuer.spec | nindent 2 }}
 {{- end }}
diff --git a/charts/cloudzero-agent/templates/webhooks.yaml b/charts/cloudzero-agent/templates/webhooks.yaml
index 747450b8..eef7369c 100644
--- a/charts/cloudzero-agent/templates/webhooks.yaml
+++ b/charts/cloudzero-agent/templates/webhooks.yaml
@@ -27,7 +27,7 @@ webhooks:
         name: {{ include "cloudzero-agent.serviceName" $ }}
         path: "{{ $configs.path }}"
         port: {{ $.Values.insightsController.service.port }}
-      {{- if and $.Values.insightsController.webhooks.certificate.enabled $.Values.insightsController.webhooks.issuer.enabled }}
+      {{- if or (and $.Values.insightsController.certificate.enabled $.Values.insightsController.issuer.enabled) (gt (len $.Values.insightsController.webhooks.caBundle) 1 ) }}
       caBundle: {{ $.Values.insightsController.webhooks.caBundle }}
       {{- end }}
     admissionReviewVersions: ["v1"]
diff --git a/charts/cloudzero-agent/values.yaml b/charts/cloudzero-agent/values.yaml
index a3f7d584..ed9b49ba 100644
--- a/charts/cloudzero-agent/values.yaml
+++ b/charts/cloudzero-agent/values.yaml
@@ -7,7 +7,7 @@ clusterName: null
 # -- Region the cluster is running in.
 region: null
 
-# -- CloudZero API key. Required if useExistingSecret is false.
+# -- CloudZero API key. Required if existingSecretName is null.
 apiKey: null
 # -- If set, the agent will use the API key in this Secret to authenticate with CloudZero.
 existingSecretName: null
@@ -164,6 +164,7 @@ insightsController:
     tls:
       enabled: true
       useManagedCertificate: true
+      nameOverride: ""
       mountPath: /etc/certs
     port: 8443
     read_timeout: 10s
@@ -180,15 +181,15 @@ insightsController:
   podLabels: {}
   service:
     port: 443
+  issuer:
+    enabled: true
+    spec:
+      selfSigned: {}
+  certificate:
+    enabled: true
   webhooks:
     annotations: {}
     namespaceSelector: {}  # This denotes no specific selection, applies to all namespaces
-    issuer:
-      enabled: true
-      spec:
-        selfSigned: {}
-    certificate:
-      enabled: true
     caBundle: ''  # by default, this is empty, and the value is populated by cert-manager's ca-injector if cert-manager is used
     configurations:
       pods:
diff --git a/charts/cloudzero-certificate/.helmignore b/charts/cloudzero-certificate/.helmignore
new file mode 100644
index 00000000..0e8a0eb3
--- /dev/null
+++ b/charts/cloudzero-certificate/.helmignore
@@ -0,0 +1,23 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*.orig
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/
diff --git a/charts/cloudzero-certificate/Chart.yaml b/charts/cloudzero-certificate/Chart.yaml
new file mode 100644
index 00000000..3fdc0b18
--- /dev/null
+++ b/charts/cloudzero-certificate/Chart.yaml
@@ -0,0 +1,5 @@
+apiVersion: v2
+name: cloudzero-certificate
+description: Creates a TLS certificate to be used by the CloudZero Insights Controller
+type: application
+version: 0.1.0
diff --git a/charts/cloudzero-certificate/templates/NOTES.txt b/charts/cloudzero-certificate/templates/NOTES.txt
new file mode 100644
index 00000000..2496c3ee
--- /dev/null
+++ b/charts/cloudzero-certificate/templates/NOTES.txt
@@ -0,0 +1,11 @@
+Get the caBundle value by running:
+
+CA_BUNDLE=$(kubectl get secret -n {{ .Release.Namespace }} {{ include "cloudzero-certificate.secretName" . }} -o jsonpath='{.data.ca\.crt}')
+
+This value should be used in the cloudzero-agent helm chart as shown:
+
+```
+insightsController:
+  webhooks:
+    caBundle: $CA_BUNDLE
+```
diff --git a/charts/cloudzero-certificate/templates/_helpers.tpl b/charts/cloudzero-certificate/templates/_helpers.tpl
new file mode 100644
index 00000000..7600ddc0
--- /dev/null
+++ b/charts/cloudzero-certificate/templates/_helpers.tpl
@@ -0,0 +1,71 @@
+{{/*
+Expand the name of the chart.
+*/}}
+{{- define "cloudzero-certificate.name" -}}
+{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{/*
+Create a default fully qualified app name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+If release name contains chart name it will be used as a full name.
+*/}}
+{{- define "cloudzero-certificate.fullname" -}}
+{{- if .Values.fullnameOverride }}
+{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- $name := default .Chart.Name .Values.nameOverride }}
+{{- if contains $name .Release.Name }}
+{{- .Release.Name | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }}
+{{- end }}
+{{- end }}
+{{- end }}
+
+{{/*
+Create chart name and version as used by the chart label.
+*/}}
+{{- define "cloudzero-certificate.chart" -}}
+{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{/*
+Common labels
+*/}}
+{{- define "cloudzero-certificate.labels" -}}
+helm.sh/chart: {{ include "cloudzero-certificate.chart" . }}
+{{ include "cloudzero-certificate.selectorLabels" . }}
+{{- if .Chart.AppVersion }}
+app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
+{{- end }}
+app.kubernetes.io/managed-by: {{ .Release.Service }}
+{{- end }}
+
+{{/*
+Selector labels
+*/}}
+{{- define "cloudzero-certificate.selectorLabels" -}}
+app.kubernetes.io/name: {{ include "cloudzero-certificate.name" . }}
+app.kubernetes.io/instance: {{ .Release.Name }}
+{{- end }}
+
+{{/*
+Create the name of the secret to use
+*/}}
+{{- define "cloudzero-certificate.secretName" -}}
+{{- default (include "cloudzero-certificate.fullname" .) .Values.secret.name }}
+{{- end }}
+
+{{/*
+Generate certificate for the webhook server
+*/}}
+{{- define "cloudzero-certificate.genCerts" -}}
+{{- $releaseName := required "`cloudzeroAgentReleaseName` must be supplied. This value should be the name of the cloudzero-agent helm release that will be created" .Values.cloudzeroAgentReleaseName -}}
+{{- $dnsName :=  printf "%s-svc.%s.cluster.local" $releaseName $.Release.Namespace -}}
+{{- $ca := genCA "cloudzero-agent-ca" 365 -}}
+{{- $cert := genSignedCert $dnsName nil (list $dnsName) 9999999 $ca -}}
+ca.crt: {{ $cert.Cert | b64enc }}
+tls.crt: {{ $cert.Cert | b64enc }}
+tls.key: {{ $cert.Key | b64enc }}
+{{- end -}}
diff --git a/charts/cloudzero-certificate/templates/secret.yaml b/charts/cloudzero-certificate/templates/secret.yaml
new file mode 100644
index 00000000..461dfca4
--- /dev/null
+++ b/charts/cloudzero-certificate/templates/secret.yaml
@@ -0,0 +1,9 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  labels:
+    {{- include "cloudzero-certificate.labels" . | nindent 4 }}
+  name: {{ include "cloudzero-certificate.secretName" . }}
+  namespace: {{ .Release.Namespace }}
+data:
+{{- include "cloudzero-certificate.genCerts" . | nindent 2 }}
diff --git a/charts/cloudzero-certificate/values.yaml b/charts/cloudzero-certificate/values.yaml
new file mode 100644
index 00000000..ae181171
--- /dev/null
+++ b/charts/cloudzero-certificate/values.yaml
@@ -0,0 +1,13 @@
+cloudzeroAgentReleaseName: null
+
+nameOverride: ""
+fullnameOverride: ""
+
+secret:
+  name: ""
+
+serviceAccount:
+  name: ""
+
+role:
+  name: ""

From fc96f302b2f11c76c9f5601d4f68cf088bd75c96 Mon Sep 17 00:00:00 2001
From: JB <josephbarnett@users.noreply.github.com>
Date: Thu, 12 Dec 2024 13:00:10 -0300
Subject: [PATCH 20/26] CP 24028 add insights controller scape config (#120)

CP-24028: add scrape target for insights container
CP-22734: Bump insights image release version
Enhance README for helm repo management
Add release note for next beta version
Update release process for customer version numbers in betas
---
 .../build-and-publish-beta-chart.yml          | 28 +++++++++---------
 charts/cloudzero-agent/README.md              | 29 ++++++++++++++++---
 .../docs/releases/1.0.0-beta-4.md             | 13 +++++++++
 charts/cloudzero-agent/templates/cm.yaml      | 29 +++++++++++++++----
 charts/cloudzero-agent/values.yaml            |  2 +-
 5 files changed, 76 insertions(+), 25 deletions(-)
 create mode 100644 charts/cloudzero-agent/docs/releases/1.0.0-beta-4.md

diff --git a/.github/workflows/build-and-publish-beta-chart.yml b/.github/workflows/build-and-publish-beta-chart.yml
index f7833384..e100445b 100644
--- a/.github/workflows/build-and-publish-beta-chart.yml
+++ b/.github/workflows/build-and-publish-beta-chart.yml
@@ -5,9 +5,8 @@ on:
   workflow_dispatch:
     inputs:
       version:
-        description: 'Version to use for the release (beta suffix will be automatically appended (e.g. "0.0.1" + "-beta"))'
-        required: false
-        default: ''
+        description: 'Version to use for the release. Must contain "beta" (e.g. "0.0.1-beta").'
+        required: true
       branch:
         description: 'Branch to use for the release'
         required: true
@@ -54,18 +53,18 @@ jobs:
       - name: Package Cloudzero Agent Chart
         run: helm package charts/cloudzero-agent/ --destination .deploy
 
-      # Step 3: Determine Version
-      - name: Get Github Tag Version
-        id: version
-        uses: flatherskevin/semver-action@v1
-        with:
-          incrementLevel: patch
-          source: tags
-
-      - name: Determine Chart Version
+      # Step 3: Validate and Set Version
+      - name: Validate Input Version
         run: |
-          NEW_VERSION=${{ github.event.inputs.version || steps.version.outputs.nextVersion }}
-          echo "NEW_VERSION=$NEW_VERSION-beta" >> $GITHUB_ENV
+          if [[ -z "${{ github.event.inputs.version }}" ]]; then
+            echo "Version input is required."
+            exit 1
+          fi
+          if [[ "${{ github.event.inputs.version }}" != *"beta"* ]]; then
+            echo "Version must contain 'beta'. Provided version: ${{ github.event.inputs.version }}"
+            exit 1
+          fi
+          echo "NEW_VERSION=${{ github.event.inputs.version }}" >> $GITHUB_ENV
 
       - name: Update Chart Version
         run: |
@@ -128,7 +127,6 @@ jobs:
 
       - name: Save Index in GH Pages
         run: |
-          # copy the new chart and index.yaml
           git add beta
           git commit -m "Updating ${{ env.NEW_VERSION }} Index"
           git push origin gh-pages
diff --git a/charts/cloudzero-agent/README.md b/charts/cloudzero-agent/README.md
index 63084c53..8881458d 100644
--- a/charts/cloudzero-agent/README.md
+++ b/charts/cloudzero-agent/README.md
@@ -17,14 +17,36 @@ For the latest release, see [Releases](https://github.com/Cloudzero/cloudzero-ch
 
 ## Installation
 
-### Get Helm Repository Info
+### Adding Helm Repository Information
+
+To use the chart or a beta version, you must add the repository to Helm. Refer to the [`helm repo`](https://helm.sh/docs/helm/helm_repo/) documentation for command details.
+
+#### 1. Add the Helm Chart Repository
 
 ```console
 helm repo add cloudzero https://cloudzero.github.io/cloudzero-charts
-helm repo update
+```
+> Note: If you intend to use a beta version, refer to the [beta installation document](./BETA-INSTALLATION.md) for the appropriate channel.
+
+#### 2. Add Repository Dependencies
+
+**Cert Manager**
+```console
+helm repo add cert-manager https://charts.jetstack.io    
 ```
 
-_See [`helm repo`](https://helm.sh/docs/helm/helm_repo/) for command documentation._
+**Metrics Exporter**
+```console
+helm repo add prometheus-community https://prometheus-community.github.io/helm-charts
+```
+
+#### 3. Update the Helm Repositories
+
+Ensure that the most recent chart versions are available:
+
+```console
+helm repo update
+```
 
 ### Install Helm Chart
 
@@ -53,7 +75,6 @@ Alternatively, [install the cert-manager CRDs directly](https://cert-manager.io/
 ```console
 kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.16.2/cert-manager.yaml
 ```
-Alternatively, [install the cert-manager CRDs directly](https://cert-manager.io/docs/installation/helm/).
 
 3. Fill out all required fields in the `configuration.example.yaml` file in this directory. Rename the file as necessary. Below is an example of a completed configuration file:
 ```yaml
diff --git a/charts/cloudzero-agent/docs/releases/1.0.0-beta-4.md b/charts/cloudzero-agent/docs/releases/1.0.0-beta-4.md
new file mode 100644
index 00000000..ce5dd0d5
--- /dev/null
+++ b/charts/cloudzero-agent/docs/releases/1.0.0-beta-4.md
@@ -0,0 +1,13 @@
+## [1.0.0-beta-4](https://github.com/Cloudzero/cloudzero-agent/compare/v1.0.1-beta...v1.0.0-beta-4) (2024-12-12)
+
+The Insights controller now exposes a Prometheus Metrics endpoint, enabling CloudZero to monitor its operations.
+
+### Upgrade Steps
+* Upgrade with:
+```sh
+helm upgrade --install <RELEASE_NAME> cloudzero-beta/cloudzero-agent -n <NAMESPACE> --create-namespace -f configuration.example.yaml --version 1.0.0-beta-1
+```
+For more details, see the [beta installation instructions](https://github.com/Cloudzero/cloudzero-charts/blob/develop/charts/cloudzero-agent/BETA-INSTALLATION.md).
+
+### Improvements
+* Added insights controller scrape configuration for operational monitoring. More information is available in the [insights controller documentation](https://github.com/Cloudzero/cloudzero-insights-controller/blob/develop/docs/statistics.md).
diff --git a/charts/cloudzero-agent/templates/cm.yaml b/charts/cloudzero-agent/templates/cm.yaml
index 3b063962..35f93d00 100644
--- a/charts/cloudzero-agent/templates/cm.yaml
+++ b/charts/cloudzero-agent/templates/cm.yaml
@@ -118,11 +118,30 @@ data:
           kubeconfig_file: ""
           follow_redirects: true
           enable_http2: true
-    {{- end }}
-    {{- if .Values.prometheusConfig.scrapeJobs.additionalScrapeJobs -}}
-    {{ toYaml .Values.prometheusConfig.scrapeJobs.additionalScrapeJobs | toString | nindent 6 }}
-    {{- end}}
-    {{- end}}
+      {{- end }}
+      {{- if .Values.insightsController.enabled }}
+      - job_name: cloudzero-insights-controller-job
+        metrics_path: /metrics
+        scheme: https
+        enable_compression: true
+        tls_config:
+          insecure_skip_verify: true
+        follow_redirects: true
+        enable_http2: true
+        kubernetes_sd_configs:
+          - role: endpoints
+            kubeconfig_file: ""
+            follow_redirects: true
+            enable_http2: true
+        relabel_configs:
+          - source_labels: [__meta_kubernetes_endpoints_name]
+            action: keep
+            regex: {{ include "cloudzero-agent.insightsController.server.webhookFullname" . }}-svc
+      {{- end }}
+      {{- if .Values.prometheusConfig.scrapeJobs.additionalScrapeJobs -}}
+      {{ toYaml .Values.prometheusConfig.scrapeJobs.additionalScrapeJobs | toString | nindent 6 }}
+      {{- end}}
+  {{- end}}
     remote_write:
       - url: 'https://{{ include "cloudzero-agent.cleanString" .Values.host }}/v1/container-metrics?cluster_name={{ include "cloudzero-agent.cleanString" .Values.clusterName | urlquery }}&cloud_account_id={{ include "cloudzero-agent.cleanString" .Values.cloudAccountId | urlquery }}&region={{ include "cloudzero-agent.cleanString" .Values.region | urlquery }}'
         authorization:
diff --git a/charts/cloudzero-agent/values.yaml b/charts/cloudzero-agent/values.yaml
index ed9b49ba..ba3719fd 100644
--- a/charts/cloudzero-agent/values.yaml
+++ b/charts/cloudzero-agent/values.yaml
@@ -159,7 +159,7 @@ insightsController:
     replicaCount: 3
     image:
       repository: ghcr.io/cloudzero/cloudzero-insights-controller/cloudzero-insights-controller
-      tag: 0.0.2
+      tag: 0.0.3
       pullPolicy: Always
     tls:
       enabled: true

From 698c7bfb55111d9047facb7d483562d4a9339bd9 Mon Sep 17 00:00:00 2001
From: Automated CZ Release <ops@cloudzero.com>
Date: Thu, 12 Dec 2024 16:10:46 +0000
Subject: [PATCH 21/26] Update Chart.yaml to version 1.0.0-beta-4

---
 charts/cloudzero-agent/Chart.lock               | 4 ++--
 charts/cloudzero-agent/Chart.yaml               | 2 +-
 charts/cloudzero-insights-controller/Chart.yaml | 2 +-
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/charts/cloudzero-agent/Chart.lock b/charts/cloudzero-agent/Chart.lock
index e49b79bc..72f52640 100644
--- a/charts/cloudzero-agent/Chart.lock
+++ b/charts/cloudzero-agent/Chart.lock
@@ -5,5 +5,5 @@ dependencies:
 - name: cert-manager
   repository: https://charts.jetstack.io
   version: v1.15.3
-digest: sha256:6135d395ad03a38895bd8f1d71fb84856d76b89978b5b4250f6d154c6e33b128
-generated: "2024-12-06T14:53:41.361206-05:00"
+digest: sha256:ef28b222788ce38b63857e7427d8f1dbb7c24e6d01e196049ac5af6209d132f6
+generated: "2024-12-12T16:10:45.087982225Z"
diff --git a/charts/cloudzero-agent/Chart.yaml b/charts/cloudzero-agent/Chart.yaml
index daaaf820..77bafdd8 100644
--- a/charts/cloudzero-agent/Chart.yaml
+++ b/charts/cloudzero-agent/Chart.yaml
@@ -2,7 +2,7 @@ apiVersion: v2
 name: cloudzero-agent
 description: A chart for using Prometheus in agent mode to send cluster metrics to the CloudZero platform.
 type: application
-version: 0.0.0-dev
+version: 1.0.0-beta-4
 maintainers:
   - name: CloudZero
     email: support@cloudzero.com
diff --git a/charts/cloudzero-insights-controller/Chart.yaml b/charts/cloudzero-insights-controller/Chart.yaml
index 548dc81a..cab0e924 100644
--- a/charts/cloudzero-insights-controller/Chart.yaml
+++ b/charts/cloudzero-insights-controller/Chart.yaml
@@ -2,7 +2,7 @@ apiVersion: v2
 name: cloudzero-insights-controller
 description: Provides telemetry to the CloudZero platform to enabling complex cost allocation and analysis.
 type: application
-version: 0.0.0-dev
+version: 1.0.0-beta-4
 appVersion: "0.0.2"
 dependencies:
   - name: cert-manager

From 8c882014a629fad85ebcad3f96a6f2ee9377ceee Mon Sep 17 00:00:00 2001
From: JB <josephbarnett@users.noreply.github.com>
Date: Fri, 13 Dec 2024 11:36:49 -0300
Subject: [PATCH 22/26] CP 23892 add healthcheck (#121)

* CP-23892, CP-24009, CP-23959: release note
* add healthcheck support
* bump value of insights controller
---
 .../docs/releases/1.0.0-beta-5.md             | 18 ++++++++++++++
 .../templates/insights-deploy.yaml            | 24 ++++++++++++++++++-
 charts/cloudzero-agent/values.yaml            | 11 ++++++++-
 3 files changed, 51 insertions(+), 2 deletions(-)
 create mode 100644 charts/cloudzero-agent/docs/releases/1.0.0-beta-5.md

diff --git a/charts/cloudzero-agent/docs/releases/1.0.0-beta-5.md b/charts/cloudzero-agent/docs/releases/1.0.0-beta-5.md
new file mode 100644
index 00000000..2b8ca6d5
--- /dev/null
+++ b/charts/cloudzero-agent/docs/releases/1.0.0-beta-5.md
@@ -0,0 +1,18 @@
+## [1.0.0-beta-5](https://github.com/Cloudzero/cloudzero-agent/compare/v1.0.1-beta...v1.0.0-beta-5) (2024-12-12)
+
+The following new features are available in this version of the Beta Chart release:
+
+* **Automatic detection and reconfiguration of secrets rotation.**
+* **Automatic detection and reconfiguration of TLS Certificate rotation.**
+
+### Upgrade Steps
+* Upgrade with:
+```sh
+helm upgrade --install <RELEASE_NAME> cloudzero-beta/cloudzero-agent -n <NAMESPACE> --create-namespace -f configuration.example.yaml --version 1.0.0-beta-5
+```
+For more details, see the [beta installation instructions](https://github.com/Cloudzero/cloudzero-charts/blob/develop/charts/cloudzero-agent/BETA-INSTALLATION.md).
+
+### Improvements
+
+* **Availability Enhancement**: Healthcheck support ensures that requests are only forwarded to replica instances that are ready to accept work.
+* **Security Policy Enhancements**: The application can now react to changes in the Cloudzero API Secret or TLS Certificates. In production environments, these secret values will rotate and update periodically. Instead of restarting the service, which can be costly, the application can now react to key changes and reinitialize the related layer.
diff --git a/charts/cloudzero-agent/templates/insights-deploy.yaml b/charts/cloudzero-agent/templates/insights-deploy.yaml
index 79236aca..bd6b5d11 100644
--- a/charts/cloudzero-agent/templates/insights-deploy.yaml
+++ b/charts/cloudzero-agent/templates/insights-deploy.yaml
@@ -69,6 +69,28 @@ spec:
               {{- toYaml . | nindent 12 }}
             {{- end }}
           {{- end }}
+          {{- if and .Values.insightsController.server.healthCheck.enabled }}
+          livenessProbe:
+            httpGet:
+              scheme: HTTPS
+              path: {{ .Values.insightsController.server.healthCheck.path }}
+              port: {{ .Values.insightsController.server.healthCheck.port }}
+            initialDelaySeconds: {{ .Values.insightsController.server.healthCheck.initialDelaySeconds }}
+            periodSeconds: {{ .Values.insightsController.server.healthCheck.periodSeconds }}
+            timeoutSeconds: {{ .Values.insightsController.server.healthCheck.timeoutSeconds }}
+            successThreshold: {{ .Values.insightsController.server.healthCheck.successThreshold }}
+            failureThreshold: {{ .Values.insightsController.server.healthCheck.failureThreshold }}
+          readinessProbe:
+            httpGet:
+              scheme: HTTPS
+              path: {{ .Values.insightsController.server.healthCheck.path }}
+              port: {{ .Values.insightsController.server.healthCheck.port }}
+            initialDelaySeconds: {{ .Values.insightsController.server.healthCheck.initialDelaySeconds }}
+            periodSeconds: {{ .Values.insightsController.server.healthCheck.periodSeconds }}
+            timeoutSeconds: {{ .Values.insightsController.server.healthCheck.timeoutSeconds }}
+            successThreshold: {{ .Values.insightsController.server.healthCheck.successThreshold }}
+            failureThreshold: {{ .Values.insightsController.server.healthCheck.failureThreshold }}
+          {{- end }}
       {{- if or .Values.insightsController.volumes .Values.insightsController.server.tls.useManagedCertificate }}
       volumes:
         - name: insights-server-config
@@ -100,4 +122,4 @@ spec:
       tolerations:
         {{- toYaml . | nindent 8 }}
       {{- end }}
-{{- end }}
\ No newline at end of file
+{{- end }}
diff --git a/charts/cloudzero-agent/values.yaml b/charts/cloudzero-agent/values.yaml
index ba3719fd..5e69603a 100644
--- a/charts/cloudzero-agent/values.yaml
+++ b/charts/cloudzero-agent/values.yaml
@@ -159,7 +159,7 @@ insightsController:
     replicaCount: 3
     image:
       repository: ghcr.io/cloudzero/cloudzero-insights-controller/cloudzero-insights-controller
-      tag: 0.0.3
+      tag: 0.0.4
       pullPolicy: Always
     tls:
       enabled: true
@@ -170,6 +170,15 @@ insightsController:
     read_timeout: 10s
     write_timeout: 10s
     idle_timeout: 120s
+    healthCheck:
+      enabled: true
+      path: /healthz
+      port: 8443
+      initialDelaySeconds: 15
+      periodSeconds: 20
+      timeoutSeconds: 3
+      successThreshold: 1
+      failureThreshold: 5
   volumeMounts: []
   volumes: []
   resources: {}

From c782fcacba22b40ae371e8550e3513b2e4c6d660 Mon Sep 17 00:00:00 2001
From: Automated CZ Release <ops@cloudzero.com>
Date: Fri, 13 Dec 2024 14:40:59 +0000
Subject: [PATCH 23/26] Update Chart.yaml to version 1.0.0-beta-5

---
 charts/cloudzero-agent/Chart.yaml               | 2 +-
 charts/cloudzero-insights-controller/Chart.yaml | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/charts/cloudzero-agent/Chart.yaml b/charts/cloudzero-agent/Chart.yaml
index 77bafdd8..798cf4e0 100644
--- a/charts/cloudzero-agent/Chart.yaml
+++ b/charts/cloudzero-agent/Chart.yaml
@@ -2,7 +2,7 @@ apiVersion: v2
 name: cloudzero-agent
 description: A chart for using Prometheus in agent mode to send cluster metrics to the CloudZero platform.
 type: application
-version: 1.0.0-beta-4
+version: 1.0.0-beta-5
 maintainers:
   - name: CloudZero
     email: support@cloudzero.com
diff --git a/charts/cloudzero-insights-controller/Chart.yaml b/charts/cloudzero-insights-controller/Chart.yaml
index cab0e924..d021a331 100644
--- a/charts/cloudzero-insights-controller/Chart.yaml
+++ b/charts/cloudzero-insights-controller/Chart.yaml
@@ -2,7 +2,7 @@ apiVersion: v2
 name: cloudzero-insights-controller
 description: Provides telemetry to the CloudZero platform to enabling complex cost allocation and analysis.
 type: application
-version: 1.0.0-beta-4
+version: 1.0.0-beta-5
 appVersion: "0.0.2"
 dependencies:
   - name: cert-manager

From 7a95c28650ba1333f9e676f8b01d5db8a8bd233f Mon Sep 17 00:00:00 2001
From: josephbarnett <joe.barnett@cloudzero.com>
Date: Sun, 15 Dec 2024 21:01:35 -0300
Subject: [PATCH 24/26] fix beta deploy script

---
 .github/workflows/build-and-publish-beta-chart.yml   | 3 +--
 charts/cloudzero-agent/docs/releases/1.0.0-beta-4.md | 2 +-
 2 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/build-and-publish-beta-chart.yml b/.github/workflows/build-and-publish-beta-chart.yml
index e100445b..9aacd131 100644
--- a/.github/workflows/build-and-publish-beta-chart.yml
+++ b/.github/workflows/build-and-publish-beta-chart.yml
@@ -138,13 +138,12 @@ jobs:
           rm -fr .deploy charts/cloudzero-agent/charts
           git reset --hard 
           # now checkout docs from main
-          git checkout main -- charts/cloudzero-agent/docs charts/cloudzero-agent/README.md README.md
+          git checkout ${{ github.event.inputs.branch }} -- charts/cloudzero-agent/docs charts/cloudzero-agent/README.md README.md
           git add README.md charts/cloudzero-agent/docs charts/cloudzero-agent/README.md
           git commit -m "Update docs for ${{ env.NEW_VERSION }}"
           git push origin gh-pages
         continue-on-error: true
 
-
       # Step 5: Create GitHub Release
       - name: Create Release
         uses: softprops/action-gh-release@v2
diff --git a/charts/cloudzero-agent/docs/releases/1.0.0-beta-4.md b/charts/cloudzero-agent/docs/releases/1.0.0-beta-4.md
index ce5dd0d5..4b8de8c0 100644
--- a/charts/cloudzero-agent/docs/releases/1.0.0-beta-4.md
+++ b/charts/cloudzero-agent/docs/releases/1.0.0-beta-4.md
@@ -5,7 +5,7 @@ The Insights controller now exposes a Prometheus Metrics endpoint, enabling Clou
 ### Upgrade Steps
 * Upgrade with:
 ```sh
-helm upgrade --install <RELEASE_NAME> cloudzero-beta/cloudzero-agent -n <NAMESPACE> --create-namespace -f configuration.example.yaml --version 1.0.0-beta-1
+helm upgrade --install <RELEASE_NAME> cloudzero-beta/cloudzero-agent -n <NAMESPACE> --create-namespace -f configuration.example.yaml --version 1.0.0-beta-4
 ```
 For more details, see the [beta installation instructions](https://github.com/Cloudzero/cloudzero-charts/blob/develop/charts/cloudzero-agent/BETA-INSTALLATION.md).
 

From 266472aba570bddcc81f1455445aecdfaa833d19 Mon Sep 17 00:00:00 2001
From: Thomas Evans <thomas.evans@thomas.evans-JD972PFM97>
Date: Thu, 19 Dec 2024 11:59:56 -0600
Subject: [PATCH 25/26] First swing at minimizing README.

---
 charts/cloudzero-agent/README.md | 272 ++-----------------------------
 1 file changed, 15 insertions(+), 257 deletions(-)

diff --git a/charts/cloudzero-agent/README.md b/charts/cloudzero-agent/README.md
index 8881458d..59951610 100644
--- a/charts/cloudzero-agent/README.md
+++ b/charts/cloudzero-agent/README.md
@@ -17,255 +17,36 @@ For the latest release, see [Releases](https://github.com/Cloudzero/cloudzero-ch
 
 ## Installation
 
-### Adding Helm Repository Information
+### Easy Install (Most teams will use this!)
 
 To use the chart or a beta version, you must add the repository to Helm. Refer to the [`helm repo`](https://helm.sh/docs/helm/helm_repo/) documentation for command details.
 
-#### 1. Add the Helm Chart Repository
+#### 1. Install the Helm Chart
 
 ```console
 helm repo add cloudzero https://cloudzero.github.io/cloudzero-charts
+helm install cloudzero cloudzero/cloudzero-agent \
+    --set apiKey=<YOUR_CLOUDZERO_API_KEY>\
+    --set clusterName=<YOUR_CLUSTER_NAME>
 ```
-> Note: If you intend to use a beta version, refer to the [beta installation document](./BETA-INSTALLATION.md) for the appropriate channel.
 
-#### 2. Add Repository Dependencies
+### Advanced Install (Usually when you have specific security requirements.)
 
-**Cert Manager**
-```console
-helm repo add cert-manager https://charts.jetstack.io    
-```
-
-**Metrics Exporter**
-```console
-helm repo add prometheus-community https://prometheus-community.github.io/helm-charts
-```
-
-#### 3. Update the Helm Repositories
-
-Ensure that the most recent chart versions are available:
-
-```console
-helm repo update
-```
-
-### Install Helm Chart
-
-The chart can be installed directly with Helm or any other common Kubernetes deployment tools.
-
-If installing with Helm directly, execute the following steps:
-
-1. Ensure that the most recent chart version is available:
-```console
-helm repo update
-```
-
-2. Provision a TLS certificate; by default, this chart deploys a `ValidatingWebhookConfiguration` resource, which requires a certificate in order validate requests to the webhook server. See related Kubernetes documentation [here](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#configure-admission-webhooks-on-the-fly).
-
-There are several options for provisioning this certificate. The default is to use a third party tool, [cert-manager](https://cert-manager.io/). If you would prefer not to use cert-manager, see the [Certificate Management](#certificate-management) section for other options.
-
-To use `cert-manager` for certificate management, first install the `cert-manager` CRDs with the following:
-```console
-helm install <RELEASE_NAME> cloudzero/cloudzero-agent \
-    --set insightsController.webhook.issuer.enabled=false \
-    --set insightsController.webhook.certificate.enabled=false \
-    --set insightsController.cert-manager.installCRDs=true
-```
-Alternatively, [install the cert-manager CRDs directly](https://cert-manager.io/docs/installation/helm/) with:
+#### 1. Create and Configure a Values File
 
-```console
-kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.16.2/cert-manager.yaml
-```
-
-3. Fill out all required fields in the `configuration.example.yaml` file in this directory. Rename the file as necessary. Below is an example of a completed configuration file:
 ```yaml
-# -- Account ID of the account the cluster is running in. This must be a string - even if it is a number in your system.
-cloudAccountId: YOUR_CLOUD_ACCOUNT_ID
-# -- Name of the clusters.
-clusterName: YOUR_CLUSTER_NAME
-# -- Region the cluster is running in.
-region: YOUR_CLOUD_REGION
-# -- CloudZero API key. Required if existingSecretName is null.
-apiKey: YOUR_CLOUDZERO_API_KEY
-# -- If set, the agent will use the API key in this Secret to authenticate with CloudZero.
-existingSecretName: YOUR_EXISTING_API_KEY_K8S_SECRET
-
-# label and annotation configuration (managed in the 'insightsController' section). See the below 'Labels and Annotations' section for more details.
-insightsController:
-  # -- By default, a ValidatingAdmissionWebhook will be deployed that records all created labels and annotations
-  enabled: true
-  labels:
-    # -- This value MUST be set to either true or false. The installation will fail otherwise
-    enabled: true
-    # -- This value MUST be set to a list of regular expressions which will be used to gather labels from pods, deployments, statefulsets, daemonsets, cronjobs, jobs, nodes, and namespaces
-    patterns:
-      - '^foo' # -- Match all labels whose key starts with "foo"
-      - 'bar$' # -- Match all labels whose key ends with "bar"
-  annotations:
-    # -- By default, the gathering of annotations is not enabled. To enable, set this field to true
-    enabled: false
-    patterns:
-      - '.*' # -- match all annotations. This is not recommended.
-cert-manager:
-  # -- Your cluster may already have cert-manager running, in which case this value can be set to false.
-  enabled: true
-```
-
-4. Install the helm chart using the completed configuration file:
-```console
-helm install <RELEASE_NAME> cloudzero/cloudzero-agent -f configuration.example.yaml
-```
-
-### Mandatory Values
-
-There are several mandatory values that must be specified for the chart to install properly. Below are the required settings along with strategies for providing custom values during installation:
-
-| Key               | Type   | Default               | Description                                                                                                             |
-|-------------------|--------|-----------------------|-------------------------------------------------------------------------------------------------------------------------|
-| cloudAccountId    | string | `nil`                 | Account ID in AWS or Subscription ID in Azure or Project Number in GCP where the cluster is running. Must be a string due to Helm limitations.  |
-| clusterName       | string | `nil`                 | Name of the cluster. Must be RFC 1123 compliant.                                                                         |
-| host              | string | `"api.cloudzero.com"` | CloudZero host to send metrics to.                                                                                      |
-| apiKey            | string | `nil`                 | The CloudZero API key to use for exporting metrics. Only used if `existingSecretName` is not set.                       |
-| existingSecretName| string | `nil`                 | Name of the secret that contains the CloudZero API key. Required if not providing the API key via `apiKey`.             |
-| region            | string | `nil`                 | Region where the cluster is running (e.g., `us-east-1`, `eastus`). For more information, see AWS or Azure documentation. |
-| insightsController.labels.enabled            | string | `nil`                 | If enabled, labels for pods, deployments, statefulsets, daemonsets, cronjobs, jobs, nodes, and namespaces |
-| insightsController.labels.patterns            | string | `nil`                 | An array of regular expressions, which are used to match specific label keys |
+# -- values.yaml
 
-#### Overriding Default Values
+# -- clusterName is required to identify this cluster in the CloudZero dashboard.
+clusterName: <YOUR_CLUSTER_NAME>
 
-Default values are specified in the chart's `values.yaml` file. If you need to change any of these values, it is recommended to create a `values-override.yaml` based on the `configuration.example.yaml`  file for your customizations.
+# -- apiKey is the CloudZero apiKey generated in the CloudZero platform.
+apiKey: <YOUR_CLOUDZERO_API_KEY>
 
-##### Using the `--values` Flag
-
-You can use the `--values` (or short form `-f`) flag in your Helm commands to override values in the chart with a new file. Specify the name of the file after the `--values` flag:
-
-```console
-helm install <RELEASE_NAME> cloudzero/cloudzero-agent \
-    -f values-override.yaml
+# -- Other values here...
 ```
 
-Ensure `values-override.yaml` contains only the values you wish to override from `values.yaml`.
-
-> Note it is possible to save values for different environments, or based on other criteria into separate values files and multiple files using the `-f` helm parameters.
-
-##### Using the `--set` Flag
-
-You can use the `--set` flag in Helm commands to directly set or override specific values from `values.yaml`. Use dot notation to specify nested values:
-
-```console
-helm install <RELEASE_NAME> cloudzero/cloudzero-agent \
-    --set existingSecretName=<NAME_OF_SECRET> \
-    --set clusterName=<CLUSTER_NAME> \
-    --set-string cloudAccountId=<CLOUD_ACCOUNT_ID> \
-    --set region=<REGION> \
-    --set server.resources.limits.memory=2048Mi \
-    -f values-override.yaml
-```
-
-### Labels and Annotations
-
-This chart allows the exporting of labels and annotations from the following resources:
-- `Pod`
-- `Deployment`
-- `StatefulSet`
-- `Daemonset`
-- `Job`
-- `CronJob`
-- `Node`
-- `Namespace`
-
-Additional Notes:
-- Labels and annotations exports are managed in the `insightsController` section of the `values.yaml` file.
-- By default, only labels from pods and namespaces are exported. To enable more resources, see the `insightsController.labels.resources` and `insightsController.annotations.resources` section of the `values.yaml` file.
-- To disambiguate labels/annotations between resources, a prefix representing the resource type is prepended to the label key in the [CloudZero Explorer](https://app.cloudzero.com/explorer). For example, a `foo=bar` node label would be presented as `node:foo: bar`. The exception is pod labels which do not have resource prefixes for backward compatibility with previous versions.
-- Annotations are not exported by default; see the `insightsController.annotations.enabled` setting to enable. To disambiguate annotations from labels, an `annotation` prefix is prepended to the annotation key; i.e., an `foo: bar` annotation on a namespace would be represented in the Explorer as `node:annotation:foo: bar`
-- For both labels and annotations, the `patterns` array applies across all resource types; i.e., setting `['^foo']` for `insightsController.labels.patterns` will match label keys that start with `foo` for all resource types set to `true` in `insightsController.labels.resources`.
-
-### Certificate Management
-
-The default behavior of the chart is to deploy [cert-manager](https://github.com/cert-manager/cert-manager/tree/master) to create and manage the certificate, but there are several alternate options if using `cert-manager` is not possible:
-
-#### Option 1: use the `cloudzero-certificate` chart
-
-The `cloudzero-certificate` chart, which is maintained in this repo, creates a certificate and stores it in a Secret. The `cloudzero-agent` resources can then access this Secret. To install this helm chart, first decide what the release name of the `cloudzero-agent` will be (shown as `EXAMPLE_CLOUDZERO_AGENT_RELEASE_NAME` here), as the release name will be used as the DNS name in the certificate. Then, do the following:
-
-1. Create a Secret using the `cloudzero-certificate` chart and get the CA bundle value:
-```console
-helm repo update
-
-# Install the chart, which creates a Secret with a TLS certificate
-helm upgrade --install --namespace <YOUR_NAMESPACE> <YOUR_RELEASE_NAME> cloudzero --set cloudzeroAgentReleaseName=<EXAMPLE_CLOUDZERO_AGENT_RELEASE_NAME>
-
-# Get the CA bundle value by running:
-CA_BUNDLE=$(kubectl get secret -n <YOUR_NAMESPACE> <YOUR_RELEASE_NAME>-cloudzero-certificate -o jsonpath='{.data.ca\.crt}')
-
-# Confirm that CA_BUNDLE is set:
-echo $CA_BUNDLE
-```
-
-2. Next, set the following in your `configuration.example.yaml` file in addition to the existing values:
-```yaml
-insightsController:
-  server:
-    tls:
-      nameOverride: <YOUR_RELEASE_NAME>-cloudzero-certificate # This should be the name of the secret created in the previous step
-  webhooks:
-    caBundle: $CA_BUNDLE # This should be the value of the CA_BUNDLE variable in the previous step
-  certificate:
-    enabled: false
-  issuer:
-    enabled: false
-cert-manager:
-  enabled: false
-```
-
-3. Finally, continue on with the rest of the installation in the [Installation](#installation) section. The only new requirement is that when installing the `cloudzero-agent` helm chart, you must use the same value for the release name as was set in `cloudzeroAgentReleaseName` in step #1. For example:
-
-```console
-helm install <EXAMPLE_CLOUDZERO_AGENT_RELEASE_NAME> cloudzero/cloudzero-agent -f configuration.example.yaml
-```
-
-#### Option 2: bring your own certificate
-
-The `cloudzero-agent` chart can also use any certificate provided by an external source. The common name of the certificate should be `<RELEASE_NAME>.<RELEASE_NAMESPACE>.cluster.local`. A Kubernetes Secret should be created with the keys:
-```
-ca.crt: <base64 encoded caBundle>
-tls.crt: <base64 encoded certificate>
-tls.key: <base64 encoded key>
-```
-Finally, set the following in your `configuration.example.yaml` file in addition to the existing values:
-```yaml
-insightsController:
-  server:
-    tls:
-      nameOverride: <YOUR_TLS_SECRET_NAME>-cloudzero-certificate # This should be the name of the secret created in the previous step
-  webhooks:
-    caBundle: $CA_BUNDLE # This should be the value of the `ca.crt` value created in the Secret
-  certificate:
-    enabled: false
-  issuer:
-    enabled: false
-cert-manager:
-  enabled: false
-```
-
-### Secret Management
-
-The chart requires a CloudZero API key to send metric data. Admins can retrieve API keys [here](https://app.cloudzero.com/organization/api-keys).
-
-The API key is typically stored in a Secret in the cluster. The `cloudzero-agent` chart will create a Secret if the API key is provided via the `apiKey` argument. Alternatively, the API key can be stored in a Secret external to the chart; this Secret name would then be set as the `existingSecretName` argument. If creating a Secret external to the chart, ensure the Secret is in the same namespace as the chart and follows this format:
-
-**Example User-Created Secret Content**
-```yaml
-data:
-  value: <API_KEY>
-```
-
-Example of creating a secret:
-```console
-kubectl create secret -n example-namespace generic example-secret-name --from-literal=value=<example-api-key-value>
-```
-
-The secret can then be used with `existingSecretName`.
+Default values are specified in the chart's `values.yaml` file. Please reference this file for available override values.
 
 ### Memory Sizing
 
@@ -281,32 +62,9 @@ helm repo update
 Next, upgrade the installation to the latest chart version:
 
 ```console
-helm upgrade --install <RELEASE_NAME> cloudzero/cloudzero-agent -f configuration.example.yaml
+helm upgrade --install cloudzero cloudzero/cloudzero-agent
 ```
 
-#### Passing Values to Subcharts
-
-Values can be passed to subcharts like [kube-state-metrics](https://github.com/prometheus-community/helm-charts/blob/main/charts/kube-state-metrics/values.yaml) by adding entries in `values-override.yaml` as per their specifications.
-
-A common addition may be to pull the container images from custom image registries/repositories:
-
-`values-override.yaml`
-```yaml
-kubeStateMetrics:
-  enabled: true
-  image:
-    registry: my-custom-registry.io
-    repository: my-custom-kube-state-metrics/kube-state-metrics
-```
-
-## Dependencies
-
-| Repository                                         | Name                     | Version |
-|----------------------------------------------------|--------------------------|---------|
-| https://prometheus-community.github.io/helm-charts | kube-state-metrics       | 5.15.*  |
-
-Note that while `kube-state-metrics` is listed as a dependency, it is referred to as `cloudzero-state-metrics` within the helm chart. This is to enforce the idea that this KSM deployment is used exclusively by the `cloudzero-agent`.
-
 ## Enabling Release Notifications
 
 To receive a notification when a new version of the chart is [released](https://github.com/Cloudzero/cloudzero-charts/releases), you can [watch the repository](https://docs.github.com/en/account-and-profile/managing-subscriptions-and-notifications-on-github/setting-up-notifications/configuring-notifications#configuring-your-watch-settings-for-an-individual-repository):

From f255adaaafe35192d0a38820e8ed07325250466b Mon Sep 17 00:00:00 2001
From: Thomas Evans <thomas.evans@thomas.evans-JD972PFM97>
Date: Thu, 19 Dec 2024 12:33:35 -0600
Subject: [PATCH 26/26] Took a pop at the values file as well.

---
 charts/cloudzero-agent/values.yaml | 17 +++++++++++++----
 1 file changed, 13 insertions(+), 4 deletions(-)

diff --git a/charts/cloudzero-agent/values.yaml b/charts/cloudzero-agent/values.yaml
index 5e69603a..14076419 100644
--- a/charts/cloudzero-agent/values.yaml
+++ b/charts/cloudzero-agent/values.yaml
@@ -1,14 +1,23 @@
+# ############### #
+# Required Values #
+# ############### #
+
+# -- Name of the clusters.
+clusterName: null
+# -- CloudZero API key. Required if existingSecretName is null.
+apiKey: null
+
+
+# Default Values
 # -- CloudZero host to send metrics to.
 host: api.cloudzero.com
+
+
 # -- Account ID of the account the cluster is running in. This must be a string - even if it is a number in your system.
 cloudAccountId: null
-# -- Name of the clusters.
-clusterName: null
 # -- Region the cluster is running in.
 region: null
 
-# -- CloudZero API key. Required if existingSecretName is null.
-apiKey: null
 # -- If set, the agent will use the API key in this Secret to authenticate with CloudZero.
 existingSecretName: null