diff --git a/backend/src/lib/cluster/resources.ts b/backend/src/lib/cluster/resources.ts index 068ded6..c7bac1d 100644 --- a/backend/src/lib/cluster/resources.ts +++ b/backend/src/lib/cluster/resources.ts @@ -1,8 +1,9 @@ import type { KubernetesObjectApi, V1EnvVar, - V1Ingress, V1Namespace, + V1NetworkPolicy, + V1NetworkPolicyPeer, V1Secret, } from "@kubernetes/client-node"; import { randomBytes } from "node:crypto"; @@ -13,6 +14,8 @@ import type { Organization, WorkloadConfig, } from "../../db/models.ts"; +import { logger } from "../../index.ts"; +import { env } from "../env.ts"; import { getOctokit } from "../octokit.ts"; import { createIngressConfig } from "./resources/ingress.ts"; import { createServiceConfig } from "./resources/service.ts"; @@ -37,6 +40,28 @@ export const MAX_STS_NAME_LEN = 60; export const getRandomTag = (): string => randomBytes(4).toString("hex"); export const RANDOM_TAG_LEN = 8; + +let allowedIngressPeers: V1NetworkPolicyPeer[] | null; +const getAllowedIngressPeers = (): V1NetworkPolicyPeer[] | null => { + if (!env.CREATE_INGRESS_NETPOL || !env.ALLOW_INGRESS_FROM) { + return null; + } + + if (!allowedIngressPeers) { + const allowedLabels = JSON.parse(env.ALLOW_INGRESS_FROM) as { + [key: string]: string; + }[]; + allowedIngressPeers = allowedLabels.map((labels) => ({ + namespaceSelector: { + matchLabels: labels, + }, + podSelector: {}, + })); + } + + return allowedIngressPeers; +}; + export interface K8sObject { apiVersion: string; kind: string; @@ -127,6 +152,55 @@ const createSecretConfig = ( }; }; +const createIngressNetPol = ({ + name, + namespace, + groupLabels, +}: { + name: string; + namespace: string; + groupLabels: { [key: string]: string }; +}): V1NetworkPolicy & K8sObject => { + if (!env.CREATE_INGRESS_NETPOL) { + return null; + } + + if (!env.ALLOW_INGRESS_FROM) { + logger.warn( + "ALLOW_INGRESS_FROM is not set, skipping network policy creation", + ); + return null; + } + + return { + apiVersion: "networking.k8s.io/v1", + kind: "NetworkPolicy", + metadata: { + name, + namespace, + }, + spec: { + podSelector: { + matchLabels: groupLabels, + }, + policyTypes: ["Ingress"], + ingress: [ + { + _from: [ + ...getAllowedIngressPeers(), + { + namespaceSelector: { + matchLabels: groupLabels, // Allow ingress from pods in namespaces of this group + }, + podSelector: {}, + }, + ], + }, + ], + }, + } satisfies V1NetworkPolicy; +}; + const applyLabels = (config: K8sObject, labels: { [key: string]: string }) => { config.metadata.labels = { ...config.metadata.labels, ...labels }; if (config.spec?.template) { @@ -141,29 +215,37 @@ const applyLabels = (config: K8sObject, labels: { [key: string]: string }) => { } }; -export const createAppConfigsFromDeployment = async ( - org: Organization, - app: App, - appGroup: AppGroup, - deployment: Deployment, - conf: WorkloadConfig, -) => { +export const createAppConfigsFromDeployment = async ({ + org, + app, + appGroup, + deployment, + config, + migrating = false, +}: { + org: Organization; + app: App; + appGroup: AppGroup; + deployment: Deployment; + config: WorkloadConfig; + migrating?: boolean; +}) => { const namespace = createNamespaceConfig(app.namespace, app.projectId); const configs: K8sObject[] = []; const octokit = - conf.source === "GIT" ? await getOctokit(org.githubInstallationId) : null; + config.source === "GIT" ? await getOctokit(org.githubInstallationId) : null; const secretName = `${app.name}-secrets-${deployment.id}`; const envVars = await getEnvVars( - conf.getEnv(), + config.getEnv(), secretName, octokit, deployment, - conf, + config, app, ); - const secretData = getEnvRecord(conf.getEnv()); + const secretData = getEnvRecord(config.getEnv()); if (secretData !== null) { const secretConfig = createSecretConfig( secretData, @@ -177,20 +259,20 @@ export const createAppConfigsFromDeployment = async ( const params = { deploymentId: deployment.id, - collectLogs: conf.collectLogs, + collectLogs: config.collectLogs, name: app.name, namespace: app.namespace, serviceName: app.namespace, - image: conf.imageTag, + image: config.imageTag, env: envVars, logIngestSecret: app.logIngestSecret, - subdomain: conf.subdomain, - createIngress: conf.createIngress, - port: conf.port, - replicas: conf.replicas, - mounts: conf.mounts, - requests: conf.requests, - limits: conf.limits, + subdomain: config.subdomain, + createIngress: config.createIngress, + port: config.port, + replicas: config.replicas, + mounts: config.mounts, + requests: config.requests, + limits: config.limits, }; const svc = createServiceConfig(params); @@ -204,46 +286,113 @@ export const createAppConfigsFromDeployment = async ( } const appGroupLabel = `${appGroup.name.replaceAll(" ", "_")}-${appGroup.id}-${org.id}`; - const labels = { + const groupLabels = { "anvilops.rcac.purdue.edu/app-group-id": appGroup.id.toString(), + "app.kubernetes.io/part-of": appGroupLabel, + }; + const labels = { + ...groupLabels, "anvilops.rcac.purdue.edu/app-id": app.id.toString(), "anvilops.rcac.purdue.edu/deployment-id": deployment.id.toString(), "app.kubernetes.io/name": app.name, - "app.kubernetes.io/part-of": appGroupLabel, "app.kubernetes.io/managed-by": "anvilops", }; - applyLabels(namespace, labels); - for (let config of configs) { - applyLabels(config, labels); + + if (migrating) { + // When migrating off AnvilOps, remove the labels by setting their values to null + const deletedLabels = Object.keys(labels).reduce( + (deleted, key) => ({ ...deleted, [key]: null }), + {}, + ); + applyLabels(namespace, deletedLabels); + for (let config of configs) { + applyLabels(config, deletedLabels); + } + } else { + const netpol = createIngressNetPol({ + name: params.name, + namespace: params.namespace, + groupLabels, + }); + + if (netpol) { + configs.push(netpol); + } + + applyLabels(namespace, labels); + for (let config of configs) { + applyLabels(config, labels); + } } + const postCreate = async (api: KubernetesObjectApi) => { // Clean up secrets and ingresses from previous deployments of the app - const secrets = (await api - .list("v1", "Secret", app.namespace) - .then((data) => data.items) - .then((data) => - data.map((d) => ({ ...d, apiVersion: "v1", kind: "Secret" })), - )) as (V1Secret & K8sObject)[]; - const ingresses = (await api - .list("networking.k8s.io/v1", "Ingress", app.namespace) - .then((data) => data.items) - .then((data) => - data.map((d) => ({ - ...d, + const outdatedResources = []; + + if (migrating) { + if (env.CREATE_INGRESS_NETPOL) { + // When migrating, AnvilOps-specific labels are removed, so grouping network policies will not work. + // Delete all network policies that are managed by AnvilOps. + const netpols = await api + .list("networking.k8s.io/v1", "NetworkPolicy", app.namespace) + .then((data) => + data.items.map((item) => ({ + ...item, + apiVersion: "networking.k8s.io/v1", + kind: "NetworkPolicy", + })), + ); + + outdatedResources.push( + ...netpols.filter( + (netpol) => + netpol.metadata.labels?.["app.kubernetes.io/managed-by"] === + "anvilops", + ), + ); + } + } else { + const resourceTypes = [ + { + apiVersion: "v1", + kind: "Secret", + }, + { apiVersion: "networking.k8s.io/v1", kind: "Ingress", - })), - )) as (V1Ingress & K8sObject)[]; + }, + ]; + + const resourceLists = await Promise.all( + resourceTypes.map((type) => + api.list(type.apiVersion, type.kind, app.namespace).then((data) => + data.items.map((item) => ({ + ...item, + apiVersion: type.apiVersion, + kind: type.kind, + })), + ), + ), + ); + + outdatedResources.concat( + resourceLists + .flat() + .filter( + (resource) => + parseInt( + resource.metadata.labels?.[ + "anvilops.rcac.purdue.edu/deployment-id" + ], + ) !== deployment.id, + ), + ); + } await Promise.all( - [...secrets, ...ingresses] - .filter( - (secret) => - parseInt( - secret.metadata.labels["anvilops.rcac.purdue.edu/deployment-id"], - ) !== deployment.id, - ) - .map((secret) => api.delete(secret).catch((err) => console.error(err))), + outdatedResources.map((resource) => + api.delete(resource).catch((err) => console.error(err)), + ), ); }; return { namespace, configs, postCreate }; diff --git a/backend/src/lib/env.ts b/backend/src/lib/env.ts index aad1b66..eeabfc6 100644 --- a/backend/src/lib/env.ts +++ b/backend/src/lib/env.ts @@ -240,6 +240,14 @@ const variables = { * Annotations to add to end users' Ingress configurations */ INGRESS_ANNOTATIONS: { required: false }, + /** + * Whether to create network policies in tenant namespaces to restrict ingress. Ingress will be restricted to namespaces with allowed labels and apps in the same group. + */ + CREATE_INGRESS_NETPOL: { required: false }, + /** + * Labels identifying namespaces to allow ingress to tenant pods. AnvilOps will not create network policies unless CREATE_INGRESS_NETPOL is true and ALLOW_INGRESS_FROM is set. + */ + ALLOW_INGRESS_FROM: { required: false }, /** * The storageClassName to use when provisioning tenant apps. If you omit this value, storage-related options will be hidden. */ diff --git a/backend/src/service/deleteApp.ts b/backend/src/service/deleteApp.ts index 9cbc9aa..6f62088 100644 --- a/backend/src/service/deleteApp.ts +++ b/backend/src/service/deleteApp.ts @@ -46,8 +46,8 @@ export async function deleteApp( message: "Failed to delete namespace", }); } - } else if (config.appType === "workload" && config.collectLogs) { - // If the log shipper was enabled, redeploy without it + } else if (config.appType === "workload") { + // Redeploy without the log shipper and without anvilops-related labels config.collectLogs = false; // <-- Disable log shipping const app = await db.app.getById(lastDeployment.appId); @@ -57,13 +57,14 @@ export async function deleteApp( ]); const { namespace, configs, postCreate } = - await createAppConfigsFromDeployment( + await createAppConfigsFromDeployment({ org, app, appGroup, - lastDeployment, + deployment: lastDeployment, config, - ); + migrating: true, // Deploy without any anvilops-related labels + }); const { KubernetesObjectApi: api } = await getClientsForRequest( userId, diff --git a/backend/src/service/helper/deployment.ts b/backend/src/service/helper/deployment.ts index ddc5b56..3d7f322 100644 --- a/backend/src/service/helper/deployment.ts +++ b/backend/src/service/helper/deployment.ts @@ -387,13 +387,13 @@ export class DeploymentService { // If we're creating a deployment directly from an existing image tag, just deploy it now try { const { namespace, configs, postCreate } = - await createAppConfigsFromDeployment( + await createAppConfigsFromDeployment({ org, app, appGroup, deployment, config, - ); + }); const api = getClientForClusterUsername( app.clusterUsername, "KubernetesObjectApi", diff --git a/backend/src/service/updateDeployment.ts b/backend/src/service/updateDeployment.ts index a9458f4..cd72384 100644 --- a/backend/src/service/updateDeployment.ts +++ b/backend/src/service/updateDeployment.ts @@ -100,13 +100,13 @@ export async function updateDeployment(secret: string, newStatus: string) { if (newStatus === "DEPLOYING") { const { namespace, configs, postCreate } = - await createAppConfigsFromDeployment( + await createAppConfigsFromDeployment({ org, app, appGroup, deployment, config, - ); + }); try { const api = getClientForClusterUsername( diff --git a/charts/anvilops/README.md b/charts/anvilops/README.md index ffaac70..291e1fd 100644 --- a/charts/anvilops/README.md +++ b/charts/anvilops/README.md @@ -50,6 +50,9 @@ If an option has a ⭐ beside it, you will likely have to change it to fit your | | **App ingress configuration** | _These values influence the generated `Ingress` configuration created for every app._ | | | ⭐ | `anvilops.apps.ingress.className` | Ingress class name | nginx | | | `anvilops.apps.ingress.annotations` | Annotations added to end users' `Ingress` resources. | | +| | **App network policy configuration** | _These values control NetworkPolicy resources created for tenant apps to restrict ingress traffic._ | | +| ⭐ | `anvilops.apps.netpol.createIngressNetworkPolicy` | If `true`, creates NetworkPolicy resources in tenant namespaces that restrict ingress traffic. Requires `allowedIngressMatchLabels` to be configured. | false | +| ⭐ | `anvilops.apps.netpol.allowedIngressMatchLabels` | A list of label selectors that specify which namespaces are allowed to send ingress traffic to tenant apps. Apps in the same app group are always allowed. See notes below for important considerations. | | | | **Rancher configuration** | _AnvilOps can integrate with Rancher to deploy applications inside Projects._ | | | ⭐ | `rancher.enabled` | Enable Rancher integrations. | false | | ⭐ | `rancher.apiBase` | Base URL of the Rancher v3 API. | | @@ -116,6 +119,35 @@ For example, if your `appDomain` was `https://example.com` (users' apps would be - Domain: `*.example.com` - IP: the public IP address of your ingress controller +## Network Policy Configuration + +When `anvilops.apps.netpol.createIngressNetworkPolicy` is `true` and `anvilops.apps.netpol.allowedIngressMatchLabels` is configured, AnvilOps creates NetworkPolicy resources in tenant namespaces to restrict ingress. These policies allowlist ingress from: + +1. Namespaces matching any of the label selectors specified in `allowedIngressMatchLabels` +2. Apps within the same app group + +All other ingress traffic is blocked by default. + +### Configuring `allowedIngressMatchLabels` + +The `allowedIngressMatchLabels` field accepts a list of label selectors identifying namespaces to allow traffic from. For example, this configuration allowlists traffic from `kube-system` and `ingress-nginx`. + +```yaml +allowedIngressMatchLabels: + - kubernetes.io/metadata.name: kube-system + - kubernetes.io/metadata.name: ingress-nginx +``` + +- **The network policy will block ingress from all namespaces that do not have an allowlisted label and are not part of the app group.** This could include many system services. + +- If `createIngressNetworkPolicy` is set to `true`, make sure to include all namespaces that need to communicate with tenant apps in `allowedIngressMatchLabels`. + +### Rancher Project Isolation + +When project isolation is enabled, it may be sufficient to set `allowedIngressMatchLabels` to an empty list(`[]`). This is because Rancher will create network policies in each namespace to allow traffic from the System project(as well as the current project). + +Setting `createIngressNetworkPolicy: true` will still ensure that apps that are in the same group can communicate across projects. + ## Required Secrets Every secret in this section must exist for AnvilOps to run. diff --git a/charts/anvilops/templates/anvilops/anvilops-deployment.yaml b/charts/anvilops/templates/anvilops/anvilops-deployment.yaml index c5f9b9f..25badbc 100644 --- a/charts/anvilops/templates/anvilops/anvilops-deployment.yaml +++ b/charts/anvilops/templates/anvilops/anvilops-deployment.yaml @@ -76,7 +76,7 @@ spec: value: /etc/ssl/certs/anvilops-tls.crt {{- end }} - name: ANVILOPS_VERSION - value: {{ .Chart.Version }} + value: {{ .Chart.Version | quote }} - name: CLUSTER_CONFIG_PATH value: /opt/config/cluster.json - name: POSTGRES_USER @@ -118,7 +118,7 @@ spec: name: session-secret key: secret - name: BASE_URL - value: {{ .Values.anvilops.env.baseURL }} + value: {{ .Values.anvilops.env.baseURL | quote }} - name: GITHUB_WEBHOOK_SECRET valueFrom: secretKeyRef: @@ -162,7 +162,7 @@ spec: {{- if .Values.rancher.enabled }} {{- with .Values.rancher.apiBase }} - name: RANCHER_API_BASE - value: {{ . }} + value: {{ . | quote }} {{- end }} - name: RANCHER_TOKEN valueFrom: @@ -218,6 +218,16 @@ spec: - name: INGRESS_ANNOTATIONS value: {{ . | toJson | quote }} {{- end }} + {{- if .Values.anvilops.apps.netpol.createIngressNetworkPolicy }} + - name: CREATE_INGRESS_NETPOL + value: "true" + {{- with .Values.anvilops.apps.netpol.allowedIngressMatchLabels }} + - name: ALLOW_INGRESS_FROM + value: {{ toJson . | quote }} + {{- else -}} + {{- fail "anvilops.apps.netpol.createIngressNetworkPolicy is enabled, but anvilops.apps.netpol.allowedIngressMatchLabels does not have a value." -}} + {{- end }} + {{- end }} - name: STORAGE_CLASS_NAME value: {{ .Values.tenants.storageClassName }} - name: STORAGE_ACCESS_MODES @@ -229,7 +239,7 @@ spec: - name: CHART_PROJECT_NAME value: {{ .Values.anvilops.env.harborChartRepoName }} - name: ALLOW_HELM_DEPLOYMENTS - value: "{{ .Values.anvilops.env.allowHelmDeployments }}" + value: {{ .Values.anvilops.env.allowHelmDeployments | quote }} - name: BUILDKITD_ADDRESS value: {{ .Values.buildkitd.address }} - name: FILE_BROWSER_IMAGE diff --git a/charts/anvilops/templates/anvilops/anvilops-netpol.yaml b/charts/anvilops/templates/anvilops/anvilops-netpol.yaml index d5c91cc..f4a8a6b 100644 --- a/charts/anvilops/templates/anvilops/anvilops-netpol.yaml +++ b/charts/anvilops/templates/anvilops/anvilops-netpol.yaml @@ -7,7 +7,7 @@ metadata: spec: podSelector: matchLabels: - app: anvilops + app.kubernetes.io/name: anvilops ingress: - {} policyTypes: diff --git a/charts/anvilops/templates/jobs/rotate-rancher-credentials.yaml b/charts/anvilops/templates/jobs/rotate-rancher-credentials.yaml index 3269f09..cf8501c 100644 --- a/charts/anvilops/templates/jobs/rotate-rancher-credentials.yaml +++ b/charts/anvilops/templates/jobs/rotate-rancher-credentials.yaml @@ -29,7 +29,7 @@ spec: - name: RANCHER_SECRET_NAME value: rancher-config - name: RANCHER_TOKEN_TTL - value: {{ .Values.rancher.tokenTtl }} + value: {{ .Values.rancher.tokenTtl | quote }} {{- if .Values.anvilops.serviceAccount.useKubeconfig }} - name: KUBECONFIG value: /opt/creds/kubeconfig diff --git a/charts/anvilops/values.yaml b/charts/anvilops/values.yaml index 462d2d4..296fe97 100644 --- a/charts/anvilops/values.yaml +++ b/charts/anvilops/values.yaml @@ -88,12 +88,19 @@ anvilops: annotations: # kubernetes.io/ingress.class: nginx # kubernetes.io/tls-acme: "true" + netpol: + createIngressNetworkPolicy: false + # allowedIngressMatchLabels: + # - kubernetes.io/metadata.name: kube-system + # - kubernetes.io/metadata.name: ingress-nginx + rancher: enabled: false apiBase: "" refreshTokens: true refreshSchedule: "0 0 25 * *" - tokenTtl: 2592000000 # 30 days + tokenTtl: "2592000000" # 30 days + postgres: generateCredentials: true image: postgres:17 diff --git a/tilt/local-values.yaml b/tilt/local-values.yaml index 6346f45..ed17b75 100644 --- a/tilt/local-values.yaml +++ b/tilt/local-values.yaml @@ -50,6 +50,12 @@ anvilops: ingress: className: nginx annotations: + netpol: + createIngressNetworkPolicy: true + allowedIngressMatchLabels: + - kubernetes.io/metadata.name: kube-system + - kubernetes.io/metadata.name: ingress-nginx + postgres: generateCredentials: false image: postgres:17