Refresh kubeconfig and rancher API token every month

backend/README.md

@@ -88,7 +88,7 @@ docker run -p 5432:5432 --rm -it -v anvilops:/var/lib/postgresql/data -e POSTGRE
 
 For access control, AnvilOps can integrate with the Rancher API. If you are not using Rancher, leave the following values unset.
 
-In development, set the environment variable `RANCHER_API_BASE` to the Rancher v3 API base URL (e.g. https://composable.anvil.rcac.purdue.edu/v3). In production, set the `api-base` key of the `rancher-config` secret instead. Also provide a non-cluster scoped token (base64-encoded) for the AnvilOps service user's account, under the `RANCHER_TOKEN` environment variable or the `api-token` key of `rancher-config`.
+In development, set the environment variable `RANCHER_API_BASE` to the Rancher v3 API base URL (e.g. https://composable.anvil.rcac.purdue.edu/v3). Also provide a non-cluster scoped token (base64-encoded) for the AnvilOps service user's account, under the `RANCHER_TOKEN` environment variable or the `api-token` key of the secret `rancher-config`.
 
 If you would like to make a sandbox project available to users, set the environment variable `SANDBOX_ID` to its project ID. In production, set the the `sandbox-id` key of the `rancher-config` secret.
 
@@ -116,7 +116,11 @@ Set the environment variable `LOGIN_TYPE` to the name of the login method that A
 
 ### Kubernetes API
 
-A kubeconfig file is needed to manage resources through the Kubernetes API. Specify the file by setting `KUBECONFIG` environment variable to the location where the file is mounted within the pod. In development, if `KUBECONFIG` is not set, a kubeconfig file will be loaded from `$HOME/.kube`. In production, set the key `kubeconfig` in the secret `kube-auth` to the kubeconfig file.
+A kubeconfig file is needed to manage resources through the Kubernetes API. Specify the file by setting `KUBECONFIG` environment variable to its path. In development, if `KUBECONFIG` is not set, a kubeconfig file will be loaded from `$HOME/.kube`.
+
+In production, create a secret `kube-auth` and set the key `kubeconfig` in the secret `kube-auth` to the kubeconfig file.
+
+On Rancher-managed clusters, AnvilOps can automatically refresh the kubeconfig file. `kubeconfig` can be omitted from the secret, because AnvilOps will automatically fetch a kubeconfig during installation. In `kube-auth`, set the key `cluster-id` to the cluster ID associated with the kubeconfig. When viewing a cluster in Rancher, the URL will look something like `https://<RANCHER_SERVER>/dashboard/c/<cluster id>/explorer`.
 
 Ensure that the user associated with the kubeconfig has permission to read namespaces globally.
 
@@ -125,6 +129,7 @@ Ensure that the user associated with the kubeconfig has permission to read names
 **Note for Rancher-managed clusters**
 
 If your cluster uses a Rancher version < v2.10, the kubeconfig file must be configured to use an [Authorized Cluster Endpoint](https://ranchermanager.docs.rancher.com/reference-guides/rancher-manager-architecture/communicating-with-downstream-user-clusters#4-authorized-cluster-endpoint). This is to avoid a [bug](https://github.com/rancher/rancher/issues/41988) related to user impersonation. See the documentation for your Rancher version on configuring an Authorized Cluster Endpoint and using its context in your kubeconfig.
+In order to correctly refresh the kubeconfig, set the key `use-cluster-name` in the secret `kube-auth` to the name of the endpoint.
 
 ---
 

backend/src/jobs/getKubeconfig.ts

@@ -0,0 +1,72 @@
+import {
+  CoreV1Api,
+  KubeConfig,
+  PatchStrategy,
+  setHeaderOptions,
+} from "@kubernetes/client-node";
+import { exit } from "node:process";
+import yaml from "yaml";
+
+const KUBECONFIG_SECRET_NAME = process.env.KUBECONFIG_SECRET_NAME;
+const CLUSTER_ID = process.env.CLUSTER_ID;
+const USE_CLUSTER_NAME = process.env.USE_CLUSTER_NAME;
+
+const RANCHER_API_BASE = process.env.RANCHER_API_BASE;
+const RANCHER_TOKEN = process.env.RANCHER_TOKEN;
+const CURRENT_NAMESPACE = process.env.CURRENT_NAMESPACE;
+
+if (!RANCHER_API_BASE || !RANCHER_TOKEN) {
+  console.log(
+    "RANCHER_API_BASE or RANCHER_TOKEN not set, cannot get kubeconfig",
+  );
+  exit(1);
+}
+
+if (!CLUSTER_ID) {
+  console.log("CLUSTER_ID not set, cannot get kubeconfig");
+  exit(1);
+}
+
+const kcReq = await fetch(
+  `${RANCHER_API_BASE}/clusters/${CLUSTER_ID}?action=generateKubeconfig`,
+  {
+    method: "POST",
+    headers: {
+      Authorization: `Basic ${RANCHER_TOKEN}`,
+      Accept: "application/json",
+    },
+  },
+);
+
+if (!kcReq.ok) {
+  throw new Error("Failed to get kubeconfig: " + kcReq.statusText);
+}
+
+const kubeConfigRes = await kcReq.json();
+let kubeConfig = kubeConfigRes["config"];
+
+if (USE_CLUSTER_NAME) {
+  const body = yaml.parse(kubeConfig);
+  body["current-context"] = USE_CLUSTER_NAME;
+  kubeConfig = yaml.stringify(body);
+}
+
+const kc = new KubeConfig();
+kc.loadFromString(kubeConfig);
+
+const api = kc.makeApiClient(CoreV1Api);
+
+await api.patchNamespacedSecret(
+  {
+    name: KUBECONFIG_SECRET_NAME,
+    namespace: CURRENT_NAMESPACE,
+    body: {
+      data: {
+        kubeconfig: Buffer.from(kubeConfig, "utf-8").toString("base64"),
+      },
+    },
+  },
+  setHeaderOptions("Content-Type", PatchStrategy.MergePatch),
+);
+
+console.log("Kubeconfig patched successfully");

backend/src/jobs/rotateRancherCredentials.ts

@@ -0,0 +1,184 @@
+import {
+  AppsV1Api,
+  CoreV1Api,
+  KubeConfig,
+  PatchStrategy,
+  setHeaderOptions,
+  type V1Deployment,
+} from "@kubernetes/client-node";
+import { exit } from "node:process";
+import * as yaml from "yaml";
+
+const RANCHER_API_BASE = process.env.RANCHER_API_BASE;
+const RANCHER_TOKEN = process.env.RANCHER_TOKEN;
+const RANCHER_SECRET_NAME = process.env.RANCHER_SECRET_NAME;
+const RANCHER_TOKEN_TTL = parseInt(process.env.RANCHER_TOKEN_TTL, 10);
+
+if (!RANCHER_API_BASE || !RANCHER_TOKEN) {
+  console.log("RANCHER_API_BASE or RANCHER_TOKEN not set, skipping rotation");
+  exit(1);
+}
+
+const KUBECONFIG_SECRET_NAME = process.env.KUBECONFIG_SECRET_NAME;
+const CURRENT_NAMESPACE = process.env.CURRENT_NAMESPACE;
+const USE_CLUSTER_NAME = process.env.USE_CLUSTER_NAME;
+const CLUSTER_ID = process.env.CLUSTER_ID;
+
+const kc = new KubeConfig();
+kc.loadFromDefault();
+
+const api = kc.makeApiClient(CoreV1Api);
+
+const rancherTokenReq = await fetch(`${RANCHER_API_BASE}/tokens`, {
+  method: "POST",
+  headers: {
+    Authorization: `Basic ${RANCHER_TOKEN}`,
+    Accept: "application/json",
+    "Content-Type": "application/json",
+  },
+  body: JSON.stringify({
+    type: "token",
+    ttl: RANCHER_TOKEN_TTL,
+  }),
+});
+
+if (!rancherTokenReq.ok) {
+  throw new Error(
+    "Failed to generate rancher token: " + rancherTokenReq.statusText,
+  );
+}
+
+const tokenRes = await rancherTokenReq.json();
+const token = Buffer.from(tokenRes["token"], "utf-8").toString("base64");
+
+await api.patchNamespacedSecret(
+  {
+    name: RANCHER_SECRET_NAME,
+    namespace: CURRENT_NAMESPACE,
+    body: {
+      data: {
+        "api-token": Buffer.from(token, "utf-8").toString("base64"),
+      },
+    },
+  },
+  setHeaderOptions("Content-Type", PatchStrategy.MergePatch),
+);
+
+console.log("Rancher token patched successfully");
+
+if (KUBECONFIG_SECRET_NAME) {
+  if (!CLUSTER_ID) {
+    console.log("CLUSTER_ID not set, skipping kubeconfig rotation");
+  } else {
+    const kcReq = await fetch(
+      `${RANCHER_API_BASE}/clusters/${CLUSTER_ID}?action=generateKubeconfig`,
+      {
+        method: "POST",
+        headers: {
+          Authorization: `Basic ${RANCHER_TOKEN}`,
+          Accept: "application/json",
+        },
+      },
+    );
+
+    if (!kcReq.ok) {
+      throw new Error("Failed to regenerate kubeconfig: " + kcReq.statusText);
+    }
+
+    const kubeConfigRes = await kcReq.json();
+    let kubeConfig = kubeConfigRes["config"];
+
+    if (USE_CLUSTER_NAME) {
+      const body = yaml.parse(kubeConfig);
+      body["current-context"] = USE_CLUSTER_NAME;
+      kubeConfig = yaml.stringify(body);
+    }
+
+    await api.patchNamespacedSecret(
+      {
+        name: KUBECONFIG_SECRET_NAME,
+        namespace: process.env.CURRENT_NAMESPACE,
+        body: {
+          data: {
+            kubeconfig: Buffer.from(kubeConfig, "utf-8").toString("base64"),
+          },
+        },
+      },
+      setHeaderOptions("Content-Type", PatchStrategy.MergePatch),
+    );
+
+    console.log("Kubeconfig patched successfully");
+  }
+}
+
+const app = kc.makeApiClient(AppsV1Api);
+const isDeploymentReady = async (deployment: V1Deployment) => {
+  const deploy = await app.readNamespacedDeployment({
+    name: deployment.metadata?.name,
+    namespace: CURRENT_NAMESPACE,
+  });
+  return deploy.status?.updatedReplicas === deploy.status?.replicas;
+};
+
+// Restart the deployment
+const deployment = await app.patchNamespacedDeployment(
+  {
+    name: "anvilops",
+    namespace: CURRENT_NAMESPACE,
+    body: {
+      spec: {
+        template: {
+          metadata: {
+            annotations: {
+              "kubectl.kubernetes.io/restartedAt": new Date().toISOString(),
+            },
+          },
+        },
+      },
+    },
+  },
+  setHeaderOptions("Content-Type", PatchStrategy.MergePatch),
+);
+console.log("Deployment restarted");
+
+let ready = false;
+const maxDelay = 5000;
+const maxRetries = 8;
+for (let i = 0; i < maxRetries; i++) {
+  if (await isDeploymentReady(deployment)) {
+    ready = true;
+    break;
+  }
+  const delay = Math.min(500 * Math.pow(2, i), maxDelay);
+  await new Promise((resolve) => setTimeout(resolve, delay));
+}
+
+if (!ready) {
+  throw new Error("Timed out waiting for deployment to restart");
+}
+
+// Delete the previous Kubeconfig
+if (KUBECONFIG_SECRET_NAME && CLUSTER_ID) {
+  const oldKubeconfigName = kc.getCurrentUser().token.split(":")[0];
+  await fetch(`${RANCHER_API_BASE}/tokens/${oldKubeconfigName}`, {
+    method: "DELETE",
+    headers: {
+      Authorization: `Basic ${token}`,
+      Accept: "application/json",
+    },
+  });
+  console.log("Deleted previous Kubeconfig");
+}
+
+// Delete the previous Rancher token
+const rancherTokenName = Buffer.from(RANCHER_TOKEN, "base64")
+  .toString("utf-8")
+  .split(":")[0];
+await fetch(`${RANCHER_API_BASE}/tokens/${rancherTokenName}`, {
+  method: "DELETE",
+  headers: {
+    Authorization: `Basic ${token}`,
+    Accept: "application/json",
+  },
+});
+console.log("Deleted previous Rancher token");

backend/src/lib/cluster/rancher.ts

@@ -1,11 +1,7 @@
-import { KubeConfig } from "@kubernetes/client-node";
 import { getOrCreate } from "../cache.ts";
 import { env } from "../env.ts";
 import { getClientForClusterUsername } from "./kubernetes.ts";
 
-const kc = new KubeConfig();
-kc.loadFromDefault();
-
 const token = env["RANCHER_TOKEN"];
 const headers = {
   Authorization: `Basic ${token}`,