diff --git a/backend/src/handlers/getAppStatus.ts b/backend/src/handlers/getAppStatus.ts index bae9aef..e10c683 100644 --- a/backend/src/handlers/getAppStatus.ts +++ b/backend/src/handlers/getAppStatus.ts @@ -1,5 +1,5 @@ import { once } from "node:events"; -import { AppNotFoundError } from "../service/common/errors.ts"; +import { AppNotFoundError, ValidationError } from "../service/common/errors.ts"; import { getAppStatus, type StatusUpdate } from "../service/getAppStatus.ts"; import { json, type HandlerMap } from "../types.ts"; import type { AuthenticatedRequest } from "./index.ts"; @@ -41,6 +41,10 @@ export const getAppStatusHandler: HandlerMap["getAppStatus"] = async ( update, ); } catch (e) { + if (e instanceof ValidationError) { + return json(400, res, { code: 400, message: e.message }); + } + if (e instanceof AppNotFoundError) { return json(404, res, { code: 404, message: "App not found." }); } diff --git a/backend/src/lib/cluster/resources.ts b/backend/src/lib/cluster/resources.ts index 068ded6..d89057b 100644 --- a/backend/src/lib/cluster/resources.ts +++ b/backend/src/lib/cluster/resources.ts @@ -17,6 +17,7 @@ import { getOctokit } from "../octokit.ts"; import { createIngressConfig } from "./resources/ingress.ts"; import { createServiceConfig } from "./resources/service.ts"; import { + createDeploymentConfig, createStatefulSetConfig, generateAutomaticEnvVars, } from "./resources/statefulset.ts"; @@ -37,6 +38,10 @@ export const MAX_STS_NAME_LEN = 60; export const getRandomTag = (): string => randomBytes(4).toString("hex"); export const RANDOM_TAG_LEN = 8; + +export const isStatefulSet = (config: WorkloadConfig) => + config.mounts.length > 0; + export interface K8sObject { apiVersion: string; kind: string; @@ -195,9 +200,13 @@ export const createAppConfigsFromDeployment = async ( const svc = createServiceConfig(params); const ingress = createIngressConfig(params); - const statefulSet = await createStatefulSetConfig(params); - configs.push(statefulSet, svc); + const deploymentSpec = + params.mounts.length === 0 + ? await createDeploymentConfig(params) + : await createStatefulSetConfig(params); + + configs.push(deploymentSpec, svc); if (ingress !== null) { // ^ Can be null if APP_DOMAIN is not set, meaning no Ingress should be created for the app configs.push(ingress); diff --git a/backend/src/lib/cluster/resources/statefulset.ts b/backend/src/lib/cluster/resources/statefulset.ts index 9485c72..a7a08bf 100644 --- a/backend/src/lib/cluster/resources/statefulset.ts +++ b/backend/src/lib/cluster/resources/statefulset.ts @@ -1,4 +1,8 @@ -import type { V1EnvVar, V1StatefulSet } from "@kubernetes/client-node"; +import type { + V1Deployment, + V1EnvVar, + V1StatefulSet, +} from "@kubernetes/client-node"; import crypto from "node:crypto"; import type { Octokit } from "octokit"; import type { App, Deployment, WorkloadConfig } from "../../../db/models.ts"; @@ -195,3 +199,68 @@ export const createStatefulSetConfig = async ( return base; }; + +export const createDeploymentConfig = async ( + params: DeploymentParams, +): Promise => { + const base = { + apiVersion: "apps/v1", + kind: "Deployment", + metadata: { + name: params.name, + namespace: params.namespace, + }, + spec: { + selector: { + matchLabels: { + app: params.name, + }, + }, + replicas: params.replicas, + template: { + metadata: { + labels: { + app: params.name, + }, + }, + spec: { + automountServiceAccountToken: false, + // Set to an empty array (instead of undefined) so that disabling collectLogs in an existing app + // removes the initContainer + initContainers: [], + volumes: [], // same as above + containers: [ + { + name: params.name, + image: params.image, + imagePullPolicy: "Always", + ports: [ + { + containerPort: params.port, + protocol: "TCP", + }, + ], + resources: { + requests: params.requests, + limits: params.limits, + }, + env: params.env, + lifecycle: {}, + }, + ], + }, + }, + }, + }; + + if (params.collectLogs) { + base.spec.template = await wrapWithLogExporter( + "runtime", + params.logIngestSecret, + params.deploymentId, + base.spec.template, + ); + } + + return base; +}; diff --git a/backend/src/service/getAppByID.ts b/backend/src/service/getAppByID.ts index cc943aa..dee3432 100644 --- a/backend/src/service/getAppByID.ts +++ b/backend/src/service/getAppByID.ts @@ -15,40 +15,55 @@ export async function getAppByID(appId: number, userId: number) { throw new AppNotFoundError(); } + const [org, appGroup, currentConfig] = await Promise.all([ + db.org.getById(app.orgId), + db.appGroup.getById(app.appGroupId), + db.deployment.getConfig(recentDeployment.id), + ]); + // Fetch the current StatefulSet to read its labels const getK8sDeployment = async () => { + if (currentConfig.appType !== "workload") { + return null; + } try { const { AppsV1Api: api } = await getClientsForRequest( userId, app.projectId, ["AppsV1Api"], ); - return await api.readNamespacedStatefulSet({ - namespace: app.namespace, - name: app.name, - }); + if (currentConfig.asWorkloadConfig().mounts.length > 0) { + return await api.readNamespacedStatefulSet({ + namespace: app.namespace, + name: app.name, + }); + } else { + return await api.readNamespacedDeployment({ + namespace: app.namespace, + name: app.name, + }); + } } catch {} }; - const [org, appGroup, currentConfig, activeDeployment] = await Promise.all([ - db.org.getById(app.orgId), - db.appGroup.getById(app.appGroupId), - db.deployment.getConfig(recentDeployment.id), - (await getK8sDeployment())?.spec?.template?.metadata?.labels?.[ - "anvilops.rcac.purdue.edu/deployment-id" - ], + // Fetch repository info if this app is deployed from a Git repository + const [{ repoId, repoURL }, activeDeployment] = await Promise.all([ + (async () => { + if (currentConfig.source === "GIT" && org.githubInstallationId) { + const octokit = await getOctokit(org.githubInstallationId); + const repo = await getRepoById(octokit, currentConfig.repositoryId); + return { repoId: repo.id, repoURL: repo.html_url }; + } else { + return { repoId: undefined, repoURL: undefined }; + } + })(), + getK8sDeployment(), ]); - // Fetch repository info if this app is deployed from a Git repository - const { repoId, repoURL } = await (async () => { - if (currentConfig.source === "GIT" && org.githubInstallationId) { - const octokit = await getOctokit(org.githubInstallationId); - const repo = await getRepoById(octokit, currentConfig.repositoryId); - return { repoId: repo.id, repoURL: repo.html_url }; - } else { - return { repoId: undefined, repoURL: undefined }; - } - })(); + const activeDeploymentId = + activeDeployment?.metadata?.labels?.[ + "anvilops.rcac.purdue.edu/deployment-id" + ]; return { id: app.id, @@ -68,7 +83,9 @@ export async function getAppByID(appId: number, userId: number) { name: !appGroup.isMono ? appGroup.name : undefined, id: app.appGroupId, }, - activeDeployment: activeDeployment ? parseInt(activeDeployment) : undefined, + activeDeployment: activeDeploymentId + ? parseInt(activeDeploymentId) + : undefined, deploymentCount, }; } diff --git a/backend/src/service/getAppStatus.ts b/backend/src/service/getAppStatus.ts index 2d47588..e40c719 100644 --- a/backend/src/service/getAppStatus.ts +++ b/backend/src/service/getAppStatus.ts @@ -1,17 +1,19 @@ import { AbortError, + V1StatefulSet, type CoreV1EventList, type KubernetesListObject, type KubernetesObject, + type V1Deployment, type V1PodCondition, type V1PodList, - type V1StatefulSet, type Watch, } from "@kubernetes/client-node"; -import { metrics, ValueType } from "@opentelemetry/api"; +import { ValueType, metrics } from "@opentelemetry/api"; import { db } from "../db/index.ts"; import { getClientsForRequest } from "../lib/cluster/kubernetes.ts"; -import { AppNotFoundError } from "./common/errors.ts"; +import { isStatefulSet } from "../lib/cluster/resources.ts"; +import { AppNotFoundError, ValidationError } from "./common/errors.ts"; const meter = metrics.getMeter("app_status_viewer"); const concurrentViewers = meter.createUpDownCounter( @@ -31,20 +33,27 @@ export async function getAppStatus( abortController: AbortController, callback: (status: StatusUpdate) => Promise, ) { - const app = await db.app.getById(appId, { - requireUser: { id: userId }, - }); + const [app, config] = await Promise.all([ + db.app.getById(appId, { + requireUser: { id: userId }, + }), + db.app.getDeploymentConfig(appId), + ]); if (!app) { throw new AppNotFoundError(); } + if (config.appType === "helm") { + throw new ValidationError("Cannot get app status for helm apps"); + } + let pods: V1PodList; - let statefulSet: V1StatefulSet; + let deployment: V1StatefulSet | V1Deployment; let events: CoreV1EventList; const update = async () => { - if (!pods || !events || !statefulSet) return; + if (!pods || !events || !deployment) return; const newStatus = { pods: pods.items.map((pod) => ({ id: pod.metadata?.uid, @@ -63,24 +72,25 @@ export async function getAppStatus( image: pod.status?.containerStatuses?.[0]?.image, containerReady: pod.status?.containerStatuses?.[0]?.ready, containerState: pod.status?.containerStatuses?.[0]?.state, - lastState: pod.status?.containerStatuses?.[0].lastState, - ip: pod.status.podIP, + lastState: pod.status?.containerStatuses?.[0]?.lastState, + ip: pod.status?.podIP, })), events: events.items.map((event) => ({ reason: event.reason, message: event.message, count: event.count, - firstTimestamp: event.firstTimestamp.toISOString(), - lastTimestamp: event.lastTimestamp.toISOString(), + firstTimestamp: event.firstTimestamp?.toISOString(), + lastTimestamp: event.lastTimestamp?.toISOString(), })), - statefulSet: { - readyReplicas: statefulSet.status.readyReplicas, - updatedReplicas: statefulSet.status.currentReplicas, - replicas: statefulSet.status.replicas, - generation: statefulSet.metadata.generation, - observedGeneration: statefulSet.status.observedGeneration, - currentRevision: statefulSet.status.currentRevision, - updateRevision: statefulSet.status.updateRevision, + deployment: { + readyReplicas: deployment.status?.readyReplicas, + replicas: deployment.spec?.replicas, + generation: deployment.metadata?.generation, + observedGeneration: deployment.status?.observedGeneration, + ...(deployment instanceof V1StatefulSet && { + currentRevision: deployment.status?.currentRevision, + updateRevision: deployment.status?.updateRevision, + }), }, }; @@ -128,25 +138,44 @@ export async function getAppStatus( ); abortController.signal.addEventListener("abort", () => podWatcher.abort()); - const statefulSetWatcher = await watchList( - watch, - `/apis/apps/v1/namespaces/${ns}/statefulsets`, - async () => - await apps.listNamespacedStatefulSet({ - namespace: ns, - }), - {}, - async (newValue) => { - statefulSet = newValue.items.find( - (it) => it.metadata.name === app.name, - ); - await update(); - }, - close, - ); - abortController.signal.addEventListener("abort", () => - statefulSetWatcher.abort(), - ); + let watcher: Awaited>; + if (isStatefulSet(config.asWorkloadConfig())) { + watcher = await watchList( + watch, + `/apis/apps/v1/namespaces/${ns}/statefulsets`, + async () => + await apps.listNamespacedStatefulSet({ + namespace: ns, + }), + {}, + async (newValue) => { + deployment = newValue.items.find( + (it) => it.metadata.name === app.name, + ); + await update(); + }, + close, + ); + } else { + watcher = await watchList( + watch, + `/apis/apps/v1/namespaces/${ns}/deployments`, + async () => + await apps.listNamespacedDeployment({ + namespace: ns, + }), + {}, + async (newValue) => { + deployment = newValue.items.find( + (it) => it.metadata.name === app.name, + ); + await update(); + }, + close, + ); + } + + abortController.signal.addEventListener("abort", () => watcher.abort()); const fieldSelector = `involvedObject.kind=StatefulSet,involvedObject.name=${app.name},type=Warning`; diff --git a/backend/src/service/helper/app.ts b/backend/src/service/helper/app.ts index c695fda..a1d602d 100644 --- a/backend/src/service/helper/app.ts +++ b/backend/src/service/helper/app.ts @@ -137,7 +137,7 @@ export class AppService { app.type === "update" ? app.existingAppId : undefined, ); } else if (app.config.appType === "helm") { - if (!env.ALLOW_HELM_DEPLOYMENTS) { + if (env.ALLOW_HELM_DEPLOYMENTS !== "true") { throw new ValidationError("Helm deployments are disabled"); } } diff --git a/backend/src/service/helper/deploymentConfig.ts b/backend/src/service/helper/deploymentConfig.ts index f5d4a21..3657655 100644 --- a/backend/src/service/helper/deploymentConfig.ts +++ b/backend/src/service/helper/deploymentConfig.ts @@ -162,7 +162,7 @@ export class DeploymentConfigService { replicas: config.replicas, port: config.port, mounts: config.mounts, - commitHash: "unknown", + commitHash: null, imageTag: config.imageTag, }; } diff --git a/backend/src/service/listCharts.ts b/backend/src/service/listCharts.ts index 2adac62..c359010 100644 --- a/backend/src/service/listCharts.ts +++ b/backend/src/service/listCharts.ts @@ -5,7 +5,7 @@ import { getRepositoriesByProject } from "../lib/registry.ts"; import { ValidationError } from "./common/errors.ts"; export async function listCharts() { - if (!env.ALLOW_HELM_DEPLOYMENTS) { + if (env.ALLOW_HELM_DEPLOYMENTS !== "true") { throw new ValidationError("Helm deployments are disabled"); } return JSON.parse( diff --git a/backend/src/service/listRepoBranches.ts b/backend/src/service/listRepoBranches.ts index 59c7ac0..59f7ab4 100644 --- a/backend/src/service/listRepoBranches.ts +++ b/backend/src/service/listRepoBranches.ts @@ -32,6 +32,10 @@ export async function listRepoBranches( repo: repo.name, }); + if (branches.data.length === 0) { + throw new RepositoryNotFoundError(); + } + return { default: repo.default_branch, branches: branches.data.map((branch) => branch.name), diff --git a/frontend/src/components/config/workload/git/EnabledGitConfigFields.tsx b/frontend/src/components/config/workload/git/EnabledGitConfigFields.tsx index 6cdf36f..647b922 100644 --- a/frontend/src/components/config/workload/git/EnabledGitConfigFields.tsx +++ b/frontend/src/components/config/workload/git/EnabledGitConfigFields.tsx @@ -1,4 +1,3 @@ -import { ImportRepoDialog } from "./ImportRepoDialog"; import { Input } from "@/components/ui/input"; import { Label } from "@/components/ui/label"; import { RadioGroup, RadioGroupItem } from "@/components/ui/radio-group"; @@ -12,6 +11,7 @@ import { SelectValue, } from "@/components/ui/select"; import { api } from "@/lib/api"; +import type { CommonFormFields, GitFormFields } from "@/lib/form.types"; import clsx from "clsx"; import { BookMarked, @@ -23,7 +23,7 @@ import { Hammer, } from "lucide-react"; import { useEffect, useState } from "react"; -import type { CommonFormFields, GitFormFields } from "@/lib/form.types"; +import { ImportRepoDialog } from "./ImportRepoDialog"; export const EnabledGitConfigFields = ({ orgId, @@ -84,6 +84,16 @@ export const EnabledGitConfigFields = ({ }, { enabled: orgId !== undefined && repositoryId !== undefined, + // Sometimes when a repository was just created, the list of branches is empty and listRepoBranches returns a 404 + retry(failureCount, error) { + if (error && "code" in error && error.code === 404) { + return failureCount < 3; + } + return false; + }, + retryDelay(attemptIndex) { + return Math.min(500 * 2 ** attemptIndex, 4000); + }, }, ); diff --git a/frontend/src/lib/api.ts b/frontend/src/lib/api.ts index 82d399a..5a359d2 100644 --- a/frontend/src/lib/api.ts +++ b/frontend/src/lib/api.ts @@ -47,6 +47,17 @@ const onError = ( // Don't show the error toast for the initial /user/me request return; } + // Don't show error toast for 404s on the branches endpoint + if ( + args.length === 1 && + "code" in error && + error.code === 404 && + typeof args[0].queryHash === "string" && + args[0].queryHash.includes("/repos/") && + args[0].queryHash.includes("/branches") + ) { + return; + } toast.error( `Something went wrong: ${error.message ?? JSON.stringify(error)}`, ); diff --git a/frontend/src/pages/app/StatusTab.tsx b/frontend/src/pages/app/StatusTab.tsx index 5db353a..8f65fc8 100644 --- a/frontend/src/pages/app/StatusTab.tsx +++ b/frontend/src/pages/app/StatusTab.tsx @@ -50,7 +50,7 @@ export const StatusTab = ({ ); const pods = status?.pods; - const statefulSet = status?.statefulSet; + const deployment = status?.deployment; const events = status?.events; const activePods = pods?.filter( @@ -61,8 +61,8 @@ export const StatusTab = ({ ); const updating = - statefulSet?.currentRevision !== statefulSet?.updateRevision || - statefulSet?.generation !== statefulSet?.observedGeneration; + deployment?.currentRevision !== deployment?.updateRevision || + deployment?.generation !== deployment?.observedGeneration; const { mutateAsync: deletePod, @@ -74,10 +74,9 @@ export const StatusTab = ({ <>

Pods{" "} - {statefulSet && ( + {deployment && ( - ({statefulSet?.readyReplicas ?? 0}/{statefulSet?.replicas ?? 0}{" "} - ready) + ({deployment?.readyReplicas ?? 0}/{deployment?.replicas ?? 0} ready) )}

@@ -109,10 +108,8 @@ export const StatusTab = ({

Update In Progress ( - {statefulSet?.updatedReplicas - ? statefulSet.replicas! - statefulSet.updatedReplicas - : (statefulSet?.readyReplicas ?? 0)} - /{statefulSet?.replicas}) + {activePods?.filter((it) => it.podReady)?.length ?? 0}/ + {deployment?.replicas})

New pods are being created to replace old ones.

@@ -260,15 +257,18 @@ const PodStatusText = ({ pod }: { pod: Pod }) => { This container is crashing repeatedly. It will be restarted{" "} {getRestartTime(pod)}.

- {lastState(pod, "terminated")?.finishedAt && ( -

- It most recently exited at{" "} - {timeFormat.format( - new Date(lastState(pod, "terminated")!.finishedAt!), - )}{" "} - with status code {lastState(pod, "terminated")?.exitCode}. -

- )} + {(() => { + const terminated = lastState(pod, "terminated"); + return ( + terminated?.finishedAt && ( +

+ It most recently exited at{" "} + {timeFormat.format(new Date(terminated.finishedAt))} with + status code {terminated.exitCode}. +

+ ) + ); + })()} ); case "ErrImagePull": diff --git a/openapi/openapi.yaml b/openapi/openapi.yaml index d9483b0..0aea7cd 100644 --- a/openapi/openapi.yaml +++ b/openapi/openapi.yaml @@ -2655,13 +2655,11 @@ components: $ref: "#/components/schemas/ContainerState" ip: type: string - statefulSet: + deployment: type: object properties: readyReplicas: type: number - updatedReplicas: - type: number replicas: type: number generation: @@ -2689,6 +2687,7 @@ components: lastTimestamp: type: string format: date-time + required: [pods, events, deployment] RancherProject: type: object properties: