feat: make api work with external Git source (#915)

* feat: make api work with external Git source

* feat: only get internalRepoUrls if Gitea is enabled

* fix: teamApps setting enabled correctly

* fix: rename gitea references to git

* feat: add git repoUrl to settingsinfo.yaml

* feat: update src/otomi-stack.ts

Co-authored-by: Ferruh <63190600+ferruhcihan@users.noreply.github.com>

---------

Co-authored-by: svcAPLBot <174728082+svcAPLBot@users.noreply.github.com>
Co-authored-by: Ferruh <63190600+ferruhcihan@users.noreply.github.com>

Author: 3 people

docs/gitops.md

@@ -116,7 +116,7 @@ sequenceDiagram
 
 ## Version two
 
-The locking mechanism is removed by removing the master session controller concept. The session repo controller pulls and pushes from/to Gitea instead of master repo. The Session Controller is also renamed to `Git handler` to not confuse it with user session.
+The locking mechanism is removed by removing the master session controller concept. The session repo controller pulls and pushes from/to Git instead of master repo. The Session Controller is also renamed to `Git handler` to not confuse it with user session.
 
 **Sequence diagram for the accepted request**
 The following diagram presents GitOps without locking mechanism. It is worth noting that is performs eight operations less comparing to its predecessor.

src/app.ts

@@ -27,19 +27,15 @@ import {
   CHECK_LATEST_COMMIT_INTERVAL,
   cleanEnv,
   EXPRESS_PAYLOAD_LIMIT,
-  GIT_PASSWORD,
   GIT_PUSH_RETRIES,
-  GIT_USER,
   TRUST_PROXY,
 } from 'src/validators'
 import swaggerUi from 'swagger-ui-express'
-import giteaCheckLatest from './gitea/connect'
+import getLatestRemoteCommitSha from './git/connect'
 import { getBuildStatus, getSealedSecretStatus, getServiceStatus, getWorkloadStatus } from './k8s_operations'
 
 const env = cleanEnv({
   CHECK_LATEST_COMMIT_INTERVAL,
-  GIT_USER,
-  GIT_PASSWORD,
   EXPRESS_PAYLOAD_LIMIT,
   GIT_PUSH_RETRIES,
   TRUST_PROXY,
@@ -54,14 +50,13 @@ type OtomiSpec = {
   valuesSchema: Record<string, any>
 }
 
-// get the latest commit from Gitea and checks it against the local values
-const checkAgainstGitea = async () => {
-  const encodedToken = Buffer.from(`${env.GIT_USER}:${env.GIT_PASSWORD}`).toString('base64')
+// get the latest commit from Git and checks it against the local values
+const checkAgainstGit = async () => {
   const otomiStack = await getSessionStack()
-  const latestOtomiVersion = await giteaCheckLatest(encodedToken)
+  const latestOtomiVersion = await getLatestRemoteCommitSha()
   // check the local version against the latest online version
   // if the latest online is newer it will be pulled locally
-  if (latestOtomiVersion && latestOtomiVersion.data[0].sha !== otomiStack.git.commitSha) {
+  if (latestOtomiVersion && latestOtomiVersion !== otomiStack.git.commitSha) {
     debug('Local values differentiate from Git repository, retrieving latest values')
     // Remove all .dec files
     await otomiStack.git.git.clean([CleanOptions.FORCE, CleanOptions.IGNORED_ONLY, CleanOptions.RECURSIVE])
@@ -207,7 +202,7 @@ export async function initApp(inOtomiStack?: OtomiStack) {
   if (!env.isTest) {
     const gitCheckVersionInterval = env.CHECK_LATEST_COMMIT_INTERVAL * 60 * 1000
     setInterval(async function () {
-      await checkAgainstGitea()
+      await checkAgainstGit()
     }, gitCheckVersionInterval)
   }
   let server: Server | undefined

src/git/connect.ts

@@ -0,0 +1,31 @@
+import Debug from 'debug'
+import simpleGit from 'simple-git'
+import { cleanEnv, GIT_BRANCH, GIT_PASSWORD, GIT_REPO_URL, GIT_USER } from 'src/validators'
+
+const debug = Debug('otomi:git-connect')
+
+const env = cleanEnv({
+  GIT_REPO_URL,
+  GIT_BRANCH,
+  GIT_USER,
+  GIT_PASSWORD,
+})
+
+export default async function getLatestRemoteCommitSha(): Promise<string | undefined> {
+  try {
+    const git = simpleGit()
+    const repoUrl = new URL(env.GIT_REPO_URL)
+    repoUrl.username = encodeURIComponent(env.GIT_USER)
+    repoUrl.password = encodeURIComponent(env.GIT_PASSWORD)
+    const result = await git.listRemote(['--refs', repoUrl.toString(), env.GIT_BRANCH])
+    const [sha] = result.trim().split(/\s/)
+    if (!sha) {
+      debug('No remote commit found for branch %s', env.GIT_BRANCH)
+      return undefined
+    }
+    return sha
+  } catch (error: any) {
+    debug('Git remote error: ', error.message)
+    return undefined
+  }
+}

src/gitea/connect.ts

@@ -228,6 +228,42 @@ Settings:
       type: object
       additionalProperties: false
       properties:
+        git:
+          type: object
+          title: Git Configuration
+          description: |
+            Git configuration for APL values repository.
+          additionalProperties: false
+          properties:
+            repoUrl:
+              type: string
+              description: |
+                The base URL of the Git repository (without credentials).
+              pattern: '^https?://.+'
+            username:
+              type: string
+              description: |
+                Username for authenticating with the Git repository.
+                Defaults to 'otomi-admin' for internal Gitea.
+            password:
+              type: string
+              description: Password or token for authenticating with the Git repository
+              x-secret: '{{ randAlphaNum 20 }}'
+            email:
+              type: string
+              description: |
+                Email address to use for Git commits.
+                Defaults to 'pipeline@cluster.local' for internal Gitea.
+              format: email
+            branch:
+              type: string
+              description: The branch to use in the Git repository
+          required:
+            - repoUrl
+            - username
+            - password
+            - email
+            - branch
         adminPassword:
           description: Master admin password that will be used for all apps that are not configured to use their own password.
           $ref: 'definitions.yaml#/adminPassword'

src/openapi/settings.yaml

@@ -50,6 +50,15 @@ SettingsInfo:
         hasExternalIDP:
           type: boolean
           default: false
+        git:
+          properties:
+            repoUrl:
+              type: string
+              description: The base URL of the Git repository (without credentials).
+              pattern: '^https?://.+'
+            branch:
+              type: string
+              description: The branch to use in the Git repository.
     smtp:
       properties:
         smarthost:

src/openapi/settingsinfo.yaml

@@ -1,4 +1,4 @@
-import { CoreV1Api, User as k8sUser, KubeConfig, V1ObjectReference } from '@kubernetes/client-node'
+import { CoreV1Api, KubeConfig, User as k8sUser, V1ObjectReference } from '@kubernetes/client-node'
 import Debug from 'debug'
 
 import { getRegions, ObjectStorageKeyRegions } from '@linode/api-v4'
@@ -295,10 +295,21 @@ export default class OtomiStack {
 
   getSettingsInfo(): SettingsInfo {
     const settings = this.getSettings(['cluster', 'dns', 'otomi', 'smtp', 'ingress'])
+    const otomiInfo = pick(settings.otomi, [
+      'hasExternalDNS',
+      'hasExternalIDP',
+      'isPreInstalled',
+      'aiEnabled',
+      'git.repoUrl',
+      'git.branch',
+    ])
+    if (otomiInfo.git?.repoUrl?.includes('gitea-http.gitea.svc.cluster.local')) {
+      otomiInfo.git.repoUrl = `https://gitea.${settings.cluster?.domainSuffix}/otomi/values`
+    }
     return {
       cluster: pick(settings.cluster, ['name', 'domainSuffix', 'apiServer', 'provider', 'linode']),
       dns: pick(settings.dns, ['zones']),
-      otomi: pick(settings.otomi, ['hasExternalDNS', 'hasExternalIDP', 'isPreInstalled', 'aiEnabled']),
+      otomi: otomiInfo,
       smtp: pick(settings.smtp, ['smarthost']),
       ingressClassNames: map(settings.ingress?.classes, 'className') ?? [],
     } as SettingsInfo
@@ -1292,10 +1303,11 @@ export default class OtomiStack {
 
   async getInternalRepoUrls(teamId: string): Promise<string[]> {
     if (env.isDev || !teamId || teamId === 'admin') return []
-    const { cluster, otomi } = this.getSettings(['cluster', 'otomi'])
     const gitea = this.getApp('gitea')
-    const username = (gitea?.values?.adminUsername ?? '') as string
-    const password = (gitea?.values?.adminPassword ?? otomi?.adminPassword ?? '') as string
+    if (!gitea?.values?.enabled) return []
+    const { cluster, otomi } = this.getSettings(['cluster', 'otomi'])
+    const username = (otomi?.git?.username ?? '') as string
+    const password = (otomi?.git?.password ?? '') as string
     const orgName = `team-${teamId}`
     const domainSuffix = cluster?.domainSuffix
     const internalRepoUrls = (await getGiteaRepoUrls(username, password, orgName, domainSuffix)) || []

src/otomi-stack.ts

@@ -28,7 +28,7 @@ export const EDITOR_INACTIVITY_TIMEOUT = num({
 })
 export const GIT_BRANCH = str({ desc: 'The git repo branch', default: 'main' })
 export const CHECK_LATEST_COMMIT_INTERVAL = num({
-  desc: 'Interval in minutes for how much time in between each gitea latest commit check',
+  desc: 'Interval in minutes for how much time in between each git latest commit check',
   default: 2,
 })
 export const GIT_EMAIL = str({ desc: 'The git user email', default: 'not@us.ed' })