diff --git a/.github/workflows/check-acceptance-test-code.yml b/.github/workflows/check-acceptance-test-code.yml index b7e13c3b04..28a0daf9de 100644 --- a/.github/workflows/check-acceptance-test-code.yml +++ b/.github/workflows/check-acceptance-test-code.yml @@ -12,7 +12,7 @@ jobs: name: Scala dependencies and code check runs-on: ubuntu-24.04 steps: - - uses: actions/checkout@v4.1.7 + - uses: actions/checkout@v4.2.2 - name: Setup JDK uses: actions/setup-java@v4 with: diff --git a/.github/workflows/check-acceptance-test-fmt.yml b/.github/workflows/check-acceptance-test-fmt.yml index 2a16017abf..2bca5503ea 100644 --- a/.github/workflows/check-acceptance-test-fmt.yml +++ b/.github/workflows/check-acceptance-test-fmt.yml @@ -12,7 +12,7 @@ jobs: name: Scala formatting check runs-on: ubuntu-24.04 steps: - - uses: actions/checkout@v4.1.7 + - uses: actions/checkout@v4.2.2 - name: Setup JDK uses: actions/setup-java@v4 with: diff --git a/.github/workflows/create-release-branch.yml b/.github/workflows/create-release-branch.yml index 07122e32ef..71d722f272 100644 --- a/.github/workflows/create-release-branch.yml +++ b/.github/workflows/create-release-branch.yml @@ -16,7 +16,7 @@ jobs: create-release-pr: runs-on: ubuntu-24.04 steps: - - uses: actions/checkout@v4.1.7 + - uses: actions/checkout@v4.2.2 with: fetch-depth: 0 token: "${{ secrets.RENKUBOT_GITHUB_TOKEN }}" diff --git a/.github/workflows/cron-jobs.yaml b/.github/workflows/cron-jobs.yaml index 8c771e10b3..9138f2b4c7 100644 --- a/.github/workflows/cron-jobs.yaml +++ b/.github/workflows/cron-jobs.yaml @@ -11,7 +11,7 @@ jobs: runs-on: ubuntu-24.04 steps: - name: renku teardown - uses: SwissDataScienceCenter/renku-actions/cleanup-renku-ci-deployments@v1.17.0 + uses: SwissDataScienceCenter/renku-actions/cleanup-renku-ci-deployments@v1.18.2 env: GITLAB_TOKEN: ${{ secrets.DEV_GITLAB_TOKEN }} RENKUBOT_KUBECONFIG: ${{ secrets.RENKUBOT_DEV_KUBECONFIG }} diff --git a/.github/workflows/publish-helm-chart.yml b/.github/workflows/publish-helm-chart.yml index 6edc56dcbf..8e67bad766 100644 --- a/.github/workflows/publish-helm-chart.yml +++ b/.github/workflows/publish-helm-chart.yml @@ -9,13 +9,13 @@ jobs: publish-chart: runs-on: ubuntu-24.04 steps: - - uses: actions/checkout@v4.1.7 + - uses: actions/checkout@v4.2.2 with: fetch-depth: 0 - name: Set version id: vars run: echo "tag=${GITHUB_REF#refs/*/}" >> $GITHUB_OUTPUT - - uses: SwissDataScienceCenter/renku-actions/publish-chart@v1.17.0 + - uses: SwissDataScienceCenter/renku-actions/publish-chart@v1.18.2 env: CHART_DIR: helm-chart/ CHART_NAME: renku diff --git a/.github/workflows/publish-master-merges.yaml b/.github/workflows/publish-master-merges.yaml index 355af66c71..200d5b9a8a 100644 --- a/.github/workflows/publish-master-merges.yaml +++ b/.github/workflows/publish-master-merges.yaml @@ -14,7 +14,7 @@ jobs: publish-chart: runs-on: ubuntu-24.04 steps: - - uses: actions/checkout@v4.1.7 + - uses: actions/checkout@v4.2.2 with: fetch-depth: 0 - uses: azure/setup-helm@v4 @@ -35,7 +35,7 @@ jobs: - id: set-version run: | echo "publish_version=${{ steps.bump-semver.outputs.new_version }}.$(echo ${{ github.sha }} | cut -c 1-7)" >> $GITHUB_ENV - - uses: SwissDataScienceCenter/renku-actions/publish-chart@v1.17.0 + - uses: SwissDataScienceCenter/renku-actions/publish-chart@v1.18.2 env: CHART_DIR: helm-chart/ CHART_TAG: "--tag ${{env.publish_version}}" diff --git a/.github/workflows/pull-request-test.yml b/.github/workflows/pull-request-test.yml index aa718ee9f6..7226f65af6 100644 --- a/.github/workflows/pull-request-test.yml +++ b/.github/workflows/pull-request-test.yml @@ -23,7 +23,7 @@ jobs: runs-on: ubuntu-24.04 if: github.event.action != 'closed' steps: - - uses: actions/checkout@v4.1.7 + - uses: actions/checkout@v4.2.2 - uses: actions/setup-java@v4 with: distribution: "temurin" @@ -71,11 +71,11 @@ jobs: extra-values: ${{ steps.deploy-comment.outputs.extra-values || steps.deploy-comment-azure.outputs.extra-values }} steps: - id: deploy-comment - uses: SwissDataScienceCenter/renku-actions/check-pr-description@v1.18.1 + uses: SwissDataScienceCenter/renku-actions/check-pr-description@v1.18.2 with: pr_ref: ${{ github.event.number }} - id: deploy-comment-azure - uses: SwissDataScienceCenter/renku-actions/check-pr-description@v1.18.1 + uses: SwissDataScienceCenter/renku-actions/check-pr-description@v1.18.2 with: string: /AzureDeploy pr_ref: ${{ github.event.number }} @@ -91,7 +91,7 @@ jobs: name: ci-renku-${{ github.event.number }} url: https://ci-renku-${{ github.event.number }}.dev.renku.ch steps: - - uses: actions/checkout@v4.1.7 + - uses: actions/checkout@v4.2.2 - name: Find deployment url if: needs.check-deploy.outputs.switch-deploy == 'true' uses: peter-evans/find-comment@v3 @@ -111,7 +111,7 @@ jobs: You can access the deployment of this PR at https://ci-renku-${{ github.event.number }}.dev.renku.ch - name: renku build and deploy if: needs.check-deploy.outputs.switch-deploy == 'true' - uses: SwissDataScienceCenter/renku-actions/deploy-renku@v1.18.1 + uses: SwissDataScienceCenter/renku-actions/deploy-renku@v1.18.2 env: DOCKER_PASSWORD: ${{ secrets.RENKU_DOCKER_PASSWORD }} DOCKER_USERNAME: ${{ secrets.RENKU_DOCKER_USERNAME }} @@ -143,7 +143,7 @@ jobs: id-token: write if: github.event.action != 'closed' steps: - - uses: actions/checkout@v4.1.7 + - uses: actions/checkout@v4.2.2 - name: Find deployment url if: needs.check-deploy.outputs.azure-deploy == 'true' uses: peter-evans/find-comment@v3 @@ -188,7 +188,7 @@ jobs: - name: renku build and deploy if: needs.check-deploy.outputs.azure-deploy == 'true' - uses: SwissDataScienceCenter/renku-actions/deploy-renku@v1.18.1 + uses: SwissDataScienceCenter/renku-actions/deploy-renku@v1.18.2 env: DOCKER_PASSWORD: ${{ secrets.RENKU_DOCKER_PASSWORD }} DOCKER_USERNAME: ${{ secrets.RENKU_DOCKER_USERNAME }} @@ -246,7 +246,7 @@ jobs: cat "${{ github.workspace }}/renkubot-kube.config" >> $GITHUB_ENV echo "EOF" >> $GITHUB_ENV - - uses: SwissDataScienceCenter/renku-actions/test-renku@v1.18.1 + - uses: SwissDataScienceCenter/renku-actions/test-renku@v1.18.2 with: kubeconfig: ${{ needs.check-deploy.outputs.azure-deploy == 'true' && env.RENKUBOT_KUBECONFIG || secrets.RENKUBOT_DEV_KUBECONFIG }} renku-release: ci-renku-${{ github.event.number }} @@ -273,7 +273,7 @@ jobs: rstudioSession, ] steps: - - uses: SwissDataScienceCenter/renku-actions/test-renku-cypress@v1.18.1 + - uses: SwissDataScienceCenter/renku-actions/test-renku-cypress@v1.18.2 if: github.event.action != 'closed' && (needs.check-deploy.outputs.switch-deploy == 'true' || needs.check-deploy.outputs.azure-deploy == 'true') && needs.check-deploy.outputs.test-legacy-enabled == 'true' with: e2e-target: ${{ matrix.tests }} @@ -300,7 +300,7 @@ jobs: sessionBasics, ] steps: - - uses: SwissDataScienceCenter/renku-actions/test-renku-cypress@v1.18.1 + - uses: SwissDataScienceCenter/renku-actions/test-renku-cypress@v1.18.2 if: github.event.action != 'closed' && (needs.check-deploy.outputs.switch-deploy == 'true' || needs.check-deploy.outputs.azure-deploy == 'true') && needs.check-deploy.outputs.test-enabled == 'true' with: e2e-folder: cypress/e2e/v2/ @@ -385,7 +385,7 @@ jobs: # Cleanup for both standard and Azure deployments - name: renku teardown - uses: SwissDataScienceCenter/renku-actions/cleanup-renku-ci-deployments@v1.18.1 + uses: SwissDataScienceCenter/renku-actions/cleanup-renku-ci-deployments@v1.18.2 env: HELM_RELEASE_REGEX: "^ci-renku-${{ github.event.number }}$" GITLAB_TOKEN: ${{ secrets.DEV_GITLAB_TOKEN }} diff --git a/.github/workflows/renku-dev-test.yaml b/.github/workflows/renku-dev-test.yaml index ec65221e5b..80e8655d6a 100644 --- a/.github/workflows/renku-dev-test.yaml +++ b/.github/workflows/renku-dev-test.yaml @@ -8,7 +8,7 @@ jobs: github.event.client_payload.message == 'Helm test succeeded' }} runs-on: ubuntu-24.04 steps: - - uses: SwissDataScienceCenter/renku-actions/test-renku@v1.17.0 + - uses: SwissDataScienceCenter/renku-actions/test-renku@v1.18.2 with: kubeconfig: ${{ secrets.RENKUBOT_DEV_KUBECONFIG }} renku-release: renku @@ -22,7 +22,7 @@ jobs: github.event.client_payload.message == 'Helm test succeeded' }} runs-on: ubuntu-24.04 steps: - - uses: actions/checkout@v4.1.7 + - uses: actions/checkout@v4.2.2 - uses: cypress-io/github-action@v6 id: cypress env: diff --git a/CHANGELOG.rst b/CHANGELOG.rst index f8dc02562d..bd2cebdd15 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,9 +1,33 @@ .. _changelog: +0.68.0 +------ + +User-Facing Changes +~~~~~~~~~~~~~~~~~~~ + +**🐞 Bug Fixes** + +- **Core Service**: Fix a bug where removing activities wouldn't actually remove them. + +Internal Changes +~~~~~~~~~~~~~~~~ + +- **Helm chart**: Update the Keycloak theme image to use non-root user by default. +- **Data services**: Added k8s cache service that caches sessions in the data services database. +- **Admin tools**: Add Harbor initialization script to setup a registry for RenkuLab v2. + +Individual Components +~~~~~~~~~~~~~~~~~~~~~ + +- `renku-data-services 0.39.0 `_ +- `renku-python 2.9.4 `_ +- `renku-python 2.9.3 `_ + 0.67.2 ------ -Renku ``0.67.2`` fixes several bugs in the data services backend. +Renku ``0.67.2`` fixes several bugs in the data services backend. User-Facing Changes ~~~~~~~~~~~~~~~~~~~ @@ -11,7 +35,7 @@ User-Facing Changes **🐞 Bug Fixes** - **Data services**: Surface more specific message when Git integrations expire. -- **Data services**: Fix a bug where modifying the resource class of a hibernated +- **Data services**: Fix a bug where modifying the resource class of a hibernated session would cause it to not start back up when resumed. - **Data services**: Data connectors were failing to copy when copying projects. @@ -1601,7 +1625,7 @@ Internal Changes Individual Components ~~~~~~~~~~~~~~~~~~~~~ -- `renku-python 2.9.2 `_ +- `renku-python 2.9.2 `_ - `renku-data-services 0.5.0 `_ - `csi-rclone 0.1.7 `_ diff --git a/CITATION.cff b/CITATION.cff new file mode 100644 index 0000000000..12872e4b2a --- /dev/null +++ b/CITATION.cff @@ -0,0 +1,61 @@ +cff-version: 1.2.0 +message: "If you use this software, please cite it as below." +title: "Renku: A platform for sustainable data science" +authors: + - family-names: "Roškar" + given-names: "Rok" + - family-names: "Ramakrishnan" + given-names: "Chandrasekhar" + - family-names: "Volpi" + given-names: "Michele" + - family-names: "Perez-Cruz" + given-names: "Fernando" + - family-names: "Gasser" + given-names: "Lilian" + - family-names: "Ozdemir" + given-names: "Firat" + - family-names: "Paitz" + given-names: "Patrick" + - family-names: "Alisafaee" + given-names: "Mohammad" + - family-names: "Fischer" + given-names: "Philipp" + - family-names: "Grubenmann" + given-names: "Ralf" + - family-names: "Harris" + given-names: "Eliza" + - family-names: "Olevski" + given-names: "Tasko" + - family-names: "Remlinger" + given-names: "Carl" + - family-names: "Salamanca" + given-names: "Luis" + - family-names: "Capon Garcia" + given-names: "Elisabet" + - family-names: "Cavazzi" + given-names: "Lorenzo" + - family-names: "Chrobasik" + given-names: "Jakub" + - family-names: "Cordoba Osnas" + given-names: "Darlin" + - family-names: "Degano" + given-names: "Alessandro" + - family-names: "Dupre" + given-names: "Jimena" + - family-names: "Johnson" + given-names: "Wesley" + - family-names: "Kettner" + given-names: "Eike" + - family-names: "Kinkead" + given-names: "Laura" + - family-names: "Murphy" + given-names: "Sean D." + - family-names: "Thiebaut" + given-names: "Flora" + - family-names: "Verscheure" + given-names: "Olivier" +date-released: "2023" +version: "36" +publisher: "Curran Associates, Inc." +url: "https://proceedings.neurips.cc/paper_files/paper/2023/file/838694e9ab6b0a193b84daaafcac0eed-Paper-Datasets_and_Benchmarks.pdf" +type: "conference-paper" diff --git a/README.md b/README.md index 35dd7c471b..1f2d57673f 100644 --- a/README.md +++ b/README.md @@ -94,3 +94,7 @@ Renku is built from several sub-repositories: operator for user session servers. - [renkulab-docker](https://github.com/SwissDataScienceCenter/renkulab-docker): base images for interactive sessions. + +## Citing Renku in research papers + +If you use the Renku platform for your research, please do cite our [paper](https://proceedings.neurips.cc/paper_files/paper/2023/hash/838694e9ab6b0a193b84daaafcac0eed-Abstract-Datasets_and_Benchmarks.html). See the citation information in the side panel of this repo for APA and BibTex formats. diff --git a/helm-chart/renku/requirements.yaml b/helm-chart/renku/requirements.yaml index 649a7c7ad8..f5e045056b 100644 --- a/helm-chart/renku/requirements.yaml +++ b/helm-chart/renku/requirements.yaml @@ -4,7 +4,7 @@ dependencies: repository: "oci://registry-1.docker.io/bitnamicharts" condition: postgresql.enabled - name: keycloakx - version: 2.1.0 + version: 7.0.1 repository: "https://codecentric.github.io/helm-charts" condition: keycloakx.enabled - name: redis diff --git a/helm-chart/renku/templates/data-service/deployment_k8s_watcher.yaml b/helm-chart/renku/templates/data-service/deployment_k8s_watcher.yaml new file mode 100644 index 0000000000..29a343279e --- /dev/null +++ b/helm-chart/renku/templates/data-service/deployment_k8s_watcher.yaml @@ -0,0 +1,85 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "renku.fullname" . }}-k8s-watcher + labels: + app: renku-k8s-watcher + chart: {{ template "renku.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +spec: + replicas: 1 + strategy: + {{- toYaml .Values.dataService.updateStrategy | nindent 4 }} + selector: + matchLabels: + app: renku-k8s-watcher + release: {{ .Release.Name }} + template: + metadata: + labels: + app: renku-k8s-watcher + release: {{ .Release.Name }} + annotations: + {{- with .Values.dataService.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + automountServiceAccountToken: {{ .Values.global.debug }} + initContainers: + {{- include "certificates.initContainer" . | nindent 8 }} + containers: + - name: k8s-watcher + image: "{{ .Values.dataService.k8sWatcher.image.repository }}:{{ .Values.dataService.k8sWatcher.image.tag }}" + imagePullPolicy: {{ .Values.dataService.k8sWatcher.image.pullPolicy }} + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + env: + - name: VERSION + value: {{ .Values.dataService.image.tag | quote }} + - name: DB_HOST + value: {{ template "postgresql.fullname" . }} + - name: DB_USER + value: {{ .Values.global.db.common.username }} + - name: DB_PASSWORD + valueFrom: + secretKeyRef: + name: {{ .Values.global.db.common.passwordSecretName }} + key: password + - name: K8S_NAMESPACE + value: {{ .Release.Namespace | quote }} + - name: KUBERNETES_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + {{- include "certificates.env.python" . | nindent 12 }} + volumeMounts: + {{- include "certificates.volumeMounts.system" . | nindent 12 }} + livenessProbe: + exec: + command: + - cat + - /tmp/cache_ready + initialDelaySeconds: 10 + periodSeconds: 10 + failureThreshold: 6 + resources: + {{ toYaml .Values.dataService.k8sWatcher.resources | nindent 12 }} + {{- with .Values.dataService.nodeSelector }} + nodeSelector: + {{ toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.dataService.affinity }} + affinity: + {{ toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.dataService.tolerations }} + tolerations: + {{ toYaml . | nindent 8 }} + {{- end }} + volumes: + {{- include "certificates.volumes" . | nindent 8 }} + serviceAccountName: {{ template "renku.fullname" . }}-k8s-watcher diff --git a/helm-chart/renku/templates/data-service/rbac_k8s_watcher.yaml b/helm-chart/renku/templates/data-service/rbac_k8s_watcher.yaml new file mode 100644 index 0000000000..3a3141dfb0 --- /dev/null +++ b/helm-chart/renku/templates/data-service/rbac_k8s_watcher.yaml @@ -0,0 +1,78 @@ +{{- if .Values.dataService.rbac.create -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ template "renku.fullname" . }}-k8s-watcher + labels: + app: {{ template "renku.name" . }} + chart: {{ template "renku.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +rules: + - apiGroups: + - {{ .Values.amalthea.crdApiGroup }} + resources: + - {{ .Values.amalthea.crdNames.plural }} + verbs: + - create + - update + - delete + - patch + - list + - get + - watch + - apiGroups: + - amalthea.dev + resources: + - amaltheasessions + verbs: + - create + - update + - delete + - patch + - list + - get + - watch + {{- if .Values.dataService.imageBuilders.enabled }} + - apiGroups: + - shipwright.io + resources: + - buildruns + verbs: + - create + - update + - delete + - patch + - list + - get + - watch + {{- end }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "renku.fullname" . }}-k8s-watcher + labels: + app: {{ template "renku.name" . }} + chart: {{ template "renku.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ template "renku.fullname" . }}-k8s-watcher + labels: + app: {{ template "renku.name" . }} + chart: {{ template "renku.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ template "renku.fullname" . }}-k8s-watcher +subjects: + - kind: ServiceAccount + name: {{ template "renku.fullname" . }}-k8s-watcher + namespace: {{ .Release.Namespace }} +{{- end -}} diff --git a/helm-chart/renku/templates/keycloak-hostname-configmap.yaml b/helm-chart/renku/templates/keycloak-hostname-configmap.yaml new file mode 100644 index 0000000000..9b8b407a34 --- /dev/null +++ b/helm-chart/renku/templates/keycloak-hostname-configmap.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: keycloak-hostname + labels: + app: {{ template "renku.name" . }} + chart: {{ template "renku.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +data: + KC_HOSTNAME: "https://{{ .Values.global.renku.domain }}/auth" diff --git a/helm-chart/renku/templates/network-policies.yaml b/helm-chart/renku/templates/network-policies.yaml index 656b4390b3..9cd63a39e1 100644 --- a/helm-chart/renku/templates/network-policies.yaml +++ b/helm-chart/renku/templates/network-policies.yaml @@ -68,6 +68,12 @@ spec: namespaceSelector: matchLabels: kubernetes.io/metadata.name: {{ .Release.Namespace }} + - podSelector: + matchLabels: + app: renku-k8s-watcher + namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: {{ .Release.Namespace }} - podSelector: matchLabels: app: keycloak-sync diff --git a/helm-chart/renku/values.yaml b/helm-chart/renku/values.yaml index 663aa4ec3d..fd2738d641 100644 --- a/helm-chart/renku/values.yaml +++ b/helm-chart/renku/values.yaml @@ -72,7 +72,7 @@ global: fullnameOverride: "" image: repository: renku/renku-core - tag: "v2.9.2" + tag: "v2.9.4" pullPolicy: IfNotPresent uiserver: ## The client secret for the renku-ui client application registered in keycloak. @@ -256,9 +256,6 @@ keycloakx: - "start" - "--http-enabled=true" - "--http-port=8080" - - "--hostname-strict=false" - - "--hostname-strict-https=false" - - "--auto-build" # The following environment variables are provided to keycloak # as extraEnvFrom secrets. # renku-keycloak-postgres @@ -274,6 +271,8 @@ keycloakx: name: renku-keycloak-postgres - secretRef: name: keycloak-password-secret + - configMapRef: + name: keycloak-hostname extraVolumeMounts: | - name: theme mountPath: /opt/keycloak/themes/renku-theme @@ -318,7 +317,7 @@ keycloakx: enabled: false extraInitContainers: | - name: theme-provider - image: renku/keycloak-theme:4.1.5 + image: renku/keycloak-theme:fa8d7f3 imagePullPolicy: IfNotPresent command: - sh @@ -330,6 +329,12 @@ keycloakx: volumeMounts: - name: theme mountPath: /theme + securityContext: + runAsNonRoot: true + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL - name: init-certificates securityContext: allowPrivilegeEscalation: false @@ -587,10 +592,10 @@ ui: custom: true repositories: - url: https://github.com/SwissDataScienceCenter/renku-project-template - ref: 0.7.2 + ref: 0.9.0 name: Renku - url: https://github.com/SwissDataScienceCenter/contributed-project-templates - ref: 0.7.0 + ref: 0.10.0 name: Community # This defines the threshold for automatically showing a preview when browsing projects' files. # Above the soft limit, the user receives a warning. Above the hard limit, no preview is available. @@ -1510,17 +1515,23 @@ dataService: create: true image: repository: renku/renku-data-service - tag: "0.38.0" + tag: "0.39.0" pullPolicy: IfNotPresent backgroundJobs: events: resources: {} image: repository: renku/data-service-background-jobs - tag: "0.38.0" + tag: "0.39.0" pullPolicy: IfNotPresent total: resources: {} + k8sWatcher: + image: + repository: renku/data-service-k8s-watcher + tag: "0.39.0" + pullPolicy: IfNotPresent + resources: {} service: type: ClusterIP port: 80 @@ -1597,7 +1608,7 @@ authz: secretsStorage: image: repository: renku/secrets-storage - tag: "0.38.0" + tag: "0.39.0" pullPolicy: IfNotPresent service: type: ClusterIP @@ -1624,7 +1635,3 @@ securityContext: nodeSelector: {} tolerations: [] affinity: {} -versions: - latest: - image: - tag: v2.9.2 diff --git a/scripts/harbor-init/README.md b/scripts/harbor-init/README.md new file mode 100644 index 0000000000..fe65d57a4c --- /dev/null +++ b/scripts/harbor-init/README.md @@ -0,0 +1,34 @@ +# Harbor initialization for Renkulab script + +This script can be used to initialize a Harbor registry for usage with Renkulab: it will create a project, a robot account in that project that uses the specified secret for authentication. + +## Prerequisite + +1. A Harbor deployment +2. Admin credentials +3. Project name +4. Robot account name and secret +5. Access to the Renkulab namespace in Kubernetes + +## Usage + +With `go` installed, the script can be simply run: + +```bash +go run harbor-init.go --url https:// --admin --password --project --robot --secret +``` + +The script is idem-potent, so if the project or robot already exist the script will move on, additionally the robot's secret will always be updated (NOTE: it will overwrite the previous one!). + +## Kubernetes secret for Renkulab + +The secret for Renkulab can now be created so that container images build can be uploaded to the Harbor project. Using the and the Kubernetes secret can be created using `kubectl` as following: + +```bash +kubectl --namespace create secret docker-registry renku-build-docker-secret --docker-server --docker-username 'robot$+' --docker-password '' +``` + +Note: the username of the robot account is composed by harbor combining the project and its username, e.g. if the project is `foo` and the robot account `bar` then the resulting username would be `robot$foo+bar`. +To make sure that the terminal respects the special characters, the string should be surrounded by single quotes ''. + +The name of the secret should be matched in the Renku Helm chart values file under `dataService.imageBuilders.pushSecretName`. diff --git a/scripts/harbor-init/go.mod b/scripts/harbor-init/go.mod new file mode 100644 index 0000000000..eb403ce848 --- /dev/null +++ b/scripts/harbor-init/go.mod @@ -0,0 +1,3 @@ +module harbor-init + +go 1.21.4 diff --git a/scripts/harbor-init/harbor-init.go b/scripts/harbor-init/harbor-init.go new file mode 100644 index 0000000000..767082e810 --- /dev/null +++ b/scripts/harbor-init/harbor-init.go @@ -0,0 +1,270 @@ +package main + +import ( + "bytes" + "encoding/json" + "flag" + "fmt" + "io" + "log" + "net/http" + "os" + "strconv" + "strings" +) + +const ( + // Harbor API version + HARBOR_API_ENDPOINT = "/api/v2.0" +) + +type SensitiveString string + +func (s SensitiveString) String() string { + return "" +} + +func (s SensitiveString) Format(w fmt.State, v rune) { + _, err := w.Write([]byte(s.String())) + if err != nil { + panic(err) + } +} + +func (s *SensitiveString) Set(value string) error { + // Set the sensitive string value + *s = SensitiveString(value) + return nil +} + +type Login struct { + Client *http.Client + Url string + Username string + Password SensitiveString +} + +func (l *Login) String() string { + return l.Url + " " + l.Username + " " + "" +} + +func (l *Login) Format(w fmt.State, v rune) { + _, err := w.Write([]byte(l.String())) + if err != nil { + panic(err) + } +} + +type Projects []struct { + Name string `json:"name"` +} + +type ProjectRobots []struct { + Name string `json:"name"` + Id int `json:"id"` + Permissions []struct { + Namespace string `json:"namespace"` + } `json:"permissions"` +} + +type ResponseRobot struct { + Id int `json:"id"` +} + +func request(method string, login Login, endpoint string, body []byte) (*http.Response, []byte, error) { + req, err := http.NewRequest(method, login.Url+endpoint, bytes.NewBuffer(body)) + if err != nil { + return nil, nil, err + } + req.SetBasicAuth(login.Username, string(login.Password)) + req.Header.Set("Content-Type", "application/json") + + resp, err := login.Client.Do(req) + if err != nil { + return nil, nil, err + } + defer resp.Body.Close() + respBody, err := io.ReadAll(resp.Body) + if err != nil { + return nil, nil, err + } + return resp, respBody, nil +} + +func main() { + urlFlag := flag.String("url", "https://harbor.example.com", "Harbor URL") + adminFlag := flag.String("admin", "admin", "Harbor admin username") + var passwordFlag SensitiveString = "" + flag.Var(&passwordFlag, "password", "Harbor admin password") + projectNameFlag := flag.String("project", "renku-build", "Project name") + robotAccountNameFlag := flag.String("robot", "renku-registry-robot", "Robot account name") + var robotSecretFlag SensitiveString = "" + flag.Var(&robotSecretFlag, "secret", "Robot account secret") + flag.Parse() + // Initialize logger to print to stdout + logger := log.New(os.Stdout, "INFO: ", log.Ldate|log.Ltime) + login := Login{} + login.Client = &http.Client{} + // Get login from environment variable + if *urlFlag == "" { + logger.Fatal("url flag is not set") + } + login.Url = strings.TrimRight(*urlFlag, "/") + HARBOR_API_ENDPOINT + login.Username = *adminFlag + if login.Username == "" { + logger.Fatal("admin flag is not set") + } + login.Password = passwordFlag + if login.Password == "" { + logger.Fatal("password flag is not set") + } + projectName := *projectNameFlag + if projectName == "" { + logger.Fatal("project flag is not set") + } + robotAccountName := *robotAccountNameFlag + if robotAccountName == "" { + logger.Fatal("robot flag is not set") + } + robotSecret := robotSecretFlag + if robotSecret == "" { + logger.Fatal("secret flag is not set") + } + + // Authenticate and get a session cookie + logger.Println("Authenticating with Harbor server") + resp, body, err := request("GET", login, "/audit-logs", nil) + if err != nil { + logger.Fatal("Error connecting to Harbor: ", err) + } + if resp.StatusCode != http.StatusOK { + logger.Fatal("Failed to authenticate:", resp.Status, err, string(body)) + return + } + logger.Println("Authenticated successfully") + + // Check if project already exists + logger.Println("Checking if project exists:", projectName) + resp, body, err = request("GET", login, "/projects?name="+projectName, nil) + if err != nil { + logger.Fatal("Error getting project Harbor: ", err) + } + if resp.StatusCode != http.StatusOK { + logger.Println("Failed to get project:", err) + return + } + var projects Projects + err = json.Unmarshal(body, &projects) + if err != nil { + logger.Println("Error unmarshaling projects data:", err) + return + } + if len(projects) == 0 { + logger.Println("The project does not exist yet, creating: ", projectName) + project := map[string]interface{}{ + "project_name": projectName, + "public": false, + } + projectData, err := json.Marshal(project) + if err != nil { + logger.Fatal("Error marshaling project data:", err) + } + + resp, _, err = request("POST", login, "/projects", projectData) + if err != nil { + logger.Fatal("Error creating project: ", err) + } + if resp.StatusCode != http.StatusCreated { + logger.Fatal("Failed to create project:", resp.Status, err) + } + logger.Println("Project created successfully") + } else { + logger.Println("Project already exists") + } + + // Check if robot account already exists + logger.Println("Checking if robot account exists:", robotAccountName) + resp, body, err = request("GET", login, "/projects/"+projectName+"/robots", nil) + if err != nil { + logger.Fatal("Error getting robot account: ", err) + } + if resp.StatusCode != http.StatusOK { + logger.Fatal("Failed to get robot account:", err) + } + var robots ProjectRobots + err = json.Unmarshal(body, &robots) + if err != nil { + logger.Fatal("Error unmarshaling robots account data:", err) + } + robotId := "" + for _, robot := range robots { + if robot.Name != "robot$"+projectName+"+"+robotAccountName { + continue + } + for _, permission := range robot.Permissions { + if permission.Namespace != projectName { + continue + } + logger.Println("Robot account already exists") + robotId = strconv.Itoa(robot.Id) + break + } + } + + robotData := map[string]interface{}{} + if robotId == "" { + // Create a robot account + logger.Println("Creating robot account:", robotAccountName) + robotData = map[string]interface{}{ + "name": robotAccountName, + "permissions": []map[string]interface{}{ + { + "access": []map[string]string{ + {"resource": "repository", "action": "list"}, + {"resource": "repository", "action": "pull"}, + {"resource": "repository", "action": "push"}, + {"resource": "repository", "action": "read"}, + }, + "kind": "project", + "namespace": projectName, + }, + }, + "duration": -1, + "level": "project", + } + robotAccountData, err := json.Marshal(robotData) + if err != nil { + logger.Fatal("Error marshaling robot account data:", err) + } + resp, respBody, err := request("POST", login, "/robots/"+robotId, robotAccountData) + if err != nil { + logger.Fatal("Error creating robot account: ", err) + } + createdRobot := ResponseRobot{} + json.Unmarshal(respBody, &createdRobot) + robotId = strconv.Itoa(createdRobot.Id) + if resp.StatusCode != http.StatusCreated { + logger.Fatal("Failed to create robot account:", resp.Status, string(respBody), err) + } + } + + // Update the robot account secret + logger.Println("Setting the secret for the robot account:", robotAccountName) + robotData = map[string]interface{}{ + "secret": string(robotSecret), + } + robotAccountData, err := json.Marshal(robotData) + if err != nil { + logger.Fatal("Error marshaling robot account data:", err) + } + + resp, respBody, err := request("PATCH", login, "/robots/"+robotId, robotAccountData) + if err != nil { + logger.Fatal("Error updating robot secret: ", err) + } + if resp.StatusCode != http.StatusOK { + logger.Fatal("Failed to update the secret of the robot account:", resp.Status, string(respBody), err) + } + + logger.Println("Robot account secret updated successfully.") +} diff --git a/scripts/init-realm/init-realm.py b/scripts/init-realm/init-realm.py index 0e283cae9d..8f4dab1d8c 100644 --- a/scripts/init-realm/init-realm.py +++ b/scripts/init-realm/init-realm.py @@ -129,14 +129,18 @@ def _check_and_create_client(keycloak_admin, new_client: OIDCClient, force: bool keycloak_admin.delete_client(realm_client["id"]) created_client_id = keycloak_admin.create_client(new_client.to_dict()) - service_account_user = keycloak_admin.get_client_service_account_user(created_client_id) + logging.info(f"Created client {created_client_id}") - if isinstance(service_account_user, dict) and service_account_user.get("id"): + # if a client does not have a service account, calling get_client_service_account_user raises a KeycloakGetError + try: + service_account_user = keycloak_admin.get_client_service_account_user(created_client_id) logging.info(f"Reassigning service account roles {new_client.service_account_roles}") realm_management_roles = keycloak_admin.get_client_roles(realm_management_client_id) matching_roles = [{"name": role["name"], "id": role["id"]} for role in realm_management_roles if role["name"] in new_client.service_account_roles ] logging.info(f"Found and assigning matching roles: {matching_roles}") keycloak_admin.assign_client_role(service_account_user["id"], realm_management_client_id, matching_roles) + except KeycloakGetError: + logging.info(f"Client {created_client_id} does not use a service account.") logging.info("done")