diff --git a/.gitignore b/.gitignore index 7bb6000..2b42cca 100644 --- a/.gitignore +++ b/.gitignore @@ -13,4 +13,5 @@ __pycache__ **/.fuse* dev/.env .venv -.env \ No newline at end of file +.env +/deployment/dev/data/stroke \ No newline at end of file diff --git a/deployment/dev/.env.example b/deployment/dev/.env.example index 8456a6a..98e9698 100644 --- a/deployment/dev/.env.example +++ b/deployment/dev/.env.example @@ -1,8 +1,8 @@ # Versions EXAFLOW=1.0.0 -PLATFORM_BACKEND=9.1.1 -PLATFORM_UI=1.0.1 -MIP=9.0.2 +PLATFORM_BACKEND=9.1.2 +PLATFORM_UI=1.0.2 +MIP=9.0.3 # Toggle authentication AUTHENTICATION=0 diff --git a/deployment/dev/docker-compose.yml b/deployment/dev/docker-compose.yml index e3a037e..8528d5d 100644 --- a/deployment/dev/docker-compose.yml +++ b/deployment/dev/docker-compose.yml @@ -180,9 +180,6 @@ services: PLATFORM_BACKEND_SERVER: platform-backend:8080 PLATFORM_BACKEND_CONTEXT: services NOTEBOOK_ENABLED: 0 - FRONTEND_VERSION: "${PLATFORM_UI}" - BACKEND_VERSION: "${PLATFORM_BACKEND}" - EXAFLOW_VERSION: "${EXAFLOW}" MIP_VERSION: "${MIP}" depends_on: - platform-backend diff --git a/deployment/kubernetes/README.md b/deployment/kubernetes/README.md index 62b7935..8ef8dc3 100644 --- a/deployment/kubernetes/README.md +++ b/deployment/kubernetes/README.md @@ -6,7 +6,7 @@ MIP now supports two Kubernetes infrastructure options: 1. **VM-based / microk8s clusters** – the remainder of this document (starting in the Requirements section) walks through preparing Ubuntu virtual machines and installing the stack on top of microk8s. 2. **Managed clusters** – for cloud-managed Kubernetes (AKS/EKS/GKE, etc.) follow the [mip-infra getting started guide](https://github.com/Medical-Informatics-Platform/mip-infra?tab=readme-ov-file#-getting-started) to provision the cluster and its base services. Once the cluster is available, return here for component configuration details as needed. -Choose between these modes via the Helm values: set `managed_cluster: true` (managed) or `false` (microk8s/VM). The templates react to this flag to deploy the components with the right assumptions for networking, storage, and access. +Choose between these modes via the Helm values: set `cluster.managed: true` (managed) or `false` (microk8s/VM). The templates react to this flag to deploy the components with the right assumptions for networking, storage, and access. ## Requirements ### Hardware @@ -53,18 +53,16 @@ Afterward, The dataset CSV files should be placed in their proper pathology fold ## Configuration Prior to deploying it (on a microk8s K8s cluster of one or more nodes), there are a few adjustments to make in `values.yaml`. Each top-level section controls a part of the stack: -* `cluster`: namespace, storage classes and whether the cluster provisions persistent volumes dynamically (`managed: true`). -* `network`: public hostname, protocols, and whether the UI is exposed directly or through a reverse proxy (`link`). +* `cluster`: storage classes and whether the cluster provisions persistent volumes dynamically (`managed: true`). +* `global`: shared public hostname used by the ingress and backend redirects. * `platform-ui`, `platform-backend`, `platformBackendDatabase`: container images and component specific options (including the platform-ui ingress/tls settings). -* `keycloak`: toggles the connection parameters to the external Keycloak instance (`enabled`, `host`, `protocol`, `realm`, `clientId`). +* `keycloak`: toggles the connection parameters to the external Keycloak instance (`enabled`, `host`, `protocol`, `realm`). Copy `values.yaml` to a new file (for example `my-values.yaml`) and edit it in-place. A few important knobs: ```yaml -network: - link: proxied # use "direct" when exposing the UI publicly +global: publicHost: mip.example.org - publicProtocol: https platform-ui: backend: @@ -72,35 +70,23 @@ platform-ui: port: 8080 context: services ingress: - redirectRootTo: /home # optional 302 redirect for the landing page tlsSecretName: platform-ui-tls keycloak: enabled: true host: iam.example.org - protocol: https - realm: MIP - clientId: mipfed ``` -The reachability diagram from the legacy profiles is still valid as a reference for deciding the correct `network.*` settings: +The reachability diagram from the legacy profiles is still valid as a reference for deciding the correct public URL: ![MIP Reachability Scheme](../docs/MIP_Configuration.png) -### MACHINE_MAIN_IP -This is the machine's main IP address. Generally, it's the IP address of the first NIC after the local one. -If the MIP is running on top of a VPN, you may want to put the VPN interface's IP address. -If you reach the machine through a public IP, if this IP is **NOT** directly assigned on the machine, but is using static NAT, you still **MUST** set the **INTERNAL** IP of the machine itself! +`global.publicHost` defaults to `hbpmip.link`. Override it in your custom values file or with `--set-string global.publicHost=` whenever a deployment needs a different public hostname. -### MACHINE_PUBLIC_FQDN -This is the public, fully qualified domain name of the MIP, the main URL on which you want to reach the MIP from the Internet. This may point: -* Directly on the public IP of the MIP, for a **direct** use case. It may be assigned on the machine or used in front as a static NAT -* On the public IP of the reverse-proxy server, for a **proxied** use case +`global.publicHost` must be the bare hostname served by the ingress, without `http://` or `https://`. -### MACHINE_PRIVATE_FQDN_OR_IP -This is **ONLY** used in a **proxied** use case situation. -It's actually the internal IP or address from which the reverse-proxy server "sees" (reaches) the MIP machine. +`keycloak.protocol` defaults to `https` and `keycloak.realm` defaults to `MIP`. Override them only when your external Keycloak differs from those defaults. -These three settings map directly to the `network` section in `values.yaml` (`publicHost`, `link`, `publicProtocol`). When running behind a reverse proxy also set `externalProtocol` to describe the protocol used between the proxy and the MIP pods. +If you deploy behind a reverse proxy or load balancer, forward the standard `X-Forwarded-*` headers to the backend so Spring can reconstruct the public request URL correctly. **WARNING!**: In **ANY** case, when you use an **EXTERNAL** KeyCloak service (i.e. iam.ebrains.eu), make sure that you use the correct *CLIENT_ID* and *CLIENT_SECRET* to match the MIP instance you're deploying! @@ -185,6 +171,7 @@ For a more in-depth guide on deploying Exaflow, please refer to the documentatio * Deploy (or upgrade) the Helm release with your customised values ``` microk8s helm3 upgrade --install mip \ + --namespace \ -f /opt/mip-deployment/kubernetes/my-values.yaml \ /opt/mip-deployment/kubernetes ``` diff --git a/deployment/kubernetes/templates/_helpers.tpl b/deployment/kubernetes/templates/_helpers.tpl deleted file mode 100644 index d908586..0000000 --- a/deployment/kubernetes/templates/_helpers.tpl +++ /dev/null @@ -1,48 +0,0 @@ -{{- define "mip.namespace" -}} -{{- default "default" .Values.cluster.namespace -}} -{{- end -}} - -{{- define "mip.instanceName" -}} -{{- printf "%s %s" (default "MIP" .Values.mip.displayName) (default "latest" .Values.mip.version) -}} -{{- end -}} - -{{- define "mip.instanceVersion" -}} -{{- $platformUi := default "" (index .Values "platform-ui").image.tag -}} -{{- $platformBackend := default "" (index .Values "platform-backend").image.tag -}} -{{- $exaflow := default "" .Values.engines.exaflow.image.tag -}} -{{- printf "Platform-ui: %s, Backend: %s, Exaflow: %s" $platformUi $platformBackend $exaflow -}} -{{- end -}} - -{{- define "mip.keycloak.enabled" -}} -{{- ternary true false (default true .Values.keycloak.enabled) -}} -{{- end -}} - -{{- define "mip.keycloak.protocol" -}} -{{- default "https" .Values.keycloak.protocol -}} -{{- end -}} - -{{- define "mip.keycloak.host" -}} -{{- default "" .Values.keycloak.host -}} -{{- end -}} - -{{- define "mip.keycloak.realm" -}} -{{- default "MIP" .Values.keycloak.realm -}} -{{- end -}} - -{{- define "mip.keycloak.clientId" -}} -{{- default "" .Values.keycloak.clientId -}} -{{- end -}} - -{{- define "mip.keycloak.sslRequired" -}} -{{- default "external" .Values.keycloak.sslRequired -}} -{{- end -}} - -{{- define "mip.keycloak.authUrl" -}} -{{- $protocol := include "mip.keycloak.protocol" . -}} -{{- $host := include "mip.keycloak.host" . -}} -{{- if and $protocol $host -}} -{{- printf "%s://%s/auth/" $protocol $host -}} -{{- else -}} -{{- "" -}} -{{- end -}} -{{- end -}} diff --git a/deployment/kubernetes/templates/mip-config.yaml b/deployment/kubernetes/templates/mip-config.yaml deleted file mode 100644 index 1905752..0000000 --- a/deployment/kubernetes/templates/mip-config.yaml +++ /dev/null @@ -1,42 +0,0 @@ -{{- $namespace := include "mip.namespace" . -}} -{{- $platformUi := index .Values "platform-ui" -}} -{{- $platformBackend := index .Values "platform-backend" -}} -{{- $platformUiDatacatalogProtocol := default .Values.datacatalog.protocol $platformUi.config.datacatalog.protocol -}} -{{- $platformUiDatacatalogHost := default .Values.datacatalog.host $platformUi.config.datacatalog.host -}} -{{- $matomoEnabled := ternary "true" "false" (default false $platformUi.config.matomo.enabled) -}} -{{- $proxyForwarding := ternary "true" "false" (default true .Values.keycloak.proxyAddressForwarding) -}} -{{- $authEnabled := default true .Values.keycloak.enabled -}} -{{- $authFlag := ternary "1" "0" $authEnabled -}} -apiVersion: v1 -kind: ConfigMap -metadata: - name: mip-config - namespace: {{ $namespace }} -data: - engines.exaflow.URL: {{ default "" .Values.engines.exaflow.url | quote }} - - mip.INSTANCE_NAME: {{ include "mip.instanceName" . | quote }} - mip.INSTANCE_VERSION: {{ include "mip.instanceVersion" . | quote }} - mip.LINK: {{ .Values.network.link | quote }} - mip.EXTERNAL_PROTOCOL: {{ .Values.network.externalProtocol | quote }} - mip.PUBLIC_PROTOCOL: {{ .Values.network.publicProtocol | quote }} - mip.PUBLIC_HOST: {{ .Values.network.publicHost | quote }} - mip.PLATFORM_UI_URL: {{ printf "%s://%s" .Values.network.publicProtocol .Values.network.publicHost | quote }} - - platform-backend.LOG_LEVEL: {{ $platformBackend.config.logLevel | quote }} - platform-backend.LOG_LEVEL_FRAMEWORK: {{ $platformBackend.config.logLevelFramework | quote }} - platform-backend.ALGORITHM_UPDATE_INTERVAL: {{ $platformBackend.config.algorithmUpdateInterval | quote }} - - platform-ui.ERROR_LOG_LEVEL: {{ $platformUi.config.errorLogLevel | quote }} - platform-ui.DATACATALOG_SERVER: {{ printf "%s://%s" $platformUiDatacatalogProtocol $platformUiDatacatalogHost | quote }} - platform-ui.MATOMO_ENABLED: {{ $matomoEnabled | quote }} - platform-ui.MATOMO_URL: {{ default "" $platformUi.config.matomo.url | quote }} - platform-ui.MATOMO_SITE_ID: {{ default "" $platformUi.config.matomo.siteId | quote }} - - keycloak.AUTHENTICATION: {{ $authFlag | quote }} - keycloak.HOSTNAME: {{ include "mip.keycloak.host" . | quote }} - keycloak.AUTH_URL: {{ include "mip.keycloak.authUrl" . | quote }} - keycloak.REALM: {{ include "mip.keycloak.realm" . | quote }} - keycloak.CLIENT_ID: {{ include "mip.keycloak.clientId" . | quote }} - keycloak.SSL_REQUIRED: {{ include "mip.keycloak.sslRequired" . | quote }} - keycloak.PROXY_ADDRESS_FORWARDING: {{ $proxyForwarding | quote }} diff --git a/deployment/kubernetes/templates/platform-backend.yaml b/deployment/kubernetes/templates/platform-backend.yaml index 7c9791f..752118a 100644 --- a/deployment/kubernetes/templates/platform-backend.yaml +++ b/deployment/kubernetes/templates/platform-backend.yaml @@ -1,5 +1,19 @@ -{{- $namespace := include "mip.namespace" . -}} +{{- $namespace := default "default" .Release.Namespace -}} {{- $platformBackend := index .Values "platform-backend" -}} +{{- $authEnabled := default true .Values.keycloak.enabled -}} +{{- $authFlag := ternary "1" "0" $authEnabled -}} +{{- $publicHost := default "" .Values.global.publicHost -}} +{{- $platformUiUrl := "" -}} +{{- if $publicHost -}} +{{- $platformUiUrl = printf "https://%s" $publicHost -}} +{{- end -}} +{{- $keycloakProtocol := default "https" .Values.keycloak.protocol -}} +{{- $keycloakHost := default "" .Values.keycloak.host -}} +{{- $keycloakRealm := default "MIP" .Values.keycloak.realm -}} +{{- $keycloakAuthUrl := "" -}} +{{- if and $keycloakProtocol $keycloakHost -}} +{{- $keycloakAuthUrl = printf "%s://%s/auth/" $keycloakProtocol $keycloakHost -}} +{{- end -}} {{- $localStorageClass := .Values.cluster.storageClasses.local -}} {{- $managedStorageClass := .Values.cluster.storageClasses.managed -}} {{- if not .Values.cluster.managed }} @@ -302,20 +316,11 @@ spec: mountPath: /var/log/platform-backend env: - name: LOG_LEVEL - valueFrom: - configMapKeyRef: - name: mip-config - key: platform-backend.LOG_LEVEL + value: {{ $platformBackend.config.logLevel | quote }} - name: LOG_LEVEL_FRAMEWORK - valueFrom: - configMapKeyRef: - name: mip-config - key: platform-backend.LOG_LEVEL_FRAMEWORK + value: {{ $platformBackend.config.logLevelFramework | quote }} - name: AUTHENTICATION - valueFrom: - configMapKeyRef: - name: mip-config - key: keycloak.AUTHENTICATION + value: {{ $authFlag | quote }} - name: PLATFORM_DB_URL value: jdbc:postgresql://localhost:5432/portal - name: PLATFORM_DB_SERVER @@ -331,25 +336,13 @@ spec: name: mip-secret key: platform-backend-db.PLATFORM_DB_PASSWORD - name: EXAFLOW_URL - valueFrom: - configMapKeyRef: - name: mip-config - key: engines.exaflow.URL + value: {{ default "" .Values.engines.exaflow.url | quote }} - name: PLATFORM_UI_BASE_URL - valueFrom: - configMapKeyRef: - name: mip-config - key: mip.PLATFORM_UI_URL + value: {{ $platformUiUrl | quote }} - name: KEYCLOAK_AUTH_URL - valueFrom: - configMapKeyRef: - name: mip-config - key: keycloak.AUTH_URL + value: {{ $keycloakAuthUrl | quote }} - name: KEYCLOAK_REALM - valueFrom: - configMapKeyRef: - name: mip-config - key: keycloak.REALM + value: {{ $keycloakRealm | quote }} - name: KEYCLOAK_CLIENT_ID valueFrom: secretKeyRef: @@ -360,16 +353,8 @@ spec: secretKeyRef: name: keycloak-credentials key: client-secret - # - name: KEYCLOAK_SSL_REQUIRED - # valueFrom: - # configMapKeyRef: - # name: mip-config - # key: keycloak.SSL_REQUIRED - name: ALGORITHM_UPDATE_INTERVAL - valueFrom: - configMapKeyRef: - name: mip-config - key: platform-backend.ALGORITHM_UPDATE_INTERVAL + value: {{ $platformBackend.config.algorithmUpdateInterval | quote }} --- apiVersion: v1 diff --git a/deployment/kubernetes/templates/platform-ui.yaml b/deployment/kubernetes/templates/platform-ui.yaml index c7dd3af..429faf0 100644 --- a/deployment/kubernetes/templates/platform-ui.yaml +++ b/deployment/kubernetes/templates/platform-ui.yaml @@ -1,15 +1,12 @@ -{{- $namespace := include "mip.namespace" . -}} +{{- $namespace := default "default" .Release.Namespace -}} {{- $platformUi := index .Values "platform-ui" -}} {{- $platformBackend := index .Values "platform-backend" -}} {{- $ingress := default (dict) $platformUi.ingress -}} {{- $backendHost := default "" $platformUi.backend.host -}} {{- $backendPort := default 8080 $platformUi.backend.port -}} {{- $backendContext := default "services" $platformUi.backend.context -}} -{{- $ingressEnabled := true -}} -{{- if hasKey $ingress "enabled" -}} -{{- $ingressEnabled = $ingress.enabled -}} -{{- end -}} -{{- $publicHost := default "" .Values.network.publicHost -}} +{{- $ingressEnabled := default true $ingress.enabled -}} +{{- $publicHost := default "" .Values.global.publicHost -}} --- apiVersion: apps/v1 kind: Deployment @@ -32,12 +29,6 @@ spec: ports: - containerPort: 80 env: - - name: FRONTEND_VERSION - value: {{ default "" $platformUi.image.tag | quote }} - - name: BACKEND_VERSION - value: {{ default "" $platformBackend.image.tag | quote }} - - name: EXAFLOW_VERSION - value: {{ default "" .Values.engines.exaflow.image.tag | quote }} - name: MIP_VERSION value: {{ default "" .Values.mip.version | quote }} {{- if $backendHost }} @@ -73,7 +64,6 @@ metadata: name: platform-ui-ingress namespace: {{ $namespace }} annotations: - kubernetes.io/ingress.class: {{ default "nginx" $ingress.className | quote }} {{- if $ingress.certManagerClusterIssuer }} cert-manager.io/cluster-issuer: {{ $ingress.certManagerClusterIssuer | quote }} {{- end }} diff --git a/deployment/kubernetes/values.yaml b/deployment/kubernetes/values.yaml index 6923a64..601e188 100644 --- a/deployment/kubernetes/values.yaml +++ b/deployment/kubernetes/values.yaml @@ -1,56 +1,35 @@ cluster: - namespace: default managed: true storageClasses: local: k8s-local-storage managed: ceph-corbo-cephfs -network: - externalIP: "" - link: proxied - externalProtocol: https - publicProtocol: https - publicHost: default +global: + publicHost: hbpmip.link engines: exaflow: url: "http://exaflow-controller-service:5000" - image: - repository: madgik/exaflow - tag: 1.0.0 mip: - displayName: MIP - version: 9.0.2 -datacatalog: - protocol: https - host: "" + version: 9.0.3 platform-ui: image: repository: hbpmip/platform-ui - tag: 1.0.1 + tag: 1.0.2 service: nodePort: 30080 ingress: enabled: true - className: nginx-public + className: haproxy-public certManagerClusterIssuer: letsencrypt-public tlsSecretName: platform-ui-tls - redirectRootTo: /home + redirectRootTo: '' backend: host: platform-backend-service port: 8080 context: services - config: - errorLogLevel: ERROR - datacatalog: - protocol: https - host: "" - matomo: - enabled: false - url: "" - siteId: "" platform-backend: image: repository: hbpmip/platform-backend - tag: 9.1.1 + tag: 9.1.2 config: logLevel: INFO logLevelFramework: INFO @@ -74,8 +53,3 @@ platformBackendDatabase: keycloak: enabled: true host: iam.ebrains.eu - protocol: https - realm: MIP - clientId: mipfedqa - sslRequired: external - proxyAddressForwarding: true