From e14779058fd814de41500dc9a7af7d73b3e2cdf9 Mon Sep 17 00:00:00 2001 From: Bruno Salzano Date: Sun, 10 Aug 2025 07:03:10 +0200 Subject: [PATCH 1/4] chore: updated olaris-api templates updated olaris-api templates using the fixed versions added to openserverless-task --- olaris-api/api-template.yaml | 7 +------ olaris-api/nginx-template.yaml | 9 ++------- olaris-api/opsfile.yml | 2 +- olaris-api/{k3s-template.yaml => traefik-template.yaml} | 6 ------ 4 files changed, 4 insertions(+), 20 deletions(-) rename olaris-api/{k3s-template.yaml => traefik-template.yaml} (94%) diff --git a/olaris-api/api-template.yaml b/olaris-api/api-template.yaml index 582019b..ced1468 100644 --- a/olaris-api/api-template.yaml +++ b/olaris-api/api-template.yaml @@ -1,6 +1,4 @@ --- -# Source: openwhisk/templates/couchdb-init-cm.yaml -# # Licensed to the Apache Software Foundation (ASF) under one or more # contributor license agreements. See the NOTICE file distributed with # this work for additional information regarding copyright ownership. @@ -15,9 +13,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# -# Kind deployment example -# --- apiVersion: apps/v1 kind: StatefulSet @@ -41,7 +36,7 @@ spec: containers: - name: nuvolaris-system-api image: ${IMAGES_SYSTEMAPI} - imagePullPolicy: IfNotPresent + imagePullPolicy: Always command: ["./run.sh"] ports: - containerPort: 5000 diff --git a/olaris-api/nginx-template.yaml b/olaris-api/nginx-template.yaml index 2255702..7a8c986 100644 --- a/olaris-api/nginx-template.yaml +++ b/olaris-api/nginx-template.yaml @@ -1,5 +1,3 @@ -# Source: openwhisk/templates/couchdb-init-cm.yaml -# # Licensed to the Apache Software Foundation (ASF) under one or more # contributor license agreements. See the NOTICE file distributed with # this work for additional information regarding copyright ownership. @@ -14,9 +12,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# -# Kind deployment example -# --- apiVersion: networking.k8s.io/v1 kind: Ingress @@ -30,9 +25,9 @@ metadata: name: nuvolaris-system-api-ingress namespace: nuvolaris spec: + ingressClassName: nginx rules: - - # - host: ${SYS_API_HOSTNAME:-localhost} + - host: ${SYS_API_HOSTNAME:-localhost} http: paths: - backend: diff --git a/olaris-api/opsfile.yml b/olaris-api/opsfile.yml index e0c9902..68ce3ae 100644 --- a/olaris-api/opsfile.yml +++ b/olaris-api/opsfile.yml @@ -69,7 +69,7 @@ tasks: desc: undeploy the admin api ignore_error: true cmds: - - kubectl -n nuvolaris delete sts/nuvolaris-system-api ing/nuvolaris-system-api-ingress + - kubectl -n nuvolaris delete sts/nuvolaris-system-api ing/nuvolaris-system-api-ingress svc/nuvolaris-system-api - | echo "System API undeployed" diff --git a/olaris-api/k3s-template.yaml b/olaris-api/traefik-template.yaml similarity index 94% rename from olaris-api/k3s-template.yaml rename to olaris-api/traefik-template.yaml index 8950f25..dff4420 100644 --- a/olaris-api/k3s-template.yaml +++ b/olaris-api/traefik-template.yaml @@ -1,6 +1,3 @@ ---- -# Source: openwhisk/templates/couchdb-init-cm.yaml -# # Licensed to the Apache Software Foundation (ASF) under one or more # contributor license agreements. See the NOTICE file distributed with # this work for additional information regarding copyright ownership. @@ -15,9 +12,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# -# Kind deployment example -# --- apiVersion: networking.k8s.io/v1 kind: Ingress From 2e3c00d08d044b72a39c96c8d033cff9b7f088e4 Mon Sep 17 00:00:00 2001 From: Bruno Salzano Date: Fri, 15 Aug 2025 00:07:00 +0200 Subject: [PATCH 2/4] deployer (tbc) --- .env.example | 8 +- README.md | 39 ++- Taskfile.yml | 36 +-- TaskfileBuilder.yml | 106 +++++++ TaskfileDev.yml | 50 ++++ deploy/buildkit/buildkitd.toml | 24 ++ deploy/samples/requirements.txt | 2 + docs/DEPLOYER.md | 60 ++++ openserverless/common/kube_api_client.py | 269 +++++++++++++++++- openserverless/common/openwhisk_authorize.py | 1 + openserverless/common/utils.py | 45 +++ openserverless/impl/auth/auth_service.py | 85 ++++-- openserverless/impl/builder/build_service.py | 269 ++++++++++++++++++ .../impl/onboard/user_validation.py | 25 +- openserverless/rest/api.py | 63 ++++ 15 files changed, 1002 insertions(+), 80 deletions(-) create mode 100644 TaskfileBuilder.yml create mode 100644 TaskfileDev.yml create mode 100644 deploy/buildkit/buildkitd.toml create mode 100644 deploy/samples/requirements.txt create mode 100644 docs/DEPLOYER.md create mode 100644 openserverless/common/utils.py create mode 100644 openserverless/impl/builder/build_service.py diff --git a/.env.example b/.env.example index 341218c..1b9843f 100644 --- a/.env.example +++ b/.env.example @@ -12,4 +12,10 @@ KUBERNETES_CERT_FILENAME=./tokens/ca.crt # if not set, the image is build with local tag and will not be pushed REGISTRY= # namespace is required if REGISTRY is set -NAMESPACE= \ No newline at end of file +NAMESPACE= + +REGISTRY_HOST=http://127.0.0.1:5000 +REGISTRY_USER=opsuser +REGISTRY_PASS=password + +ADMIN_API_URL=http://127.0.0.1:5002 \ No newline at end of file diff --git a/README.md b/README.md index affd74f..b3d4714 100644 --- a/README.md +++ b/README.md @@ -34,18 +34,23 @@ Available APIs at the moment: `PATCH /system/api/v1/auth/{login}` - Update the user password patching the corresponding wsku/\ entry. +### Build API + +`POST /system/api/build` - Perform the build of a custom image and push it to repository. + +More informations [Here](docs/DEPLOYER.md) + ### Info API `GET /system/info` - Info endpoint - ## Developer instructions You need to have access to be Apache OpenServerless admin and have access to kubernetes cluster. Refer to the [Apache OpenServerless installation page](https://openserverless.apache.org/docs/installation/install/docker/): -Give the command `task setup-developer` and it will: +Give the command `task dev:setup-developer` and it will: - extract the required ca.crt and token from operator service account - copy a sample .env file @@ -61,14 +66,22 @@ Open http://localhost:5002/system/apidocs/ to see the API documentation. Taskfile supports the following tasks: ```yaml -* build: Build the image locally -* build-and-load: Build the image and loads it to local Kind cluster -* buildx: Build the docker image using buildx. Set PUSH=1 to push the image to the registry. -* docker-login: Login to the docker registry. Set REGISTRY=ghcr or REGISTRY=dockerhub in .env to use the respective registry. -* get-tokens: Get Service Account tokens and save them to tokens directory -* image-tag: Create a new tag for the current git commit. -* run: Run the admin api locally, using configuration from .env file -* setup-developer: Setup developer environment +* build: Build the image locally +* build-and-load: Build the image and loads it to local Kind cluster +* buildx: Build the docker image using buildx. Set PUSH=1 to push the image to the registry. +* docker-login: Login to the docker registry. Set REGISTRY=ghcr or REGISTRY=dockerhub in .env to use the respective registry. +* image-tag: Create a new tag for the current git commit. +* builder:cleanjobs: Clean up old jobs +* builder:delete-image: Delete an image from the registry +* builder:get-image: Get an image from the registry +* builder:list-catalogs: List catalogs in the registry +* builder:list-images: List images in a specific catalog +* builder:logs: Show logs of the last build job +* builder:send: Send the build to the server +* builder:updatetoml: Update the buildkitd.toml file config map +* dev:get-tokens: Get Service Account tokens and save them to tokens directory +* dev:run: Run the admin api locally, using configuration from .env file +* dev:setup-developer: Setup developer environment ``` ## Build and push @@ -128,4 +141,8 @@ $ git push apache 0.1.0-incubating.2507270910 ``` This will trigger the build workflow, and the process will be visible at -https://github.com/apache/openserverless-admin-api/actions \ No newline at end of file +https://github.com/apache/openserverless-admin-api/actions + +## Additional Documentation + +- [Deployer](docs/DEPLOYER.md) \ No newline at end of file diff --git a/Taskfile.yml b/Taskfile.yml index cbb9d02..260649c 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -27,32 +27,14 @@ vars: dotenv: - .env - -tasks: - get-tokens: - desc: "Get Service Account tokens and save them to tokens directory" - silent: true - cmds: - - mkdir -p tokens - - kubectl get secret nuvolaris-wsku-secret -o jsonpath='{.data.token}' | base64 --decode > tokens/token - - kubectl get secret nuvolaris-wsku-secret -o jsonpath='{.data.ca\.crt}' | base64 --decode > tokens/ca.crt +includes: + builder: + taskfile: TaskfileBuilder.yml + dev: + taskfile: TaskfileDev.yml - setup-developer: - desc: "Setup developer environment" - silent: true - cmds: - - task: get-tokens - - | - if [ ! -f .env ]; - then cp .env.example .env - echo "Please edit .env file with your local CouchDB and Kubernetes credentials" - fi - - | - if [ ! -d .venv ]; - then uv venv - fi - - uv pip install -r pyproject.toml 2>/dev/null +tasks: docker-login-ghcr: > silent: true @@ -144,9 +126,3 @@ tasks: BASEIMG=$(task base-image-name) IMG="$BASEIMG:{{.TAG}}" kind load docker-image $IMG --name=nuvolaris - - run: - desc: | - Run the admin api locally, using configuration from .env file - cmds: - - uv run -m openserverless \ No newline at end of file diff --git a/TaskfileBuilder.yml b/TaskfileBuilder.yml new file mode 100644 index 0000000..0f24475 --- /dev/null +++ b/TaskfileBuilder.yml @@ -0,0 +1,106 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +version: '3' + +tasks: + + send: + desc: Send the build to the server + vars: + AUTH: + sh: cat ~/.wskprops | grep "AUTH" | cut -d'=' -f2 | xargs -I {} + cmds: + - if test -z "{{.SOURCE}}"; then echo "SOURCE IS NOT SET" && exit 1; fi + - if test -z "{{.TARGET}}"; then echo "TARGET IS NOT SET" && exit 1; fi + - if test -z "{{.KIND}}"; then echo "KIND IS NOT SET" && exit 1; fi + - | + echo '{"source": "{{.SOURCE}}", "target": "{{.TARGET}}", "kind": "{{.KIND}}", "file": "{{.REQUIREMENTS}}" }' | http POST $ADMIN_API_URL/system/build Content-Type:application/json Authorization:"{{.AUTH}}" + - sleep 5 + - task: logs + deps: + - cleanjobs + # - updatetoml + silent: true + + logs: + desc: Show logs of the last build job + cmds: + - kubectl -n nuvolaris logs $(kubectl get jobs.batch -o name | grep "build-") -c buildkit --follow + silent: false + + cleanjobs: + desc: Clean up old jobs + cmds: + - for I in $(kubectl get jobs -n nuvolaris | grep build | awk '{ print $1 }' | tr "\n" " "); do kubectl delete job $I; done + - for I in $(kubectl get cm -n nuvolaris | grep "cm-" | awk '{ print $1 }' | tr "\n" " "); do kubectl delete cm $I; done + silent: true + + updatetoml: + desc: Update the buildkitd.toml file config map + cmds: + - | + if test $(kubectl -n nuvolaris get cm -o name | grep nuvolaris-buildkitd-conf | wc -l) -gt 0; + then kubectl -n nuvolaris delete configmap nuvolaris-buildkitd-conf + fi + - kubectl -n nuvolaris create configmap nuvolaris-buildkitd-conf --from-file=deploy/buildkit/buildkitd.toml + silent: true + + list-catalogs: + desc: List catalogs in the registry + cmds: + - http -a $REGISTRY_USER:$REGISTRY_PASS GET "${REGISTRY_HOST}/v2/_catalog" + silent: false + + list-images: + desc: List images in a specific catalog + vars: + CATALOG: '{{.CATALOG}}' + cmds: + - if test -z "{{.CATALOG}}"; then echo "CATALOG IS NOT SET" && exit 1; fi + - http -a $REGISTRY_USER:$REGISTRY_PASS GET "${REGISTRY_HOST}/v2/{{.CATALOG}}/tags/list" + silent: false + + get-image: + desc: Get an image from the registry + vars: + IMAGE: '{{.IMAGE}}' + IMAGE_NAME: + sh: echo '{{.IMAGE}}' | cut -d':' -f1 + HASH: + sh: echo '{{.IMAGE}}' | cut -d':' -f2 + cmds: + - echo "Getting image {{.IMAGE_NAME}} with hash {{.HASH}}" + - http -a $REGISTRY_USER:$REGISTRY_PASS GET "${REGISTRY_HOST}/v2/{{.IMAGE_NAME}}/manifests/{{.HASH}}" + silent: false + + delete-image: + desc: Delete an image from the registry + vars: + IMAGE: '{{.IMAGE}}' + IMAGE_NAME: + sh: echo '{{.IMAGE}}' | cut -d':' -f1 + HASH: + sh: echo '{{.IMAGE}}' | cut -d':' -f2 + MANIFEST_DIGEST: + sh: http --headers -a $REGISTRY_USER:$REGISTRY_PASS GET "${REGISTRY_HOST}/v2/{{.IMAGE_NAME}}/manifests/{{.HASH}}" | grep -i 'Docker-Content-Digest:' | awk '{print $2}' | tr -d '\r' + cmds: + - echo 'Deleting image {{.IMAGE}}' + - echo "Deleting manifest {{.MANIFEST_DIGEST}} for image {{.IMAGE_NAME}}" + - http -a $REGISTRY_USER:$REGISTRY_PASS DELETE "${REGISTRY_HOST}/v2/{{.IMAGE_NAME}}/manifests/{{.MANIFEST_DIGEST}}" + silent: false \ No newline at end of file diff --git a/TaskfileDev.yml b/TaskfileDev.yml new file mode 100644 index 0000000..27da031 --- /dev/null +++ b/TaskfileDev.yml @@ -0,0 +1,50 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +version: '3' + +tasks: + get-tokens: + desc: "Get Service Account tokens and save them to tokens directory" + silent: true + cmds: + - mkdir -p tokens + - kubectl get secret nuvolaris-wsku-secret -o jsonpath='{.data.token}' | base64 --decode > tokens/token + - kubectl get secret nuvolaris-wsku-secret -o jsonpath='{.data.ca\.crt}' | base64 --decode > tokens/ca.crt + + setup-developer: + desc: "Setup developer environment" + silent: true + cmds: + - task: get-tokens + - | + if [ ! -f .env ]; + then cp .env.example .env + echo "Please edit .env file with your local CouchDB and Kubernetes credentials" + fi + - | + if [ ! -d .venv ]; + then uv venv + fi + - uv pip install -r pyproject.toml 2>/dev/null + + run: + desc: | + Run the admin api locally, using configuration from .env file + cmds: + - uv run -m openserverless \ No newline at end of file diff --git a/deploy/buildkit/buildkitd.toml b/deploy/buildkit/buildkitd.toml new file mode 100644 index 0000000..772e30e --- /dev/null +++ b/deploy/buildkit/buildkitd.toml @@ -0,0 +1,24 @@ +# ========================= +# Worker OCI (rootlesskit) +# ========================= +[worker.oci] + enabled = true + rootless = true + no-process-sandbox = true + snapshotter = "overlayfs" # usa overlayfs se il kernel lo consente + +[worker.containerd] + enabled = false + +# ========================= +# Registry HTTP insicuro +# ========================= +[registry."nuvolaris-registry-svc:5000"] + insecure = true + http = true + +# ========================= +# Logging +# ========================= +[log] + level = "debug" \ No newline at end of file diff --git a/deploy/samples/requirements.txt b/deploy/samples/requirements.txt new file mode 100644 index 0000000..4b48d6f --- /dev/null +++ b/deploy/samples/requirements.txt @@ -0,0 +1,2 @@ +gnews +beautifulsoup4 \ No newline at end of file diff --git a/docs/DEPLOYER.md b/docs/DEPLOYER.md new file mode 100644 index 0000000..6f4b9c1 --- /dev/null +++ b/docs/DEPLOYER.md @@ -0,0 +1,60 @@ + +# Deployer + +These tasks are useful to interact with OpenServerless Admin Api Builder + +There are some tasks to interact with OpenServerless internal registry too. + +## Available tasks + +task: Available tasks for this project: + +``` +* builder:cleanjobs: Clean up old jobs +* builder:delete-image: Delete an image from the registry +* builder:get-image: Get an image from the registry +* builder:list-catalogs: List catalogs in the registry +* builder:list-images: List images in a specific catalog +* builder:logs: Show logs of the last build job +* builder:send: Send the build to the server +* builder:updatetoml: Update the buildkitd.toml file config map +``` + +## Examples + +### Build a custom runtime + +`task builder:send SOURCE=apache/openserverless-runtime-python:v3.13-2506091954 TARGET=devel:python3.13-custom KIND=python REQUIREMENTS=$(base64 -i deploy/samples/requirements.txt)` + +### List images for the user + +`task builder:list-images CATALOG=devel` + +### Delete an image for the user + +`task builder:delete-image IMAGE=devel:alpine` + +# Useful Links + +- https://crazymax.dev/buildkit/user-guides/rootless-mode/ +- https://www.linkedin.com/pulse/kubernetes-v133-user-namespaces-revolutionizing-false-rodrigo-mqoif/ +- https://chatgpt.com/c/689c9b5b-1d3c-8333-9f25-19d016fdacd0 +- https://kubernetes.io/docs/concepts/workloads/pods/user-namespaces/ \ No newline at end of file diff --git a/openserverless/common/kube_api_client.py b/openserverless/common/kube_api_client.py index 097fa2d..46e041f 100644 --- a/openserverless/common/kube_api_client.py +++ b/openserverless/common/kube_api_client.py @@ -15,12 +15,13 @@ # specific language governing permissions and limitations # under the License. # +from datetime import time import requests as req import json import os import logging -from base64 import b64decode +from base64 import b64decode, b64encode from .validation import is_empty_arg from openserverless.config.app_config import AppConfig @@ -223,3 +224,269 @@ def update_whisk_user(self, whisk_user_dict, namespace="nuvolaris"): except Exception as ex: logging.error(f"update_whisk_user {ex}") return False + + def get_config_map(self, cm_name, namespace="nuvolaris"): + """ + Get a ConfigMap by name. + :param cm_name: Name of the ConfigMap. + :param namespace: Namespace where the ConfigMap is located. + :return: The ConfigMap data or None if not found. + """ + url = f"{self.host}/api/v1/namespaces/{namespace}/configmaps/{cm_name}" + headers = {"Authorization": self.token} + + try: + logging.info(f"GET request to {url}") + response = req.get(url, headers=headers, verify=self.ssl_ca_cert) + + if response.status_code == 200: + logging.debug( + f"GET to {url} succeeded with {response.status_code}. Body {response.text}" + ) + return json.loads(response.text) + + logging.error( + f"GET to {url} failed with {response.status_code}. Body {response.text}" + ) + return None + except Exception as ex: + logging.error(f"get_config_map {ex}") + return None + + def post_config_map(self, cm_name, file_or_dir, namespace="nuvolaris"): + + if not os.path.exists(file_or_dir): + raise ConfigException(f"File or directory {file_or_dir} does not exist.") + + configmap_data = {} + if os.path.isfile(file_or_dir): + with open(file_or_dir, "r") as f: + configmap_data[os.path.basename(file_or_dir)] = f.read() + elif os.path.isdir(file_or_dir): + for filename in os.listdir(file_or_dir): + filepath = os.path.join(file_or_dir, filename) + if os.path.isfile(filepath): + with open(filepath, "r") as f: + configmap_data[filename] = f.read() + + configmap_manifest = { + "apiVersion": "v1", + "kind": "ConfigMap", + "metadata": { + "name": cm_name + }, + "data": configmap_data + } + + url = f"{self.host}/api/v1/namespaces/{namespace}/configmaps" + headers = {"Authorization": self.token, "Content-Type": "application/json"} + + try: + logging.info(f"POST request to {url}") + response = None + response = req.post(url, data=json.dumps(configmap_manifest), headers=headers, verify=self.ssl_ca_cert) + + if response.status_code in [200, 201, 202]: + logging.debug( + f"POST to {url} succeeded with {response.status_code}. Body {response.text}" + ) + return json.loads(response.text) + + logging.error( + f"POST to {url} failed with {response.status_code}. Body {response.text}" + ) + return None + except Exception as ex: + logging.error(f"post_config_map {ex}") + return None + + def delete_config_map(self, cm_name, namespace="nuvolaris"): + url = f"{self.host}/api/v1/namespaces/{namespace}/configmaps/{cm_name}" + headers = {"Authorization": self.token} + + try: + logging.info(f"DELETE request to {url}") + response = None + response = req.delete(url, headers=headers, verify=self.ssl_ca_cert) + + if response.status_code in [200, 202]: + logging.debug( + f"DELETE to {url} succeeded with {response.status_code}. Body {response.text}" + ) + return True + + logging.error( + f"DELETE to {url} failed with {response.status_code}. Body {response.text}" + ) + return False + except Exception as ex: + logging.error(f"delete_config_map {ex}") + return False + + def get_secret(self, secret_name, namespace="nuvolaris"): + """ + Get a Kubernetes secret by name. + :param secret_name: Name of the secret. + :param namespace: Namespace where the secret is located. + :return: The secret data or None if not found. + """ + url = f"{self.host}/api/v1/namespaces/{namespace}/secrets/{secret_name}" + headers = {"Authorization": self.token} + + try: + logging.info(f"GET request to {url}") + response = req.get(url, headers=headers, verify=self.ssl_ca_cert) + + if response.status_code == 200: + logging.debug( + f"GET to {url} succeeded with {response.status_code}. Body {response.text}" + ) + return json.loads(response.text) + + logging.error( + f"GET to {url} failed with {response.status_code}. Body {response.text}" + ) + return None + except Exception as ex: + logging.error(f"get_secret {ex}") + return None + + def post_secret(self, secret_name, secret_data, namespace="nuvolaris"): + """ + Create a Kubernetes secret. + :param secret_name: Name of the secret. + :param secret_data: Dictionary containing the secret data. + :param namespace: Namespace where the secret will be created. + :return: The created secret or None if failed. + """ + url = f"{self.host}/api/v1/namespaces/{namespace}/secrets" + headers = {"Authorization": self.token, "Content-Type": "application/json"} + + secret_manifest = { + "apiVersion": "v1", + "kind": "Secret", + "metadata": {"name": secret_name}, + "data": {k: b64encode(v.encode()).decode() for k, v in secret_data.items()}, + "type": "Opaque" + } + + try: + logging.info(f"POST request to {url}") + response = req.post(url, headers=headers, json=secret_manifest, verify=self.ssl_ca_cert) + + if response.status_code in [200, 201]: + logging.debug( + f"POST to {url} succeeded with {response.status_code}. Body {response.text}" + ) + return json.loads(response.text) + + logging.error( + f"POST to {url} failed with {response.status_code}. Body {response.text}" + ) + return None + except Exception as ex: + logging.error(f"post_secret {ex}") + return None + + def delete_secret(self, secret_name, namespace="nuvolaris"): + """ + Delete a Kubernetes secret. + :param secret_name: Name of the secret to delete. + :param namespace: Namespace where the secret is located. + :return: True if deletion was successful, False otherwise. + """ + url = f"{self.host}/api/v1/namespaces/{namespace}/secrets/{secret_name}" + headers = {"Authorization": self.token} + + try: + logging.info(f"DELETE request to {url}") + response = req.delete(url, headers=headers, verify=self.ssl_ca_cert) + + if response.status_code in [200, 202]: + logging.debug( + f"DELETE to {url} succeeded with {response.status_code}. Body {response.text}" + ) + return True + + logging.error( + f"DELETE to {url} failed with {response.status_code}. Body {response.text}" + ) + return False + except Exception as ex: + logging.error(f"delete_secret {ex}") + return False + + # --- CREA JOB --- + def post_job(self, job_name, job_manifest, namespace="nuvolaris"): + url = f"{self.host}/apis/batch/v1/namespaces/{namespace}/jobs" + headers = {"Authorization": self.token} + try: + logging.info(f"POST request to {url}") + response = None + response = req.post(url, headers=headers, json=job_manifest, verify=self.ssl_ca_cert) + if response.status_code in [200, 201, 202]: + logging.debug( + f"POST to {url} succeeded with {response.status_code}. Body {response.text}" + ) + return json.loads(response.text) + logging.error( + f"POST to {url} failed with {response.status_code}. Body {response.text}" + ) + return None + except Exception as ex: + logging.error(f"post_job {ex}") + return None + + # --- OTTIENI POD --- + def get_pod_by_job_name(self, job_name, namespace="nuvolaris"): + url = f"{self.host}/api/v1/namespaces/{namespace}/pods" + headers = {"Authorization": self.token} + try: + while True: + resp = req.get(url, headers=headers, verify=self.ssl_ca_cert) + + if not response.status_code in [200, 202]: + logging.error( + f"POST to {url} failed with {response.status_code}. Body {response.text}" + ) + return None + + logging.debug( + f"POST to {url} succeeded with {response.status_code}. Body {response.text}" + ) + + pods = resp.json()["items"] + for pod in pods: + labels = pod["metadata"].get("labels", {}) + if labels.get("job-name") == job_name: + return pod["metadata"]["name"] + time.sleep(1) + + except Exception as ex: + logging.error(f"get_pod_by_job_name {ex}") + return None + + # --- LEGGI LOG POD --- + def stream_pod_logs(self, pod_name, namespace="nuvolaris"): + url = f"{self.host}/api/v1/namespaces/{namespace}/pods/{pod_name}/log?follow=true" + headers = {"Authorization": self.token} + with req.get(url, headers=headers, verify=self.ssl_ca_cert, stream=True) as r: + for line in r.iter_lines(): + if line: + print(line.decode()) + + # --- CHECK STATUS JOB --- + def check_job_status(self, job_name, namespace="nuvolaris"): + url = f"{self.host}/apis/batch/v1/namespaces/{namespace}/jobs/{job_name}" + headers = {"Authorization": self.token} + try: + resp = req.get(url, headers=headers, verify=self.ssl_ca_cert) + resp.raise_for_status() + status = resp.json()["status"] + if status.get("succeeded", 0) > 0: + return True + else: + return False + except Exception as ex: + logging.error(f"check_job_status {ex}") + return False \ No newline at end of file diff --git a/openserverless/common/openwhisk_authorize.py b/openserverless/common/openwhisk_authorize.py index a1d6b50..31fa57e 100644 --- a/openserverless/common/openwhisk_authorize.py +++ b/openserverless/common/openwhisk_authorize.py @@ -33,6 +33,7 @@ class OpenwhiskAuthorize: def __init__(self, environ=os.environ): self._db = CouchDB() self._environ = environ + def encode(self, username, password): """Returns an HTTP basic authentication encrypted string given a valid diff --git a/openserverless/common/utils.py b/openserverless/common/utils.py new file mode 100644 index 0000000..91418a7 --- /dev/null +++ b/openserverless/common/utils.py @@ -0,0 +1,45 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +def env_to_dict(user_data, key="env"): + """ + extract env from user_data and return it as a dict + + Keyword arguments: + key -- the key to extract the env from + """ + body = {} + if key in user_data: + envs = list(user_data[key]) + else: + envs = [] + + for env in envs: + body[env['key']] = env['value'] + + return body + + +def dict_to_env(env): + """ + converts an env to a key/pair suitable for user_data storage + """ + body = [] + for key in env: + body.append({"key": key, "value": env[key]}) + + return body \ No newline at end of file diff --git a/openserverless/impl/auth/auth_service.py b/openserverless/impl/auth/auth_service.py index 8138f46..d13fde7 100644 --- a/openserverless/impl/auth/auth_service.py +++ b/openserverless/impl/auth/auth_service.py @@ -27,74 +27,99 @@ USER_META_DBN = "users_metadata" + class AuthService: def __init__(self, environ=os.environ): self._environ = environ self.couch_db = CouchDB() self.kube_client = KubeApiClient() - def fetch_user_data(self, login: str): logging.info(f"searching for user {login} data") try: - selector = {"selector":{"login": {"$eq": login }}} + selector = {"selector": {"login": {"$eq": login}}} response = self.couch_db.find_doc(USER_META_DBN, json.dumps(selector)) - if(response['docs']): - docs = list(response['docs']) - if(len(docs) > 0): - return docs[0] - + if response["docs"]: + docs = list(response["docs"]) + if len(docs) > 0: + return docs[0] + logging.warning(f"OpenServerless metadata for user {login} not found!") return None except Exception as e: - logging.error(f"failed to query OpenServerless metadata for user {login}. Reason: {e}") + logging.error( + f"failed to query OpenServerless metadata for user {login}. Reason: {e}" + ) return None + + def env_to_dict(self, user_data, key="env"): + """ + extract env from user_data and return it as a dict + + Keyword arguments: + key -- the key to extract the env from + """ + body = {} + if key in user_data: + envs = list(user_data[key]) + else: + envs = [] + + for env in envs: + body[env['key']] = env['value'] + + return body def map_data(self, user_data): """ Map the internal nuvolaris user_data records to the auth response """ resp = {} - resp['login'] = user_data['login'] - resp['email'] = user_data['email'] + resp["login"] = user_data["login"] + resp["email"] = user_data["email"] - if 'env' in user_data: - resp['env'] = user_data['env'] + if "env" in user_data: + resp["env"] = user_data["env"] - if 'quota' in user_data: - resp['quota'] = user_data['quota'] + if "quota" in user_data: + resp["quota"] = user_data["quota"] - return resp + return resp def login(self, login, password): user_data = self.fetch_user_data(login) - if(user_data): - if bu.verify_password(password, user_data['password']): - #if(password == user_data['password']): + if user_data: + if bu.verify_password(password, user_data["password"]): + # if(password == user_data['password']): return res_builder.build_response_with_data(self.map_data(user_data)) else: logging.warning(f"password mismatch for user {login}") return res_builder.build_error_message(f"Invalid credentials", 401) else: - logging.warning(f"no user {login} found") - return res_builder.build_error_message(f"Invalid credentials", 401) - + logging.warning(f"no user {login} found") + return res_builder.build_error_message(f"Invalid credentials", 401) + + def update_password(self, login, old_password, new_password): user_data = self.fetch_user_data(login) - if(user_data): - if bu.verify_password(old_password, user_data['password']): - whisk_user = self.kube_client.get_whisk_user(user_data['login']) + if user_data: + if bu.verify_password(old_password, user_data["password"]): + whisk_user = self.kube_client.get_whisk_user(user_data["login"]) - whisk_user['spec']['password'] = new_password - #whisk_user['spec']['password_timestamp'] = datetime.now().isoformat() - self.kube_client.update_whisk_user(whisk_user) + whisk_user["spec"]["password"] = new_password + # whisk_user['spec']['password_timestamp'] = datetime.now().isoformat() + self.kube_client.update_whisk_user(whisk_user) - return res_builder.build_response_with_data({"status":"ok","message":"Password updated"}) + return res_builder.build_response_with_data( + {"status": "ok", "message": "Password updated"} + ) else: - return res_builder.build_error_message(f"password mismatch for user {login}", 401) + return res_builder.build_error_message( + f"password mismatch for user {login}", 401 + ) else: - return res_builder.build_error_message(f"no user {login} found", 401) + return res_builder.build_error_message(f"no user {login} found", 401) diff --git a/openserverless/impl/builder/build_service.py b/openserverless/impl/builder/build_service.py new file mode 100644 index 0000000..9f5d5b5 --- /dev/null +++ b/openserverless/impl/builder/build_service.py @@ -0,0 +1,269 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +import shutil +from openserverless.common.kube_api_client import KubeApiClient +import os +import uuid + +JOB_NAME = "build" +CM_NAME = "cm" + + +class BuildService: + def __init__(self, build_config, user_env=None): + self.build_config = build_config + + # A super userful Kube Api Client + self.kube_client = KubeApiClient() + + # generate a unique ID for the build + self.id = str(uuid.uuid4()) + + # define a unique ConfigMap and Job name based on the ID + self.cm = f"{CM_NAME}-{self.id}" + self.job_name = f"{JOB_NAME}-{self.id}" + + # user environment variables + self.user_env = user_env if user_env is not None else {} + + # define registry host + self.registry_host = self.get_registry_host() + + self.init() + + def init(self): + """ + Initialize the build service by creating the necessary ConfigMap. + """ + + # install the nuvolaris-buildkitd-conf ConfigMap if not present + cm = self.kube_client.get_config_map("nuvolaris-buildkitd-conf", namespace="nuvolaris") + if cm is None: + self.kube_client.post_config_map( + cm_name="nuvolaris-buildkitd-conf", + file_or_dir="deploy/buildkit/buildkitd.toml", + namespace="nuvolaris", + ) + + def get_registry_host(self): + """ + Retrieve the registry host + - firstly, check if the user environment has a registry host set + - otherwise retrieve the OpenServerless config map + - if not present use a default value + """ + registry_host = 'nuvolaris-registry-svc:5000' + if (self.user_env.get('REGISTRY_HOST') is not None): + return self.build_config.get('REGISTRY_HOST') + + ops_config_map = self.kube_client.get_config_map('config') + if ops_config_map is not None: + if 'annotations' in ops_config_map.get('metadata', {}): + annotations = ops_config_map['metadata']['annotations'] + if 'registry_host' in annotations: + registry_host = annotations['registry_host'] + + return registry_host + + + def create_docker_file(self) -> str: + """ + Create a Dockerfile in the current directory. + """ + source = self.build_config.get("source") + + dockerfile_content = f"FROM {source}\n" + if 'file' in self.build_config: + requirement_file = self.get_requirements_file_from_kind() + dockerfile_content += f"COPY ./{requirement_file} /tmp/{requirement_file}\n" + dockerfile_content += "RUN echo \"/bin/extend\"\n" + return dockerfile_content + + def get_requirements_file_from_kind(self) -> str: + """ + Get the requirements file based on the kind of the build. + """ + kind = self.build_config.get("kind") + if kind == 'python': + return 'requirements.txt' + elif kind == 'nodejs': + return 'package.json' + elif kind == 'php': + return 'composer.json' + elif kind == 'java': + return 'pom.xml' + elif kind == 'go': + return 'go.mod' + elif kind == 'ruby': + return 'Gemfile' + elif kind == 'dotnet': + return 'project.json' + else: + raise ValueError(f"Unsupported kind: {kind}") + + def build(self, image_name: str) -> str: + """ + Build the Docker image using the provided build configuration. + The build configuration should include the source, target, and kind. + """ + import tempfile + import base64 + + tmpdirname = tempfile.mkdtemp() + if 'file' in self.build_config: + # decode base64 self.build_config.get('file') + requirements = base64.b64decode(self.build_config.get('file')).decode('utf-8') + + requirement_file = self.get_requirements_file_from_kind() + with open(os.path.join(tmpdirname, requirement_file), 'w') as f: + f.write(requirements) + + dockerfile_path = os.path.join(tmpdirname, "Dockerfile") + with open(dockerfile_path, "w") as dockerfile: + dockerfile.write(self.create_docker_file()) + + # check if the unzipped directory contains a Dockerfile and is not empty. + if not self.check_unzip_dir(tmpdirname): + return None + + # Create a ConfigMap for the build context + cm = self.kube_client.post_config_map( + cm_name=self.cm, + file_or_dir=tmpdirname, + namespace="nuvolaris", + ) + + shutil.rmtree(tmpdirname) + + if not cm: + return None + + # retrieve credentials to access the registry + # + + job_template = self.create_build_job(image_name) + job = self.kube_client.post_job(self.job_name, job_template) + if not job: + return None + + return job + + + def check_unzip_dir(self, unzip_dir: str) -> bool: + """ + Check if the unzipped directory contains a Dockerfile and is not empty.""" + if not os.path.exists(unzip_dir): + return False + + # Check if the directory contains a Dockerfile + dockerfile_path = os.path.join(unzip_dir, "Dockerfile") + if not os.path.exists(dockerfile_path): + return False + + return True + + def create_build_job(self, image_name: str) -> dict: + """Create a Kubernetes job manifest for building the Docker image.""" + registry_image_name = f"{self.registry_host}/{image_name}" + + # --- MANIFEST DEL JOB --- + job_manifest = { + "apiVersion": "batch/v1", + "kind": "Job", + "metadata": {"name": self.job_name}, + "spec": { + "backoffLimit": 0, + "template": { + "spec": { + "restartPolicy": "Never", + "volumes": [ + { + "name": "nuvolaris-buildkitd-conf", + "configMap": {"name": "nuvolaris-buildkitd-conf"}, + }, + { + "name": "build-context-vol", + "configMap": {"name": self.cm}, + }, + {"name": "workspace", "emptyDir": {}}, + {"name": "img-cache", "emptyDir": {}}, + {"name": "cdi-etc", "emptyDir": {}}, + {"name": "cdi-run", "emptyDir": {}}, + {"name": "cdi-buildkit", "emptyDir": {}}, + { + "name": "docker-config", + "secret": { + "secretName": "registry-pull-secret", + "items": [ + { + "key": ".dockerconfigjson", + "path": "config.json", + } + ], + }, + }, + ], + "initContainers": [ + { + "name": "copy-build-context", + "image": "busybox:1.36", + "command": [ + "sh", "-c", "cp -rvL /configmap/* /workspace/ && cat /workspace/Dockerfile", + ], + "volumeMounts": [ + { "name": "build-context-vol", "mountPath": "/configmap", }, + { "name": "workspace", "mountPath": "/workspace" }, + ], + } + ], + "containers": [ + { + "name": "buildkit", + "image": "moby/buildkit:master-rootless", + "command": ["sh", "-c"], + "args": [ + "rootlesskit buildkitd --config /config/buildkitd.toml & sleep 3 && " + f"buildctl build --frontend=dockerfile.v0 --local context=/workspace --local dockerfile=/workspace --output=type=image,name={registry_image_name},push=true" + ], + "securityContext": { + "runAsUser": 1000, + "runAsGroup": 1000, + "allowPrivilegeEscalation": True, + "privileged": True, + "seccompProfile": { "type": "Unconfined" } + }, + "env": [ + { "name": "BUILDKIT_ROOTLESS", "value": "1" } + ], + "volumeMounts": [ + { "name": "nuvolaris-buildkitd-conf", "mountPath": "/config" }, + { "name": "workspace", "mountPath": "/workspace" }, + { "name": "docker-config", "mountPath": "/home/user/.docker" }, + { "name": "img-cache", "mountPath": "/tmp" }, + { "name": "cdi-etc", "mountPath": "/etc/cdi" }, + { "name": "cdi-run", "mountPath": "/var/run/cdi" }, + { "name": "cdi-buildkit", "mountPath": "/etc/buildkit/cdi" }, + ], + } + ], + } + }, + }, + } + + return job_manifest diff --git a/openserverless/impl/onboard/user_validation.py b/openserverless/impl/onboard/user_validation.py index af5b06b..354c8b3 100644 --- a/openserverless/impl/onboard/user_validation.py +++ b/openserverless/impl/onboard/user_validation.py @@ -22,8 +22,9 @@ import openserverless.common.response_builder as res_builder from openserverless.common.kube_api_client import KubeApiClient + class UserValidation: - + def __init__(self, environ=os.environ): self._environ = environ self._kube_client = KubeApiClient() @@ -36,16 +37,26 @@ def validate(self, namespace): try: if not validation.is_valid_username(namespace): - return res_builder.build_error_message(message=f"Account namespace {namespace} is not valid. ", status_code=400) - - #check that there is no wsk user already existing with the same namespace + return res_builder.build_error_message( + message=f"Account namespace {namespace} is not valid. ", + status_code=400, + ) + + # check that there is no wsk user already existing with the same namespace existing_whisk_user = self._kube_client.get_whisk_user(namespace) if existing_whisk_user: - return res_builder.build_error_message(message=f"Namespace {namespace} already exists on domain.", status_code=409) + return res_builder.build_error_message( + message=f"Namespace {namespace} already exists on domain.", + status_code=409, + ) - return res_builder.build_response_message(f"Username {namespace} is valid and available") + return res_builder.build_response_message( + f"Username {namespace} is valid and available" + ) except Exception as ex: logging.error(ex) - return res_builder.build_response_message("Un-expected error detected attempting to setup you free account. If problem persists please get in touch with us info@nuvolaris.io") \ No newline at end of file + return res_builder.build_response_message( + "Un-expected error detected attempting to setup you free account. If problem persists please get in touch with us info@nuvolaris.io" + ) diff --git a/openserverless/rest/api.py b/openserverless/rest/api.py index 7b7e66a..fc9538b 100644 --- a/openserverless/rest/api.py +++ b/openserverless/rest/api.py @@ -18,9 +18,72 @@ from openserverless import app from http import HTTPStatus +from flask import request import openserverless.common.response_builder as res_builder +from openserverless.common.utils import env_to_dict +from openserverless.error.api_error import AuthorizationError +from openserverless.impl.builder.build_service import BuildService +from openserverless.common.openwhisk_authorize import OpenwhiskAuthorize +@app.route('/system/build', methods=['POST']) +def build(): + """ + Build Endpoint + --- + tags: + - Build + responses: + 200: + description: Build Endpoint Returns Basic Configuration Data used by this API. + schema: + $ref: '#/definitions/Message' + """ + + normalized_headers = {key.lower(): value for key, value in request.headers.items()} + auth_header = normalized_headers.get('authorization', None) + + if auth_header is None: + return res_builder.build_error_message("Missing authorization header", 401) + + oa = OpenwhiskAuthorize() + try: + user_data = oa.login(auth_header) + env = env_to_dict(user_data) + if env is None: + return res_builder.build_error_message("User environment not found", status_code=HTTPStatus.UNAUTHORIZED) + + if (request.json is None): + return res_builder.build_error_message("No JSON payload provided for build.", status_code=HTTPStatus.BAD_REQUEST) + + json_data = request.json + if 'source' not in json_data: + return res_builder.build_error_message("No source provided for build.", status_code=HTTPStatus.BAD_REQUEST) + if 'target' not in json_data: + return res_builder.build_error_message("No target provided for build.", status_code=HTTPStatus.BAD_REQUEST) + if 'kind' not in json_data: + return res_builder.build_error_message("No kind provided for build.", status_code=HTTPStatus.BAD_REQUEST) + + + # validate the target + target = json_data.get('target') + target_user = str(target).split(':')[0] + if user_data.get('login') != target_user: + return res_builder.build_error_message("Invalid target for the build.", status_code=HTTPStatus.BAD_REQUEST) + + + build_service = BuildService(build_config=json_data, user_env=env) + build_success = build_service.build(json_data.get('target')) # Replace with your desired image name + + if not build_success: + return res_builder.build_error_message("Build process failed.", status_code=HTTPStatus.INTERNAL_SERVER_ERROR) + + return res_builder.build_response_message("Build process initiated successfully.", status_code=HTTPStatus.OK) + + except AuthorizationError: + return res_builder.build_error_message("Invalid authorization", 401) + + @app.route('/system/info') def info(): From d2edcff6fcb2abbac3e749c855691b2d5b98db70 Mon Sep 17 00:00:00 2001 From: Bruno Salzano Date: Fri, 15 Aug 2025 07:08:29 +0200 Subject: [PATCH 3/4] chore: moved the builder to the appropriate path Moved builder to appropriate path and changed tasks. Updated swagger definition --- README.md | 2 +- TaskfileBuilder.yml | 13 ++-- openserverless/__init__.py | 1 + openserverless/rest/api.py | 66 -------------------- openserverless/rest/auth.py | 2 +- openserverless/rest/build.py | 118 +++++++++++++++++++++++++++++++++++ 6 files changed, 128 insertions(+), 74 deletions(-) create mode 100644 openserverless/rest/build.py diff --git a/README.md b/README.md index b3d4714..35e23c2 100644 --- a/README.md +++ b/README.md @@ -36,7 +36,7 @@ Available APIs at the moment: ### Build API -`POST /system/api/build` - Perform the build of a custom image and push it to repository. +`POST /system/api/v1/build` - Perform the build of a custom image and push it to repository. More informations [Here](docs/DEPLOYER.md) diff --git a/TaskfileBuilder.yml b/TaskfileBuilder.yml index 0f24475..3477f37 100644 --- a/TaskfileBuilder.yml +++ b/TaskfileBuilder.yml @@ -30,7 +30,8 @@ tasks: - if test -z "{{.TARGET}}"; then echo "TARGET IS NOT SET" && exit 1; fi - if test -z "{{.KIND}}"; then echo "KIND IS NOT SET" && exit 1; fi - | - echo '{"source": "{{.SOURCE}}", "target": "{{.TARGET}}", "kind": "{{.KIND}}", "file": "{{.REQUIREMENTS}}" }' | http POST $ADMIN_API_URL/system/build Content-Type:application/json Authorization:"{{.AUTH}}" + echo '{"source": "{{.SOURCE}}", "target": "{{.TARGET}}", "kind": "{{.KIND}}", "file": "{{.REQUIREMENTS}}" }' | \ + curl -X POST $ADMIN_API_URL/api/v1/build -H "Content-Type: application/json" -H "Authorization: {{.AUTH}}" -d @- - sleep 5 - task: logs deps: @@ -64,7 +65,7 @@ tasks: list-catalogs: desc: List catalogs in the registry cmds: - - http -a $REGISTRY_USER:$REGISTRY_PASS GET "${REGISTRY_HOST}/v2/_catalog" + - curl -u $REGISTRY_USER:$REGISTRY_PASS $REGISTRY_HOST/v2/_catalog silent: false list-images: @@ -73,7 +74,7 @@ tasks: CATALOG: '{{.CATALOG}}' cmds: - if test -z "{{.CATALOG}}"; then echo "CATALOG IS NOT SET" && exit 1; fi - - http -a $REGISTRY_USER:$REGISTRY_PASS GET "${REGISTRY_HOST}/v2/{{.CATALOG}}/tags/list" + - curl -u $REGISTRY_USER:$REGISTRY_PASS $REGISTRY_HOST/v2/{{.CATALOG}}/tags/list silent: false get-image: @@ -86,7 +87,7 @@ tasks: sh: echo '{{.IMAGE}}' | cut -d':' -f2 cmds: - echo "Getting image {{.IMAGE_NAME}} with hash {{.HASH}}" - - http -a $REGISTRY_USER:$REGISTRY_PASS GET "${REGISTRY_HOST}/v2/{{.IMAGE_NAME}}/manifests/{{.HASH}}" + - curl -u $REGISTRY_USER:$REGISTRY_PASS $REGISTRY_HOST/v2/{{.IMAGE_NAME}}/manifests/{{.HASH}} silent: false delete-image: @@ -98,9 +99,9 @@ tasks: HASH: sh: echo '{{.IMAGE}}' | cut -d':' -f2 MANIFEST_DIGEST: - sh: http --headers -a $REGISTRY_USER:$REGISTRY_PASS GET "${REGISTRY_HOST}/v2/{{.IMAGE_NAME}}/manifests/{{.HASH}}" | grep -i 'Docker-Content-Digest:' | awk '{print $2}' | tr -d '\r' + sh: curl --silent -u $REGISTRY_USER:$REGISTRY_PASS $REGISTRY_HOST/v2/{{.IMAGE_NAME}}/manifests/{{.HASH}} | grep -i 'Docker-Content-Digest:' | awk '{print $2}' | tr -d '\r' cmds: - echo 'Deleting image {{.IMAGE}}' - echo "Deleting manifest {{.MANIFEST_DIGEST}} for image {{.IMAGE_NAME}}" - - http -a $REGISTRY_USER:$REGISTRY_PASS DELETE "${REGISTRY_HOST}/v2/{{.IMAGE_NAME}}/manifests/{{.MANIFEST_DIGEST}}" + - curl -u $REGISTRY_USER:$REGISTRY_PASS -X DELETE $REGISTRY_HOST/v2/{{.IMAGE_NAME}}/manifests/{{.MANIFEST_DIGEST}} silent: false \ No newline at end of file diff --git a/openserverless/__init__.py b/openserverless/__init__.py index 211968a..7ae1020 100644 --- a/openserverless/__init__.py +++ b/openserverless/__init__.py @@ -79,4 +79,5 @@ def log_request_info(): import openserverless.rest.api import openserverless.rest.auth +import openserverless.rest.build diff --git a/openserverless/rest/api.py b/openserverless/rest/api.py index fc9538b..16fbd83 100644 --- a/openserverless/rest/api.py +++ b/openserverless/rest/api.py @@ -17,73 +17,7 @@ # from openserverless import app -from http import HTTPStatus -from flask import request - import openserverless.common.response_builder as res_builder -from openserverless.common.utils import env_to_dict -from openserverless.error.api_error import AuthorizationError -from openserverless.impl.builder.build_service import BuildService -from openserverless.common.openwhisk_authorize import OpenwhiskAuthorize - -@app.route('/system/build', methods=['POST']) -def build(): - """ - Build Endpoint - --- - tags: - - Build - responses: - 200: - description: Build Endpoint Returns Basic Configuration Data used by this API. - schema: - $ref: '#/definitions/Message' - """ - - normalized_headers = {key.lower(): value for key, value in request.headers.items()} - auth_header = normalized_headers.get('authorization', None) - - if auth_header is None: - return res_builder.build_error_message("Missing authorization header", 401) - - oa = OpenwhiskAuthorize() - try: - user_data = oa.login(auth_header) - env = env_to_dict(user_data) - if env is None: - return res_builder.build_error_message("User environment not found", status_code=HTTPStatus.UNAUTHORIZED) - - if (request.json is None): - return res_builder.build_error_message("No JSON payload provided for build.", status_code=HTTPStatus.BAD_REQUEST) - - json_data = request.json - if 'source' not in json_data: - return res_builder.build_error_message("No source provided for build.", status_code=HTTPStatus.BAD_REQUEST) - if 'target' not in json_data: - return res_builder.build_error_message("No target provided for build.", status_code=HTTPStatus.BAD_REQUEST) - if 'kind' not in json_data: - return res_builder.build_error_message("No kind provided for build.", status_code=HTTPStatus.BAD_REQUEST) - - - # validate the target - target = json_data.get('target') - target_user = str(target).split(':')[0] - if user_data.get('login') != target_user: - return res_builder.build_error_message("Invalid target for the build.", status_code=HTTPStatus.BAD_REQUEST) - - - build_service = BuildService(build_config=json_data, user_env=env) - build_success = build_service.build(json_data.get('target')) # Replace with your desired image name - - if not build_success: - return res_builder.build_error_message("Build process failed.", status_code=HTTPStatus.INTERNAL_SERVER_ERROR) - - return res_builder.build_response_message("Build process initiated successfully.", status_code=HTTPStatus.OK) - - except AuthorizationError: - return res_builder.build_error_message("Invalid authorization", 401) - - @app.route('/system/info') def info(): diff --git a/openserverless/rest/auth.py b/openserverless/rest/auth.py index 6cf3dcf..33a916c 100644 --- a/openserverless/rest/auth.py +++ b/openserverless/rest/auth.py @@ -35,7 +35,7 @@ def password(login, **kwargs): summary: Login an OpenServerless user using login/password payload operationId: patchOpsUser security: - - openwhiskBasicAuth: [] + - openwhiskBasicAuth: [] consumes: - application/json definitions: diff --git a/openserverless/rest/build.py b/openserverless/rest/build.py new file mode 100644 index 0000000..f19a186 --- /dev/null +++ b/openserverless/rest/build.py @@ -0,0 +1,118 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +from openserverless import app +from http import HTTPStatus +from flask import request + +import openserverless.common.response_builder as res_builder +from openserverless.common.utils import env_to_dict +from openserverless.error.api_error import AuthorizationError +from openserverless.impl.builder.build_service import BuildService +from openserverless.common.openwhisk_authorize import OpenwhiskAuthorize + +@app.route('/system/api/v1/build', methods=['POST']) +def build(): + """ + Build Endpoint + --- + tags: + - Build + summary: Build an image using the provided source, target, and kind. + description: This endpoint triggers a build process based on the provided parameters. + operationId: buildImage + security: + - openwhiskBasicAuth: [] + consumes: + - application/json + parameters: + - in: body + name: BuildRequest + required: true + schema: + type: object + properties: + source: + type: string + description: Source for the build + target: + type: string + description: Target for the build + kind: + type: string + description: Kind of the build + responses: + 200: + description: Build process initiated successfully. + schema: + $ref: '#/definitions/Message' + 400: + description: Bad Request. Missing or invalid parameters. + schema: + $ref: '#/definitions/Message' + 401: + description: Unauthorized. Invalid or missing authorization header. + schema: + $ref: '#/definitions/Message' + 500: + description: Internal Server Error. Build process failed. + schema: + $ref: '#/definitions/Message' + """ + + normalized_headers = {key.lower(): value for key, value in request.headers.items()} + auth_header = normalized_headers.get('authorization', None) + + if auth_header is None: + return res_builder.build_error_message("Missing authorization header", 401) + + oa = OpenwhiskAuthorize() + try: + user_data = oa.login(auth_header) + env = env_to_dict(user_data) + if env is None: + return res_builder.build_error_message("User environment not found", status_code=HTTPStatus.UNAUTHORIZED) + + if (request.json is None): + return res_builder.build_error_message("No JSON payload provided for build.", status_code=HTTPStatus.BAD_REQUEST) + + json_data = request.json + if 'source' not in json_data: + return res_builder.build_error_message("No source provided for build.", status_code=HTTPStatus.BAD_REQUEST) + if 'target' not in json_data: + return res_builder.build_error_message("No target provided for build.", status_code=HTTPStatus.BAD_REQUEST) + if 'kind' not in json_data: + return res_builder.build_error_message("No kind provided for build.", status_code=HTTPStatus.BAD_REQUEST) + + + # validate the target + target = json_data.get('target') + target_user = str(target).split(':')[0] + if user_data.get('login') != target_user: + return res_builder.build_error_message("Invalid target for the build.", status_code=HTTPStatus.BAD_REQUEST) + + + build_service = BuildService(build_config=json_data, user_env=env) + build_success = build_service.build(json_data.get('target')) # Replace with your desired image name + + if not build_success: + return res_builder.build_error_message("Build process failed.", status_code=HTTPStatus.INTERNAL_SERVER_ERROR) + + return res_builder.build_response_message("Build process initiated successfully.", status_code=HTTPStatus.OK) + + except AuthorizationError: + return res_builder.build_error_message("Invalid authorization", 401) \ No newline at end of file From 0255ef97f5390f46e00df334e166fd738ec775c1 Mon Sep 17 00:00:00 2001 From: Bruno Salzano Date: Fri, 15 Aug 2025 14:03:31 +0200 Subject: [PATCH 4/4] test: added tests added tests and more documentation in docstrings. Added a TODO.md file to track the new things to do --- .github/cisetup.sh | 1 + .github/workflows/check.yml | 9 +++ .licenserc.yaml | 1 + TODO.md | 28 +++++++++ Taskfile.yml | 9 +++ deploy/buildkit/buildkitd.toml | 17 ++++++ openserverless/common/kube_api_client.py | 63 +++++++++++++------- openserverless/common/openwhisk_authorize.py | 42 ++++++++++--- openserverless/common/utils.py | 47 ++++++++++++++- openserverless/common/validation.py | 18 ++++++ 10 files changed, 204 insertions(+), 31 deletions(-) create mode 100644 TODO.md diff --git a/.github/cisetup.sh b/.github/cisetup.sh index c6e0aab..40c4234 100644 --- a/.github/cisetup.sh +++ b/.github/cisetup.sh @@ -18,3 +18,4 @@ # sudo sh -c "$(curl --location https://taskfile.dev/install.sh)" -- -d -b /usr/local/bin sudo apt-get -y install curl wget jq +pip install uv \ No newline at end of file diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 945097e..c454c3a 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -34,3 +34,12 @@ jobs: submodules: recursive - name: License uses: apache/skywalking-eyes@main + - name: Set up Python 3.12 + uses: actions/setup-python@v4 + with: + python-version: 3.12 + - name: Setup + run: bash .github/cisetup.sh + - name: Unit Tests + run: task utest + continue-on-error: false diff --git a/.licenserc.yaml b/.licenserc.yaml index 7e98429..9e4d818 100644 --- a/.licenserc.yaml +++ b/.licenserc.yaml @@ -28,6 +28,7 @@ header: - 'LICENSE' - 'NOTICE' - 'DISCLAIMER' + - 'deploy/samples/requirements.txt' - '**/*.json' - '**/*.service' - '**/*.txt' diff --git a/TODO.md b/TODO.md new file mode 100644 index 0000000..ebed852 --- /dev/null +++ b/TODO.md @@ -0,0 +1,28 @@ + +# TODO + +## Tests +Add integration and unit tests + +## Various + +- [ ] `openserverless.common.whis_user_data.py` - Add `with_` blocks for other new OpenServerless Services +- [ ] `openserverless.common.whisk_user_generator` - Check if `generate_whisk_user_yaml` is complete diff --git a/Taskfile.yml b/Taskfile.yml index 260649c..f4a7087 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -126,3 +126,12 @@ tasks: BASEIMG=$(task base-image-name) IMG="$BASEIMG:{{.TAG}}" kind load docker-image $IMG --name=nuvolaris + + utest: + cmds: + - | + for test in openserverless/common/{{.T}}*.py + do echo "*** [{{.KUBE}}] $test" + uv run python3 -m doctest -o ELLIPSIS $test {{.CLI_ARGS}} + done + silent: true \ No newline at end of file diff --git a/deploy/buildkit/buildkitd.toml b/deploy/buildkit/buildkitd.toml index 772e30e..f01def0 100644 --- a/deploy/buildkit/buildkitd.toml +++ b/deploy/buildkit/buildkitd.toml @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# # ========================= # Worker OCI (rootlesskit) # ========================= diff --git a/openserverless/common/kube_api_client.py b/openserverless/common/kube_api_client.py index 46e041f..d718ba2 100644 --- a/openserverless/common/kube_api_client.py +++ b/openserverless/common/kube_api_client.py @@ -22,8 +22,8 @@ import logging from base64 import b64decode, b64encode -from .validation import is_empty_arg +from openserverless.common.utils import join_host_port from openserverless.config.app_config import AppConfig from openserverless.error.config_exception import ConfigException @@ -31,16 +31,6 @@ SERVICE_PORT_ENV_NAME = "KUBERNETES_SERVICE_PORT" SERVICE_TOKEN_FILENAME = "/var/run/secrets/kubernetes.io/serviceaccount/token" SERVICE_CERT_FILENAME = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt" - - -def _join_host_port(host, port): - template = "%s:%s" - host_requires_bracketing = ":" in host or "%" in host - if host_requires_bracketing: - template = "[%s]:%s" - return template % (host, port) - - class KubeApiClient: def __init__(self, environ=os.environ): @@ -50,11 +40,6 @@ def __init__(self, environ=os.environ): self._load_incluster_config() def _parse_b64(self, encoded_str): - """ - Decode b64 encoded string - param: encoded_str a Base64 encoded string - return: decoded string - """ try: return b64decode(encoded_str).decode() except: @@ -73,7 +58,7 @@ def _load_incluster_config(self): ): raise ConfigException("Service host/port is not set.") - self.host = "https://" + _join_host_port( + self.host = "https://" + join_host_port( self._environ.get(SERVICE_HOST_ENV_NAME), self._environ.get(SERVICE_PORT_ENV_NAME), ) @@ -254,7 +239,13 @@ def get_config_map(self, cm_name, namespace="nuvolaris"): return None def post_config_map(self, cm_name, file_or_dir, namespace="nuvolaris"): - + """ + Create a ConfigMap from a file or directory. + :param cm_name: Name of the ConfigMap. + :param file_or_dir: Path to the file or directory containing the data. + :param namespace: Namespace where the ConfigMap will be created. + :return: The created ConfigMap or None if failed. + """ if not os.path.exists(file_or_dir): raise ConfigException(f"File or directory {file_or_dir} does not exist.") @@ -301,6 +292,12 @@ def post_config_map(self, cm_name, file_or_dir, namespace="nuvolaris"): return None def delete_config_map(self, cm_name, namespace="nuvolaris"): + """ + Delete a ConfigMap by name. + :param cm_name: Name of the ConfigMap to delete. + :param namespace: Namespace where the ConfigMap is located. + :return: True if deletion was successful, False otherwise. + """ url = f"{self.host}/api/v1/namespaces/{namespace}/configmaps/{cm_name}" headers = {"Authorization": self.token} @@ -416,8 +413,14 @@ def delete_secret(self, secret_name, namespace="nuvolaris"): logging.error(f"delete_secret {ex}") return False - # --- CREA JOB --- - def post_job(self, job_name, job_manifest, namespace="nuvolaris"): + def post_job(self, job_name, job_manifest, namespace="nuvolaris"): + """ + Create a Kubernetes job. + :param job_name: Name of the job. + :param job_manifest: Dictionary containing the job manifest. + :param namespace: Namespace where the job will be created. + :return: The created job or None if failed. + """ url = f"{self.host}/apis/batch/v1/namespaces/{namespace}/jobs" headers = {"Authorization": self.token} try: @@ -437,8 +440,13 @@ def post_job(self, job_name, job_manifest, namespace="nuvolaris"): logging.error(f"post_job {ex}") return None - # --- OTTIENI POD --- def get_pod_by_job_name(self, job_name, namespace="nuvolaris"): + """ + Get the pod name associated with a job by its name. + :param job_name: Name of the job. + :param namespace: Namespace where the job is located. + :return: The pod name if found, None otherwise. + """ url = f"{self.host}/api/v1/namespaces/{namespace}/pods" headers = {"Authorization": self.token} try: @@ -466,8 +474,12 @@ def get_pod_by_job_name(self, job_name, namespace="nuvolaris"): logging.error(f"get_pod_by_job_name {ex}") return None - # --- LEGGI LOG POD --- def stream_pod_logs(self, pod_name, namespace="nuvolaris"): + """ + Stream logs from a specific pod. + :param pod_name: Name of the pod to stream logs from. + :param namespace: Namespace where the pod is located. + """ url = f"{self.host}/api/v1/namespaces/{namespace}/pods/{pod_name}/log?follow=true" headers = {"Authorization": self.token} with req.get(url, headers=headers, verify=self.ssl_ca_cert, stream=True) as r: @@ -475,8 +487,13 @@ def stream_pod_logs(self, pod_name, namespace="nuvolaris"): if line: print(line.decode()) - # --- CHECK STATUS JOB --- def check_job_status(self, job_name, namespace="nuvolaris"): + """ + Check the status of a job by its name. + :param job_name: Name of the job to check. + :param namespace: Namespace where the job is located. + :return: True if the job has succeeded, False otherwise. + """ url = f"{self.host}/apis/batch/v1/namespaces/{namespace}/jobs/{job_name}" headers = {"Authorization": self.token} try: diff --git a/openserverless/common/openwhisk_authorize.py b/openserverless/common/openwhisk_authorize.py index 31fa57e..cc9c2d8 100644 --- a/openserverless/common/openwhisk_authorize.py +++ b/openserverless/common/openwhisk_authorize.py @@ -33,7 +33,6 @@ class OpenwhiskAuthorize: def __init__(self, environ=os.environ): self._db = CouchDB() self._environ = environ - def encode(self, username, password): """Returns an HTTP basic authentication encrypted string given a valid @@ -46,16 +45,36 @@ def encode(self, username, password): return f"Basic {b64encode(username_password.encode()).decode()}" def _parse_b64(self, encoded_str): + """ + Parse a base64 encoded string and return the username and password. + If the string is not base64 encoded, it will try to split it by ':'. + Raises DecodeError if the string cannot be decoded or parsed. + >>> oa = OpenwhiskAuthorize() + >>> oa._parse_b64("dXNlcm5hbWU6cGFzc3dvcmQ=") + ('username', 'password') + >>> oa._parse_b64("username:password") + ('username', 'password') + >>> oa._parse_b64("invalid_base64_string") + Traceback (most recent call last): + ... + openserverless.error.api_error.DecodeError: authentication token does not seems to be b64 encoded + """ + username = None + password = None try: - username, password = b64decode(encoded_str).decode().split(":", 1) + decoded = b64decode(encoded_str) + + credentials = decoded.decode() + if credentials.count(":") != 1: + raise DecodeError("authentication token does not seems to be b64 encoded") + username, password = credentials.split(":", 1) except: # fallback in case the token is not bas64 encoded - username, password = encoded_str.split(":", 1) + if encoded_str.count(":") == 1: + username, password = encoded_str.split(":", 1) - if not username or not password: - raise DecodeError( - "authentication token does not seems to be b64 encoded" - ) + if not username or not password: + raise DecodeError("authentication token does not seems to be b64 encoded") return username, password @@ -63,6 +82,15 @@ def decode(self, encoded_str): """Decode an encrypted HTTP basic authentication string. Returns a tuple of the form (username, password), and raises a DecodeError exception if nothing could be decoded. + >>> oa = OpenwhiskAuthorize() + >>> oa.decode("Basic dXNlcm5hbWU6cGFzc3dvcmQ=") + ('username', 'password') + >>> oa.decode("dXNlcm5hbWU6cGFzc3dvcmQ=") + ('username', 'password') + >>> oa.decode("invalid_base64_string") + Traceback (most recent call last): + ... + openserverless.error.api_error.DecodeError: authentication token does not seems to be b64 encoded """ split = encoded_str.strip().split(" ") diff --git a/openserverless/common/utils.py b/openserverless/common/utils.py index 91418a7..3931c63 100644 --- a/openserverless/common/utils.py +++ b/openserverless/common/utils.py @@ -21,6 +21,19 @@ def env_to_dict(user_data, key="env"): Keyword arguments: key -- the key to extract the env from + + >>> env_to_dict({"env": [{"key": "VAR1", "value": "value1"}, {"key": "VAR2", "value": "value2"}]}) + {'VAR1': 'value1', 'VAR2': 'value2'} + >>> env_to_dict({"env": []}) + {} + >>> env_to_dict({"other_key": [{"key": "VAR1", "value": "value1"}]}, key="other_key") + {'VAR1': 'value1'} + >>> env_to_dict({"env": [{"key": "VAR1", "value": "value1"}]}, key="env") + {'VAR1': 'value1'} + >>> env_to_dict({"env": [{"key": "VAR1", "value": "value1"}, {"key": "VAR2", "value": "value2"}]}, key="non_existent_key") + {} + >>> env_to_dict({"env": [{"key": "VAR1", "value": "value1"}]}, key="env") + {'VAR1': 'value1'} """ body = {} if key in user_data: @@ -37,9 +50,41 @@ def env_to_dict(user_data, key="env"): def dict_to_env(env): """ converts an env to a key/pair suitable for user_data storage + + >>> dict_to_env({"VAR1": "value1", "VAR2": "value2"}) + [{'key': 'VAR1', 'value': 'value1'}, {'key': 'VAR2', 'value': 'value2'}] + >>> dict_to_env({}) + [] """ body = [] for key in env: body.append({"key": key, "value": env[key]}) - return body \ No newline at end of file + return body + +def join_host_port(host, port): + """ + Join host and port into a URL format. + >>> join_host_port("localhost", "8080") + 'localhost:8080' + >>> join_host_port("localhost", 8080) + 'localhost:8080' + >>> join_host_port("localhost", "80") + 'localhost:80' + >>> join_host_port("localhost", "abcd") + Traceback (most recent call last): + ... + ValueError: Port must be numeric + + """ + template = "%s:%s" + try: + port_int = int(port) + port = str(port_int) + except (ValueError, TypeError): + raise ValueError("Port must be numeric") + + host_requires_bracketing = ":" in host or "%" in host + if host_requires_bracketing: + template = "[%s]:%s" + return template % (host, port) \ No newline at end of file diff --git a/openserverless/common/validation.py b/openserverless/common/validation.py index 47fb5a8..f0a1655 100644 --- a/openserverless/common/validation.py +++ b/openserverless/common/validation.py @@ -21,6 +21,20 @@ def is_valid_username(username): """ Verifies the given username follows nuvolaris rule + >>> is_valid_username("bruno") + True + >>> is_valid_username("bruno123") + True + >>> is_valid_username("bruno-123") + False + >>> is_valid_username("bruno_123") + False + >>> is_valid_username("bruno@123") + False + >>> is_valid_username("bruno 123") + False + >>> is_valid_username("brun") + False """ pat = re.compile(r"^[a-z0-9]{5,60}(?:[a-z0-9])?$") if re.fullmatch(pat, username): @@ -35,6 +49,10 @@ def is_empty_arg(args, arg_name): param: args param: arg_name return: True if the argument is not contained in the input args array or if it is an empty string value + >>> is_empty_arg({"arg1": "value1"}, "arg1") + False + >>> is_empty_arg({"arg1": "value1"}, "arg2") + True """ if arg_name not in args: