diff --git a/.env.example b/.env.example index 341218c..1b9843f 100644 --- a/.env.example +++ b/.env.example @@ -12,4 +12,10 @@ KUBERNETES_CERT_FILENAME=./tokens/ca.crt # if not set, the image is build with local tag and will not be pushed REGISTRY= # namespace is required if REGISTRY is set -NAMESPACE= \ No newline at end of file +NAMESPACE= + +REGISTRY_HOST=http://127.0.0.1:5000 +REGISTRY_USER=opsuser +REGISTRY_PASS=password + +ADMIN_API_URL=http://127.0.0.1:5002 \ No newline at end of file diff --git a/.github/cisetup.sh b/.github/cisetup.sh index c6e0aab..40c4234 100644 --- a/.github/cisetup.sh +++ b/.github/cisetup.sh @@ -18,3 +18,4 @@ # sudo sh -c "$(curl --location https://taskfile.dev/install.sh)" -- -d -b /usr/local/bin sudo apt-get -y install curl wget jq +pip install uv \ No newline at end of file diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 945097e..c454c3a 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -34,3 +34,12 @@ jobs: submodules: recursive - name: License uses: apache/skywalking-eyes@main + - name: Set up Python 3.12 + uses: actions/setup-python@v4 + with: + python-version: 3.12 + - name: Setup + run: bash .github/cisetup.sh + - name: Unit Tests + run: task utest + continue-on-error: false diff --git a/.licenserc.yaml b/.licenserc.yaml index 7e98429..9e4d818 100644 --- a/.licenserc.yaml +++ b/.licenserc.yaml @@ -28,6 +28,7 @@ header: - 'LICENSE' - 'NOTICE' - 'DISCLAIMER' + - 'deploy/samples/requirements.txt' - '**/*.json' - '**/*.service' - '**/*.txt' diff --git a/README.md b/README.md index affd74f..35e23c2 100644 --- a/README.md +++ b/README.md @@ -34,18 +34,23 @@ Available APIs at the moment: `PATCH /system/api/v1/auth/{login}` - Update the user password patching the corresponding wsku/\ entry. +### Build API + +`POST /system/api/v1/build` - Perform the build of a custom image and push it to repository. + +More informations [Here](docs/DEPLOYER.md) + ### Info API `GET /system/info` - Info endpoint - ## Developer instructions You need to have access to be Apache OpenServerless admin and have access to kubernetes cluster. Refer to the [Apache OpenServerless installation page](https://openserverless.apache.org/docs/installation/install/docker/): -Give the command `task setup-developer` and it will: +Give the command `task dev:setup-developer` and it will: - extract the required ca.crt and token from operator service account - copy a sample .env file @@ -61,14 +66,22 @@ Open http://localhost:5002/system/apidocs/ to see the API documentation. Taskfile supports the following tasks: ```yaml -* build: Build the image locally -* build-and-load: Build the image and loads it to local Kind cluster -* buildx: Build the docker image using buildx. Set PUSH=1 to push the image to the registry. -* docker-login: Login to the docker registry. Set REGISTRY=ghcr or REGISTRY=dockerhub in .env to use the respective registry. -* get-tokens: Get Service Account tokens and save them to tokens directory -* image-tag: Create a new tag for the current git commit. -* run: Run the admin api locally, using configuration from .env file -* setup-developer: Setup developer environment +* build: Build the image locally +* build-and-load: Build the image and loads it to local Kind cluster +* buildx: Build the docker image using buildx. Set PUSH=1 to push the image to the registry. +* docker-login: Login to the docker registry. Set REGISTRY=ghcr or REGISTRY=dockerhub in .env to use the respective registry. +* image-tag: Create a new tag for the current git commit. +* builder:cleanjobs: Clean up old jobs +* builder:delete-image: Delete an image from the registry +* builder:get-image: Get an image from the registry +* builder:list-catalogs: List catalogs in the registry +* builder:list-images: List images in a specific catalog +* builder:logs: Show logs of the last build job +* builder:send: Send the build to the server +* builder:updatetoml: Update the buildkitd.toml file config map +* dev:get-tokens: Get Service Account tokens and save them to tokens directory +* dev:run: Run the admin api locally, using configuration from .env file +* dev:setup-developer: Setup developer environment ``` ## Build and push @@ -128,4 +141,8 @@ $ git push apache 0.1.0-incubating.2507270910 ``` This will trigger the build workflow, and the process will be visible at -https://github.com/apache/openserverless-admin-api/actions \ No newline at end of file +https://github.com/apache/openserverless-admin-api/actions + +## Additional Documentation + +- [Deployer](docs/DEPLOYER.md) \ No newline at end of file diff --git a/TODO.md b/TODO.md new file mode 100644 index 0000000..ebed852 --- /dev/null +++ b/TODO.md @@ -0,0 +1,28 @@ + +# TODO + +## Tests +Add integration and unit tests + +## Various + +- [ ] `openserverless.common.whis_user_data.py` - Add `with_` blocks for other new OpenServerless Services +- [ ] `openserverless.common.whisk_user_generator` - Check if `generate_whisk_user_yaml` is complete diff --git a/Taskfile.yml b/Taskfile.yml index cbb9d02..f4a7087 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -27,32 +27,14 @@ vars: dotenv: - .env - -tasks: - get-tokens: - desc: "Get Service Account tokens and save them to tokens directory" - silent: true - cmds: - - mkdir -p tokens - - kubectl get secret nuvolaris-wsku-secret -o jsonpath='{.data.token}' | base64 --decode > tokens/token - - kubectl get secret nuvolaris-wsku-secret -o jsonpath='{.data.ca\.crt}' | base64 --decode > tokens/ca.crt +includes: + builder: + taskfile: TaskfileBuilder.yml + dev: + taskfile: TaskfileDev.yml - setup-developer: - desc: "Setup developer environment" - silent: true - cmds: - - task: get-tokens - - | - if [ ! -f .env ]; - then cp .env.example .env - echo "Please edit .env file with your local CouchDB and Kubernetes credentials" - fi - - | - if [ ! -d .venv ]; - then uv venv - fi - - uv pip install -r pyproject.toml 2>/dev/null +tasks: docker-login-ghcr: > silent: true @@ -145,8 +127,11 @@ tasks: IMG="$BASEIMG:{{.TAG}}" kind load docker-image $IMG --name=nuvolaris - run: - desc: | - Run the admin api locally, using configuration from .env file + utest: cmds: - - uv run -m openserverless \ No newline at end of file + - | + for test in openserverless/common/{{.T}}*.py + do echo "*** [{{.KUBE}}] $test" + uv run python3 -m doctest -o ELLIPSIS $test {{.CLI_ARGS}} + done + silent: true \ No newline at end of file diff --git a/TaskfileBuilder.yml b/TaskfileBuilder.yml new file mode 100644 index 0000000..3477f37 --- /dev/null +++ b/TaskfileBuilder.yml @@ -0,0 +1,107 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +version: '3' + +tasks: + + send: + desc: Send the build to the server + vars: + AUTH: + sh: cat ~/.wskprops | grep "AUTH" | cut -d'=' -f2 | xargs -I {} + cmds: + - if test -z "{{.SOURCE}}"; then echo "SOURCE IS NOT SET" && exit 1; fi + - if test -z "{{.TARGET}}"; then echo "TARGET IS NOT SET" && exit 1; fi + - if test -z "{{.KIND}}"; then echo "KIND IS NOT SET" && exit 1; fi + - | + echo '{"source": "{{.SOURCE}}", "target": "{{.TARGET}}", "kind": "{{.KIND}}", "file": "{{.REQUIREMENTS}}" }' | \ + curl -X POST $ADMIN_API_URL/api/v1/build -H "Content-Type: application/json" -H "Authorization: {{.AUTH}}" -d @- + - sleep 5 + - task: logs + deps: + - cleanjobs + # - updatetoml + silent: true + + logs: + desc: Show logs of the last build job + cmds: + - kubectl -n nuvolaris logs $(kubectl get jobs.batch -o name | grep "build-") -c buildkit --follow + silent: false + + cleanjobs: + desc: Clean up old jobs + cmds: + - for I in $(kubectl get jobs -n nuvolaris | grep build | awk '{ print $1 }' | tr "\n" " "); do kubectl delete job $I; done + - for I in $(kubectl get cm -n nuvolaris | grep "cm-" | awk '{ print $1 }' | tr "\n" " "); do kubectl delete cm $I; done + silent: true + + updatetoml: + desc: Update the buildkitd.toml file config map + cmds: + - | + if test $(kubectl -n nuvolaris get cm -o name | grep nuvolaris-buildkitd-conf | wc -l) -gt 0; + then kubectl -n nuvolaris delete configmap nuvolaris-buildkitd-conf + fi + - kubectl -n nuvolaris create configmap nuvolaris-buildkitd-conf --from-file=deploy/buildkit/buildkitd.toml + silent: true + + list-catalogs: + desc: List catalogs in the registry + cmds: + - curl -u $REGISTRY_USER:$REGISTRY_PASS $REGISTRY_HOST/v2/_catalog + silent: false + + list-images: + desc: List images in a specific catalog + vars: + CATALOG: '{{.CATALOG}}' + cmds: + - if test -z "{{.CATALOG}}"; then echo "CATALOG IS NOT SET" && exit 1; fi + - curl -u $REGISTRY_USER:$REGISTRY_PASS $REGISTRY_HOST/v2/{{.CATALOG}}/tags/list + silent: false + + get-image: + desc: Get an image from the registry + vars: + IMAGE: '{{.IMAGE}}' + IMAGE_NAME: + sh: echo '{{.IMAGE}}' | cut -d':' -f1 + HASH: + sh: echo '{{.IMAGE}}' | cut -d':' -f2 + cmds: + - echo "Getting image {{.IMAGE_NAME}} with hash {{.HASH}}" + - curl -u $REGISTRY_USER:$REGISTRY_PASS $REGISTRY_HOST/v2/{{.IMAGE_NAME}}/manifests/{{.HASH}} + silent: false + + delete-image: + desc: Delete an image from the registry + vars: + IMAGE: '{{.IMAGE}}' + IMAGE_NAME: + sh: echo '{{.IMAGE}}' | cut -d':' -f1 + HASH: + sh: echo '{{.IMAGE}}' | cut -d':' -f2 + MANIFEST_DIGEST: + sh: curl --silent -u $REGISTRY_USER:$REGISTRY_PASS $REGISTRY_HOST/v2/{{.IMAGE_NAME}}/manifests/{{.HASH}} | grep -i 'Docker-Content-Digest:' | awk '{print $2}' | tr -d '\r' + cmds: + - echo 'Deleting image {{.IMAGE}}' + - echo "Deleting manifest {{.MANIFEST_DIGEST}} for image {{.IMAGE_NAME}}" + - curl -u $REGISTRY_USER:$REGISTRY_PASS -X DELETE $REGISTRY_HOST/v2/{{.IMAGE_NAME}}/manifests/{{.MANIFEST_DIGEST}} + silent: false \ No newline at end of file diff --git a/TaskfileDev.yml b/TaskfileDev.yml new file mode 100644 index 0000000..27da031 --- /dev/null +++ b/TaskfileDev.yml @@ -0,0 +1,50 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +version: '3' + +tasks: + get-tokens: + desc: "Get Service Account tokens and save them to tokens directory" + silent: true + cmds: + - mkdir -p tokens + - kubectl get secret nuvolaris-wsku-secret -o jsonpath='{.data.token}' | base64 --decode > tokens/token + - kubectl get secret nuvolaris-wsku-secret -o jsonpath='{.data.ca\.crt}' | base64 --decode > tokens/ca.crt + + setup-developer: + desc: "Setup developer environment" + silent: true + cmds: + - task: get-tokens + - | + if [ ! -f .env ]; + then cp .env.example .env + echo "Please edit .env file with your local CouchDB and Kubernetes credentials" + fi + - | + if [ ! -d .venv ]; + then uv venv + fi + - uv pip install -r pyproject.toml 2>/dev/null + + run: + desc: | + Run the admin api locally, using configuration from .env file + cmds: + - uv run -m openserverless \ No newline at end of file diff --git a/deploy/buildkit/buildkitd.toml b/deploy/buildkit/buildkitd.toml new file mode 100644 index 0000000..f01def0 --- /dev/null +++ b/deploy/buildkit/buildkitd.toml @@ -0,0 +1,41 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# ========================= +# Worker OCI (rootlesskit) +# ========================= +[worker.oci] + enabled = true + rootless = true + no-process-sandbox = true + snapshotter = "overlayfs" # usa overlayfs se il kernel lo consente + +[worker.containerd] + enabled = false + +# ========================= +# Registry HTTP insicuro +# ========================= +[registry."nuvolaris-registry-svc:5000"] + insecure = true + http = true + +# ========================= +# Logging +# ========================= +[log] + level = "debug" \ No newline at end of file diff --git a/deploy/samples/requirements.txt b/deploy/samples/requirements.txt new file mode 100644 index 0000000..4b48d6f --- /dev/null +++ b/deploy/samples/requirements.txt @@ -0,0 +1,2 @@ +gnews +beautifulsoup4 \ No newline at end of file diff --git a/docs/DEPLOYER.md b/docs/DEPLOYER.md new file mode 100644 index 0000000..6f4b9c1 --- /dev/null +++ b/docs/DEPLOYER.md @@ -0,0 +1,60 @@ + +# Deployer + +These tasks are useful to interact with OpenServerless Admin Api Builder + +There are some tasks to interact with OpenServerless internal registry too. + +## Available tasks + +task: Available tasks for this project: + +``` +* builder:cleanjobs: Clean up old jobs +* builder:delete-image: Delete an image from the registry +* builder:get-image: Get an image from the registry +* builder:list-catalogs: List catalogs in the registry +* builder:list-images: List images in a specific catalog +* builder:logs: Show logs of the last build job +* builder:send: Send the build to the server +* builder:updatetoml: Update the buildkitd.toml file config map +``` + +## Examples + +### Build a custom runtime + +`task builder:send SOURCE=apache/openserverless-runtime-python:v3.13-2506091954 TARGET=devel:python3.13-custom KIND=python REQUIREMENTS=$(base64 -i deploy/samples/requirements.txt)` + +### List images for the user + +`task builder:list-images CATALOG=devel` + +### Delete an image for the user + +`task builder:delete-image IMAGE=devel:alpine` + +# Useful Links + +- https://crazymax.dev/buildkit/user-guides/rootless-mode/ +- https://www.linkedin.com/pulse/kubernetes-v133-user-namespaces-revolutionizing-false-rodrigo-mqoif/ +- https://chatgpt.com/c/689c9b5b-1d3c-8333-9f25-19d016fdacd0 +- https://kubernetes.io/docs/concepts/workloads/pods/user-namespaces/ \ No newline at end of file diff --git a/olaris-api/api-template.yaml b/olaris-api/api-template.yaml index 582019b..ced1468 100644 --- a/olaris-api/api-template.yaml +++ b/olaris-api/api-template.yaml @@ -1,6 +1,4 @@ --- -# Source: openwhisk/templates/couchdb-init-cm.yaml -# # Licensed to the Apache Software Foundation (ASF) under one or more # contributor license agreements. See the NOTICE file distributed with # this work for additional information regarding copyright ownership. @@ -15,9 +13,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# -# Kind deployment example -# --- apiVersion: apps/v1 kind: StatefulSet @@ -41,7 +36,7 @@ spec: containers: - name: nuvolaris-system-api image: ${IMAGES_SYSTEMAPI} - imagePullPolicy: IfNotPresent + imagePullPolicy: Always command: ["./run.sh"] ports: - containerPort: 5000 diff --git a/olaris-api/nginx-template.yaml b/olaris-api/nginx-template.yaml index 2255702..7a8c986 100644 --- a/olaris-api/nginx-template.yaml +++ b/olaris-api/nginx-template.yaml @@ -1,5 +1,3 @@ -# Source: openwhisk/templates/couchdb-init-cm.yaml -# # Licensed to the Apache Software Foundation (ASF) under one or more # contributor license agreements. See the NOTICE file distributed with # this work for additional information regarding copyright ownership. @@ -14,9 +12,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# -# Kind deployment example -# --- apiVersion: networking.k8s.io/v1 kind: Ingress @@ -30,9 +25,9 @@ metadata: name: nuvolaris-system-api-ingress namespace: nuvolaris spec: + ingressClassName: nginx rules: - - # - host: ${SYS_API_HOSTNAME:-localhost} + - host: ${SYS_API_HOSTNAME:-localhost} http: paths: - backend: diff --git a/olaris-api/opsfile.yml b/olaris-api/opsfile.yml index e0c9902..68ce3ae 100644 --- a/olaris-api/opsfile.yml +++ b/olaris-api/opsfile.yml @@ -69,7 +69,7 @@ tasks: desc: undeploy the admin api ignore_error: true cmds: - - kubectl -n nuvolaris delete sts/nuvolaris-system-api ing/nuvolaris-system-api-ingress + - kubectl -n nuvolaris delete sts/nuvolaris-system-api ing/nuvolaris-system-api-ingress svc/nuvolaris-system-api - | echo "System API undeployed" diff --git a/olaris-api/k3s-template.yaml b/olaris-api/traefik-template.yaml similarity index 94% rename from olaris-api/k3s-template.yaml rename to olaris-api/traefik-template.yaml index 8950f25..dff4420 100644 --- a/olaris-api/k3s-template.yaml +++ b/olaris-api/traefik-template.yaml @@ -1,6 +1,3 @@ ---- -# Source: openwhisk/templates/couchdb-init-cm.yaml -# # Licensed to the Apache Software Foundation (ASF) under one or more # contributor license agreements. See the NOTICE file distributed with # this work for additional information regarding copyright ownership. @@ -15,9 +12,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# -# Kind deployment example -# --- apiVersion: networking.k8s.io/v1 kind: Ingress diff --git a/openserverless/__init__.py b/openserverless/__init__.py index 211968a..7ae1020 100644 --- a/openserverless/__init__.py +++ b/openserverless/__init__.py @@ -79,4 +79,5 @@ def log_request_info(): import openserverless.rest.api import openserverless.rest.auth +import openserverless.rest.build diff --git a/openserverless/common/kube_api_client.py b/openserverless/common/kube_api_client.py index 097fa2d..d718ba2 100644 --- a/openserverless/common/kube_api_client.py +++ b/openserverless/common/kube_api_client.py @@ -15,14 +15,15 @@ # specific language governing permissions and limitations # under the License. # +from datetime import time import requests as req import json import os import logging -from base64 import b64decode -from .validation import is_empty_arg +from base64 import b64decode, b64encode +from openserverless.common.utils import join_host_port from openserverless.config.app_config import AppConfig from openserverless.error.config_exception import ConfigException @@ -30,16 +31,6 @@ SERVICE_PORT_ENV_NAME = "KUBERNETES_SERVICE_PORT" SERVICE_TOKEN_FILENAME = "/var/run/secrets/kubernetes.io/serviceaccount/token" SERVICE_CERT_FILENAME = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt" - - -def _join_host_port(host, port): - template = "%s:%s" - host_requires_bracketing = ":" in host or "%" in host - if host_requires_bracketing: - template = "[%s]:%s" - return template % (host, port) - - class KubeApiClient: def __init__(self, environ=os.environ): @@ -49,11 +40,6 @@ def __init__(self, environ=os.environ): self._load_incluster_config() def _parse_b64(self, encoded_str): - """ - Decode b64 encoded string - param: encoded_str a Base64 encoded string - return: decoded string - """ try: return b64decode(encoded_str).decode() except: @@ -72,7 +58,7 @@ def _load_incluster_config(self): ): raise ConfigException("Service host/port is not set.") - self.host = "https://" + _join_host_port( + self.host = "https://" + join_host_port( self._environ.get(SERVICE_HOST_ENV_NAME), self._environ.get(SERVICE_PORT_ENV_NAME), ) @@ -223,3 +209,301 @@ def update_whisk_user(self, whisk_user_dict, namespace="nuvolaris"): except Exception as ex: logging.error(f"update_whisk_user {ex}") return False + + def get_config_map(self, cm_name, namespace="nuvolaris"): + """ + Get a ConfigMap by name. + :param cm_name: Name of the ConfigMap. + :param namespace: Namespace where the ConfigMap is located. + :return: The ConfigMap data or None if not found. + """ + url = f"{self.host}/api/v1/namespaces/{namespace}/configmaps/{cm_name}" + headers = {"Authorization": self.token} + + try: + logging.info(f"GET request to {url}") + response = req.get(url, headers=headers, verify=self.ssl_ca_cert) + + if response.status_code == 200: + logging.debug( + f"GET to {url} succeeded with {response.status_code}. Body {response.text}" + ) + return json.loads(response.text) + + logging.error( + f"GET to {url} failed with {response.status_code}. Body {response.text}" + ) + return None + except Exception as ex: + logging.error(f"get_config_map {ex}") + return None + + def post_config_map(self, cm_name, file_or_dir, namespace="nuvolaris"): + """ + Create a ConfigMap from a file or directory. + :param cm_name: Name of the ConfigMap. + :param file_or_dir: Path to the file or directory containing the data. + :param namespace: Namespace where the ConfigMap will be created. + :return: The created ConfigMap or None if failed. + """ + if not os.path.exists(file_or_dir): + raise ConfigException(f"File or directory {file_or_dir} does not exist.") + + configmap_data = {} + if os.path.isfile(file_or_dir): + with open(file_or_dir, "r") as f: + configmap_data[os.path.basename(file_or_dir)] = f.read() + elif os.path.isdir(file_or_dir): + for filename in os.listdir(file_or_dir): + filepath = os.path.join(file_or_dir, filename) + if os.path.isfile(filepath): + with open(filepath, "r") as f: + configmap_data[filename] = f.read() + + configmap_manifest = { + "apiVersion": "v1", + "kind": "ConfigMap", + "metadata": { + "name": cm_name + }, + "data": configmap_data + } + + url = f"{self.host}/api/v1/namespaces/{namespace}/configmaps" + headers = {"Authorization": self.token, "Content-Type": "application/json"} + + try: + logging.info(f"POST request to {url}") + response = None + response = req.post(url, data=json.dumps(configmap_manifest), headers=headers, verify=self.ssl_ca_cert) + + if response.status_code in [200, 201, 202]: + logging.debug( + f"POST to {url} succeeded with {response.status_code}. Body {response.text}" + ) + return json.loads(response.text) + + logging.error( + f"POST to {url} failed with {response.status_code}. Body {response.text}" + ) + return None + except Exception as ex: + logging.error(f"post_config_map {ex}") + return None + + def delete_config_map(self, cm_name, namespace="nuvolaris"): + """ + Delete a ConfigMap by name. + :param cm_name: Name of the ConfigMap to delete. + :param namespace: Namespace where the ConfigMap is located. + :return: True if deletion was successful, False otherwise. + """ + url = f"{self.host}/api/v1/namespaces/{namespace}/configmaps/{cm_name}" + headers = {"Authorization": self.token} + + try: + logging.info(f"DELETE request to {url}") + response = None + response = req.delete(url, headers=headers, verify=self.ssl_ca_cert) + + if response.status_code in [200, 202]: + logging.debug( + f"DELETE to {url} succeeded with {response.status_code}. Body {response.text}" + ) + return True + + logging.error( + f"DELETE to {url} failed with {response.status_code}. Body {response.text}" + ) + return False + except Exception as ex: + logging.error(f"delete_config_map {ex}") + return False + + def get_secret(self, secret_name, namespace="nuvolaris"): + """ + Get a Kubernetes secret by name. + :param secret_name: Name of the secret. + :param namespace: Namespace where the secret is located. + :return: The secret data or None if not found. + """ + url = f"{self.host}/api/v1/namespaces/{namespace}/secrets/{secret_name}" + headers = {"Authorization": self.token} + + try: + logging.info(f"GET request to {url}") + response = req.get(url, headers=headers, verify=self.ssl_ca_cert) + + if response.status_code == 200: + logging.debug( + f"GET to {url} succeeded with {response.status_code}. Body {response.text}" + ) + return json.loads(response.text) + + logging.error( + f"GET to {url} failed with {response.status_code}. Body {response.text}" + ) + return None + except Exception as ex: + logging.error(f"get_secret {ex}") + return None + + def post_secret(self, secret_name, secret_data, namespace="nuvolaris"): + """ + Create a Kubernetes secret. + :param secret_name: Name of the secret. + :param secret_data: Dictionary containing the secret data. + :param namespace: Namespace where the secret will be created. + :return: The created secret or None if failed. + """ + url = f"{self.host}/api/v1/namespaces/{namespace}/secrets" + headers = {"Authorization": self.token, "Content-Type": "application/json"} + + secret_manifest = { + "apiVersion": "v1", + "kind": "Secret", + "metadata": {"name": secret_name}, + "data": {k: b64encode(v.encode()).decode() for k, v in secret_data.items()}, + "type": "Opaque" + } + + try: + logging.info(f"POST request to {url}") + response = req.post(url, headers=headers, json=secret_manifest, verify=self.ssl_ca_cert) + + if response.status_code in [200, 201]: + logging.debug( + f"POST to {url} succeeded with {response.status_code}. Body {response.text}" + ) + return json.loads(response.text) + + logging.error( + f"POST to {url} failed with {response.status_code}. Body {response.text}" + ) + return None + except Exception as ex: + logging.error(f"post_secret {ex}") + return None + + def delete_secret(self, secret_name, namespace="nuvolaris"): + """ + Delete a Kubernetes secret. + :param secret_name: Name of the secret to delete. + :param namespace: Namespace where the secret is located. + :return: True if deletion was successful, False otherwise. + """ + url = f"{self.host}/api/v1/namespaces/{namespace}/secrets/{secret_name}" + headers = {"Authorization": self.token} + + try: + logging.info(f"DELETE request to {url}") + response = req.delete(url, headers=headers, verify=self.ssl_ca_cert) + + if response.status_code in [200, 202]: + logging.debug( + f"DELETE to {url} succeeded with {response.status_code}. Body {response.text}" + ) + return True + + logging.error( + f"DELETE to {url} failed with {response.status_code}. Body {response.text}" + ) + return False + except Exception as ex: + logging.error(f"delete_secret {ex}") + return False + + def post_job(self, job_name, job_manifest, namespace="nuvolaris"): + """ + Create a Kubernetes job. + :param job_name: Name of the job. + :param job_manifest: Dictionary containing the job manifest. + :param namespace: Namespace where the job will be created. + :return: The created job or None if failed. + """ + url = f"{self.host}/apis/batch/v1/namespaces/{namespace}/jobs" + headers = {"Authorization": self.token} + try: + logging.info(f"POST request to {url}") + response = None + response = req.post(url, headers=headers, json=job_manifest, verify=self.ssl_ca_cert) + if response.status_code in [200, 201, 202]: + logging.debug( + f"POST to {url} succeeded with {response.status_code}. Body {response.text}" + ) + return json.loads(response.text) + logging.error( + f"POST to {url} failed with {response.status_code}. Body {response.text}" + ) + return None + except Exception as ex: + logging.error(f"post_job {ex}") + return None + + def get_pod_by_job_name(self, job_name, namespace="nuvolaris"): + """ + Get the pod name associated with a job by its name. + :param job_name: Name of the job. + :param namespace: Namespace where the job is located. + :return: The pod name if found, None otherwise. + """ + url = f"{self.host}/api/v1/namespaces/{namespace}/pods" + headers = {"Authorization": self.token} + try: + while True: + resp = req.get(url, headers=headers, verify=self.ssl_ca_cert) + + if not response.status_code in [200, 202]: + logging.error( + f"POST to {url} failed with {response.status_code}. Body {response.text}" + ) + return None + + logging.debug( + f"POST to {url} succeeded with {response.status_code}. Body {response.text}" + ) + + pods = resp.json()["items"] + for pod in pods: + labels = pod["metadata"].get("labels", {}) + if labels.get("job-name") == job_name: + return pod["metadata"]["name"] + time.sleep(1) + + except Exception as ex: + logging.error(f"get_pod_by_job_name {ex}") + return None + + def stream_pod_logs(self, pod_name, namespace="nuvolaris"): + """ + Stream logs from a specific pod. + :param pod_name: Name of the pod to stream logs from. + :param namespace: Namespace where the pod is located. + """ + url = f"{self.host}/api/v1/namespaces/{namespace}/pods/{pod_name}/log?follow=true" + headers = {"Authorization": self.token} + with req.get(url, headers=headers, verify=self.ssl_ca_cert, stream=True) as r: + for line in r.iter_lines(): + if line: + print(line.decode()) + + def check_job_status(self, job_name, namespace="nuvolaris"): + """ + Check the status of a job by its name. + :param job_name: Name of the job to check. + :param namespace: Namespace where the job is located. + :return: True if the job has succeeded, False otherwise. + """ + url = f"{self.host}/apis/batch/v1/namespaces/{namespace}/jobs/{job_name}" + headers = {"Authorization": self.token} + try: + resp = req.get(url, headers=headers, verify=self.ssl_ca_cert) + resp.raise_for_status() + status = resp.json()["status"] + if status.get("succeeded", 0) > 0: + return True + else: + return False + except Exception as ex: + logging.error(f"check_job_status {ex}") + return False \ No newline at end of file diff --git a/openserverless/common/openwhisk_authorize.py b/openserverless/common/openwhisk_authorize.py index a1d6b50..cc9c2d8 100644 --- a/openserverless/common/openwhisk_authorize.py +++ b/openserverless/common/openwhisk_authorize.py @@ -45,16 +45,36 @@ def encode(self, username, password): return f"Basic {b64encode(username_password.encode()).decode()}" def _parse_b64(self, encoded_str): + """ + Parse a base64 encoded string and return the username and password. + If the string is not base64 encoded, it will try to split it by ':'. + Raises DecodeError if the string cannot be decoded or parsed. + >>> oa = OpenwhiskAuthorize() + >>> oa._parse_b64("dXNlcm5hbWU6cGFzc3dvcmQ=") + ('username', 'password') + >>> oa._parse_b64("username:password") + ('username', 'password') + >>> oa._parse_b64("invalid_base64_string") + Traceback (most recent call last): + ... + openserverless.error.api_error.DecodeError: authentication token does not seems to be b64 encoded + """ + username = None + password = None try: - username, password = b64decode(encoded_str).decode().split(":", 1) + decoded = b64decode(encoded_str) + + credentials = decoded.decode() + if credentials.count(":") != 1: + raise DecodeError("authentication token does not seems to be b64 encoded") + username, password = credentials.split(":", 1) except: # fallback in case the token is not bas64 encoded - username, password = encoded_str.split(":", 1) + if encoded_str.count(":") == 1: + username, password = encoded_str.split(":", 1) - if not username or not password: - raise DecodeError( - "authentication token does not seems to be b64 encoded" - ) + if not username or not password: + raise DecodeError("authentication token does not seems to be b64 encoded") return username, password @@ -62,6 +82,15 @@ def decode(self, encoded_str): """Decode an encrypted HTTP basic authentication string. Returns a tuple of the form (username, password), and raises a DecodeError exception if nothing could be decoded. + >>> oa = OpenwhiskAuthorize() + >>> oa.decode("Basic dXNlcm5hbWU6cGFzc3dvcmQ=") + ('username', 'password') + >>> oa.decode("dXNlcm5hbWU6cGFzc3dvcmQ=") + ('username', 'password') + >>> oa.decode("invalid_base64_string") + Traceback (most recent call last): + ... + openserverless.error.api_error.DecodeError: authentication token does not seems to be b64 encoded """ split = encoded_str.strip().split(" ") diff --git a/openserverless/common/utils.py b/openserverless/common/utils.py new file mode 100644 index 0000000..3931c63 --- /dev/null +++ b/openserverless/common/utils.py @@ -0,0 +1,90 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +def env_to_dict(user_data, key="env"): + """ + extract env from user_data and return it as a dict + + Keyword arguments: + key -- the key to extract the env from + + >>> env_to_dict({"env": [{"key": "VAR1", "value": "value1"}, {"key": "VAR2", "value": "value2"}]}) + {'VAR1': 'value1', 'VAR2': 'value2'} + >>> env_to_dict({"env": []}) + {} + >>> env_to_dict({"other_key": [{"key": "VAR1", "value": "value1"}]}, key="other_key") + {'VAR1': 'value1'} + >>> env_to_dict({"env": [{"key": "VAR1", "value": "value1"}]}, key="env") + {'VAR1': 'value1'} + >>> env_to_dict({"env": [{"key": "VAR1", "value": "value1"}, {"key": "VAR2", "value": "value2"}]}, key="non_existent_key") + {} + >>> env_to_dict({"env": [{"key": "VAR1", "value": "value1"}]}, key="env") + {'VAR1': 'value1'} + """ + body = {} + if key in user_data: + envs = list(user_data[key]) + else: + envs = [] + + for env in envs: + body[env['key']] = env['value'] + + return body + + +def dict_to_env(env): + """ + converts an env to a key/pair suitable for user_data storage + + >>> dict_to_env({"VAR1": "value1", "VAR2": "value2"}) + [{'key': 'VAR1', 'value': 'value1'}, {'key': 'VAR2', 'value': 'value2'}] + >>> dict_to_env({}) + [] + """ + body = [] + for key in env: + body.append({"key": key, "value": env[key]}) + + return body + +def join_host_port(host, port): + """ + Join host and port into a URL format. + >>> join_host_port("localhost", "8080") + 'localhost:8080' + >>> join_host_port("localhost", 8080) + 'localhost:8080' + >>> join_host_port("localhost", "80") + 'localhost:80' + >>> join_host_port("localhost", "abcd") + Traceback (most recent call last): + ... + ValueError: Port must be numeric + + """ + template = "%s:%s" + try: + port_int = int(port) + port = str(port_int) + except (ValueError, TypeError): + raise ValueError("Port must be numeric") + + host_requires_bracketing = ":" in host or "%" in host + if host_requires_bracketing: + template = "[%s]:%s" + return template % (host, port) \ No newline at end of file diff --git a/openserverless/common/validation.py b/openserverless/common/validation.py index 47fb5a8..f0a1655 100644 --- a/openserverless/common/validation.py +++ b/openserverless/common/validation.py @@ -21,6 +21,20 @@ def is_valid_username(username): """ Verifies the given username follows nuvolaris rule + >>> is_valid_username("bruno") + True + >>> is_valid_username("bruno123") + True + >>> is_valid_username("bruno-123") + False + >>> is_valid_username("bruno_123") + False + >>> is_valid_username("bruno@123") + False + >>> is_valid_username("bruno 123") + False + >>> is_valid_username("brun") + False """ pat = re.compile(r"^[a-z0-9]{5,60}(?:[a-z0-9])?$") if re.fullmatch(pat, username): @@ -35,6 +49,10 @@ def is_empty_arg(args, arg_name): param: args param: arg_name return: True if the argument is not contained in the input args array or if it is an empty string value + >>> is_empty_arg({"arg1": "value1"}, "arg1") + False + >>> is_empty_arg({"arg1": "value1"}, "arg2") + True """ if arg_name not in args: diff --git a/openserverless/impl/auth/auth_service.py b/openserverless/impl/auth/auth_service.py index 8138f46..d13fde7 100644 --- a/openserverless/impl/auth/auth_service.py +++ b/openserverless/impl/auth/auth_service.py @@ -27,74 +27,99 @@ USER_META_DBN = "users_metadata" + class AuthService: def __init__(self, environ=os.environ): self._environ = environ self.couch_db = CouchDB() self.kube_client = KubeApiClient() - def fetch_user_data(self, login: str): logging.info(f"searching for user {login} data") try: - selector = {"selector":{"login": {"$eq": login }}} + selector = {"selector": {"login": {"$eq": login}}} response = self.couch_db.find_doc(USER_META_DBN, json.dumps(selector)) - if(response['docs']): - docs = list(response['docs']) - if(len(docs) > 0): - return docs[0] - + if response["docs"]: + docs = list(response["docs"]) + if len(docs) > 0: + return docs[0] + logging.warning(f"OpenServerless metadata for user {login} not found!") return None except Exception as e: - logging.error(f"failed to query OpenServerless metadata for user {login}. Reason: {e}") + logging.error( + f"failed to query OpenServerless metadata for user {login}. Reason: {e}" + ) return None + + def env_to_dict(self, user_data, key="env"): + """ + extract env from user_data and return it as a dict + + Keyword arguments: + key -- the key to extract the env from + """ + body = {} + if key in user_data: + envs = list(user_data[key]) + else: + envs = [] + + for env in envs: + body[env['key']] = env['value'] + + return body def map_data(self, user_data): """ Map the internal nuvolaris user_data records to the auth response """ resp = {} - resp['login'] = user_data['login'] - resp['email'] = user_data['email'] + resp["login"] = user_data["login"] + resp["email"] = user_data["email"] - if 'env' in user_data: - resp['env'] = user_data['env'] + if "env" in user_data: + resp["env"] = user_data["env"] - if 'quota' in user_data: - resp['quota'] = user_data['quota'] + if "quota" in user_data: + resp["quota"] = user_data["quota"] - return resp + return resp def login(self, login, password): user_data = self.fetch_user_data(login) - if(user_data): - if bu.verify_password(password, user_data['password']): - #if(password == user_data['password']): + if user_data: + if bu.verify_password(password, user_data["password"]): + # if(password == user_data['password']): return res_builder.build_response_with_data(self.map_data(user_data)) else: logging.warning(f"password mismatch for user {login}") return res_builder.build_error_message(f"Invalid credentials", 401) else: - logging.warning(f"no user {login} found") - return res_builder.build_error_message(f"Invalid credentials", 401) - + logging.warning(f"no user {login} found") + return res_builder.build_error_message(f"Invalid credentials", 401) + + def update_password(self, login, old_password, new_password): user_data = self.fetch_user_data(login) - if(user_data): - if bu.verify_password(old_password, user_data['password']): - whisk_user = self.kube_client.get_whisk_user(user_data['login']) + if user_data: + if bu.verify_password(old_password, user_data["password"]): + whisk_user = self.kube_client.get_whisk_user(user_data["login"]) - whisk_user['spec']['password'] = new_password - #whisk_user['spec']['password_timestamp'] = datetime.now().isoformat() - self.kube_client.update_whisk_user(whisk_user) + whisk_user["spec"]["password"] = new_password + # whisk_user['spec']['password_timestamp'] = datetime.now().isoformat() + self.kube_client.update_whisk_user(whisk_user) - return res_builder.build_response_with_data({"status":"ok","message":"Password updated"}) + return res_builder.build_response_with_data( + {"status": "ok", "message": "Password updated"} + ) else: - return res_builder.build_error_message(f"password mismatch for user {login}", 401) + return res_builder.build_error_message( + f"password mismatch for user {login}", 401 + ) else: - return res_builder.build_error_message(f"no user {login} found", 401) + return res_builder.build_error_message(f"no user {login} found", 401) diff --git a/openserverless/impl/builder/build_service.py b/openserverless/impl/builder/build_service.py new file mode 100644 index 0000000..9f5d5b5 --- /dev/null +++ b/openserverless/impl/builder/build_service.py @@ -0,0 +1,269 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +import shutil +from openserverless.common.kube_api_client import KubeApiClient +import os +import uuid + +JOB_NAME = "build" +CM_NAME = "cm" + + +class BuildService: + def __init__(self, build_config, user_env=None): + self.build_config = build_config + + # A super userful Kube Api Client + self.kube_client = KubeApiClient() + + # generate a unique ID for the build + self.id = str(uuid.uuid4()) + + # define a unique ConfigMap and Job name based on the ID + self.cm = f"{CM_NAME}-{self.id}" + self.job_name = f"{JOB_NAME}-{self.id}" + + # user environment variables + self.user_env = user_env if user_env is not None else {} + + # define registry host + self.registry_host = self.get_registry_host() + + self.init() + + def init(self): + """ + Initialize the build service by creating the necessary ConfigMap. + """ + + # install the nuvolaris-buildkitd-conf ConfigMap if not present + cm = self.kube_client.get_config_map("nuvolaris-buildkitd-conf", namespace="nuvolaris") + if cm is None: + self.kube_client.post_config_map( + cm_name="nuvolaris-buildkitd-conf", + file_or_dir="deploy/buildkit/buildkitd.toml", + namespace="nuvolaris", + ) + + def get_registry_host(self): + """ + Retrieve the registry host + - firstly, check if the user environment has a registry host set + - otherwise retrieve the OpenServerless config map + - if not present use a default value + """ + registry_host = 'nuvolaris-registry-svc:5000' + if (self.user_env.get('REGISTRY_HOST') is not None): + return self.build_config.get('REGISTRY_HOST') + + ops_config_map = self.kube_client.get_config_map('config') + if ops_config_map is not None: + if 'annotations' in ops_config_map.get('metadata', {}): + annotations = ops_config_map['metadata']['annotations'] + if 'registry_host' in annotations: + registry_host = annotations['registry_host'] + + return registry_host + + + def create_docker_file(self) -> str: + """ + Create a Dockerfile in the current directory. + """ + source = self.build_config.get("source") + + dockerfile_content = f"FROM {source}\n" + if 'file' in self.build_config: + requirement_file = self.get_requirements_file_from_kind() + dockerfile_content += f"COPY ./{requirement_file} /tmp/{requirement_file}\n" + dockerfile_content += "RUN echo \"/bin/extend\"\n" + return dockerfile_content + + def get_requirements_file_from_kind(self) -> str: + """ + Get the requirements file based on the kind of the build. + """ + kind = self.build_config.get("kind") + if kind == 'python': + return 'requirements.txt' + elif kind == 'nodejs': + return 'package.json' + elif kind == 'php': + return 'composer.json' + elif kind == 'java': + return 'pom.xml' + elif kind == 'go': + return 'go.mod' + elif kind == 'ruby': + return 'Gemfile' + elif kind == 'dotnet': + return 'project.json' + else: + raise ValueError(f"Unsupported kind: {kind}") + + def build(self, image_name: str) -> str: + """ + Build the Docker image using the provided build configuration. + The build configuration should include the source, target, and kind. + """ + import tempfile + import base64 + + tmpdirname = tempfile.mkdtemp() + if 'file' in self.build_config: + # decode base64 self.build_config.get('file') + requirements = base64.b64decode(self.build_config.get('file')).decode('utf-8') + + requirement_file = self.get_requirements_file_from_kind() + with open(os.path.join(tmpdirname, requirement_file), 'w') as f: + f.write(requirements) + + dockerfile_path = os.path.join(tmpdirname, "Dockerfile") + with open(dockerfile_path, "w") as dockerfile: + dockerfile.write(self.create_docker_file()) + + # check if the unzipped directory contains a Dockerfile and is not empty. + if not self.check_unzip_dir(tmpdirname): + return None + + # Create a ConfigMap for the build context + cm = self.kube_client.post_config_map( + cm_name=self.cm, + file_or_dir=tmpdirname, + namespace="nuvolaris", + ) + + shutil.rmtree(tmpdirname) + + if not cm: + return None + + # retrieve credentials to access the registry + # + + job_template = self.create_build_job(image_name) + job = self.kube_client.post_job(self.job_name, job_template) + if not job: + return None + + return job + + + def check_unzip_dir(self, unzip_dir: str) -> bool: + """ + Check if the unzipped directory contains a Dockerfile and is not empty.""" + if not os.path.exists(unzip_dir): + return False + + # Check if the directory contains a Dockerfile + dockerfile_path = os.path.join(unzip_dir, "Dockerfile") + if not os.path.exists(dockerfile_path): + return False + + return True + + def create_build_job(self, image_name: str) -> dict: + """Create a Kubernetes job manifest for building the Docker image.""" + registry_image_name = f"{self.registry_host}/{image_name}" + + # --- MANIFEST DEL JOB --- + job_manifest = { + "apiVersion": "batch/v1", + "kind": "Job", + "metadata": {"name": self.job_name}, + "spec": { + "backoffLimit": 0, + "template": { + "spec": { + "restartPolicy": "Never", + "volumes": [ + { + "name": "nuvolaris-buildkitd-conf", + "configMap": {"name": "nuvolaris-buildkitd-conf"}, + }, + { + "name": "build-context-vol", + "configMap": {"name": self.cm}, + }, + {"name": "workspace", "emptyDir": {}}, + {"name": "img-cache", "emptyDir": {}}, + {"name": "cdi-etc", "emptyDir": {}}, + {"name": "cdi-run", "emptyDir": {}}, + {"name": "cdi-buildkit", "emptyDir": {}}, + { + "name": "docker-config", + "secret": { + "secretName": "registry-pull-secret", + "items": [ + { + "key": ".dockerconfigjson", + "path": "config.json", + } + ], + }, + }, + ], + "initContainers": [ + { + "name": "copy-build-context", + "image": "busybox:1.36", + "command": [ + "sh", "-c", "cp -rvL /configmap/* /workspace/ && cat /workspace/Dockerfile", + ], + "volumeMounts": [ + { "name": "build-context-vol", "mountPath": "/configmap", }, + { "name": "workspace", "mountPath": "/workspace" }, + ], + } + ], + "containers": [ + { + "name": "buildkit", + "image": "moby/buildkit:master-rootless", + "command": ["sh", "-c"], + "args": [ + "rootlesskit buildkitd --config /config/buildkitd.toml & sleep 3 && " + f"buildctl build --frontend=dockerfile.v0 --local context=/workspace --local dockerfile=/workspace --output=type=image,name={registry_image_name},push=true" + ], + "securityContext": { + "runAsUser": 1000, + "runAsGroup": 1000, + "allowPrivilegeEscalation": True, + "privileged": True, + "seccompProfile": { "type": "Unconfined" } + }, + "env": [ + { "name": "BUILDKIT_ROOTLESS", "value": "1" } + ], + "volumeMounts": [ + { "name": "nuvolaris-buildkitd-conf", "mountPath": "/config" }, + { "name": "workspace", "mountPath": "/workspace" }, + { "name": "docker-config", "mountPath": "/home/user/.docker" }, + { "name": "img-cache", "mountPath": "/tmp" }, + { "name": "cdi-etc", "mountPath": "/etc/cdi" }, + { "name": "cdi-run", "mountPath": "/var/run/cdi" }, + { "name": "cdi-buildkit", "mountPath": "/etc/buildkit/cdi" }, + ], + } + ], + } + }, + }, + } + + return job_manifest diff --git a/openserverless/impl/onboard/user_validation.py b/openserverless/impl/onboard/user_validation.py index af5b06b..354c8b3 100644 --- a/openserverless/impl/onboard/user_validation.py +++ b/openserverless/impl/onboard/user_validation.py @@ -22,8 +22,9 @@ import openserverless.common.response_builder as res_builder from openserverless.common.kube_api_client import KubeApiClient + class UserValidation: - + def __init__(self, environ=os.environ): self._environ = environ self._kube_client = KubeApiClient() @@ -36,16 +37,26 @@ def validate(self, namespace): try: if not validation.is_valid_username(namespace): - return res_builder.build_error_message(message=f"Account namespace {namespace} is not valid. ", status_code=400) - - #check that there is no wsk user already existing with the same namespace + return res_builder.build_error_message( + message=f"Account namespace {namespace} is not valid. ", + status_code=400, + ) + + # check that there is no wsk user already existing with the same namespace existing_whisk_user = self._kube_client.get_whisk_user(namespace) if existing_whisk_user: - return res_builder.build_error_message(message=f"Namespace {namespace} already exists on domain.", status_code=409) + return res_builder.build_error_message( + message=f"Namespace {namespace} already exists on domain.", + status_code=409, + ) - return res_builder.build_response_message(f"Username {namespace} is valid and available") + return res_builder.build_response_message( + f"Username {namespace} is valid and available" + ) except Exception as ex: logging.error(ex) - return res_builder.build_response_message("Un-expected error detected attempting to setup you free account. If problem persists please get in touch with us info@nuvolaris.io") \ No newline at end of file + return res_builder.build_response_message( + "Un-expected error detected attempting to setup you free account. If problem persists please get in touch with us info@nuvolaris.io" + ) diff --git a/openserverless/rest/api.py b/openserverless/rest/api.py index 7b7e66a..16fbd83 100644 --- a/openserverless/rest/api.py +++ b/openserverless/rest/api.py @@ -17,11 +17,8 @@ # from openserverless import app -from http import HTTPStatus - import openserverless.common.response_builder as res_builder - @app.route('/system/info') def info(): """ diff --git a/openserverless/rest/auth.py b/openserverless/rest/auth.py index 6cf3dcf..33a916c 100644 --- a/openserverless/rest/auth.py +++ b/openserverless/rest/auth.py @@ -35,7 +35,7 @@ def password(login, **kwargs): summary: Login an OpenServerless user using login/password payload operationId: patchOpsUser security: - - openwhiskBasicAuth: [] + - openwhiskBasicAuth: [] consumes: - application/json definitions: diff --git a/openserverless/rest/build.py b/openserverless/rest/build.py new file mode 100644 index 0000000..f19a186 --- /dev/null +++ b/openserverless/rest/build.py @@ -0,0 +1,118 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +from openserverless import app +from http import HTTPStatus +from flask import request + +import openserverless.common.response_builder as res_builder +from openserverless.common.utils import env_to_dict +from openserverless.error.api_error import AuthorizationError +from openserverless.impl.builder.build_service import BuildService +from openserverless.common.openwhisk_authorize import OpenwhiskAuthorize + +@app.route('/system/api/v1/build', methods=['POST']) +def build(): + """ + Build Endpoint + --- + tags: + - Build + summary: Build an image using the provided source, target, and kind. + description: This endpoint triggers a build process based on the provided parameters. + operationId: buildImage + security: + - openwhiskBasicAuth: [] + consumes: + - application/json + parameters: + - in: body + name: BuildRequest + required: true + schema: + type: object + properties: + source: + type: string + description: Source for the build + target: + type: string + description: Target for the build + kind: + type: string + description: Kind of the build + responses: + 200: + description: Build process initiated successfully. + schema: + $ref: '#/definitions/Message' + 400: + description: Bad Request. Missing or invalid parameters. + schema: + $ref: '#/definitions/Message' + 401: + description: Unauthorized. Invalid or missing authorization header. + schema: + $ref: '#/definitions/Message' + 500: + description: Internal Server Error. Build process failed. + schema: + $ref: '#/definitions/Message' + """ + + normalized_headers = {key.lower(): value for key, value in request.headers.items()} + auth_header = normalized_headers.get('authorization', None) + + if auth_header is None: + return res_builder.build_error_message("Missing authorization header", 401) + + oa = OpenwhiskAuthorize() + try: + user_data = oa.login(auth_header) + env = env_to_dict(user_data) + if env is None: + return res_builder.build_error_message("User environment not found", status_code=HTTPStatus.UNAUTHORIZED) + + if (request.json is None): + return res_builder.build_error_message("No JSON payload provided for build.", status_code=HTTPStatus.BAD_REQUEST) + + json_data = request.json + if 'source' not in json_data: + return res_builder.build_error_message("No source provided for build.", status_code=HTTPStatus.BAD_REQUEST) + if 'target' not in json_data: + return res_builder.build_error_message("No target provided for build.", status_code=HTTPStatus.BAD_REQUEST) + if 'kind' not in json_data: + return res_builder.build_error_message("No kind provided for build.", status_code=HTTPStatus.BAD_REQUEST) + + + # validate the target + target = json_data.get('target') + target_user = str(target).split(':')[0] + if user_data.get('login') != target_user: + return res_builder.build_error_message("Invalid target for the build.", status_code=HTTPStatus.BAD_REQUEST) + + + build_service = BuildService(build_config=json_data, user_env=env) + build_success = build_service.build(json_data.get('target')) # Replace with your desired image name + + if not build_success: + return res_builder.build_error_message("Build process failed.", status_code=HTTPStatus.INTERNAL_SERVER_ERROR) + + return res_builder.build_response_message("Build process initiated successfully.", status_code=HTTPStatus.OK) + + except AuthorizationError: + return res_builder.build_error_message("Invalid authorization", 401) \ No newline at end of file