feat: improve build service with init container cleanup and dev tooling

.licenserc.yaml

@@ -33,4 +33,5 @@ header:
     - '**/*.service'
     - '**/*.txt'
     - 'uv.lock'
-    - '.env.example'
+    - '.env.example'
+    - '.vscode/*.json'

.vscode/launch.json

@@ -5,8 +5,10 @@
             "name": "Debug OpenServerless Admin API",
             "type": "debugpy",
             "request": "launch",
-            "module": "openserverless"            
+            "module": "openserverless",
+            "preLaunchTask": "Start Port Forwards",
+            "postDebugTask": "Stop Port Forwards"
         }
-        
+
     ]
 }

.vscode/start-port-forwards.sh

@@ -0,0 +1,60 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+set -e
+
+# Log start time for debugging
+echo "$(date): Starting port forwards..." >> /tmp/pf-start.log
+
+# Clean up any existing port forwards first
+pkill -f 'kubectl port-forward -n nuvolaris registry-0' 2>/dev/null || true
+pkill -f 'kubectl -n nuvolaris port-forward couchdb-0' 2>/dev/null || true
+rm -f /tmp/pf-registry.pid /tmp/pf-couchdb.pid /tmp/pf-registry.log /tmp/pf-couchdb.log
+sleep 1
+
+# Start port forwards in background with nohup to detach from terminal
+nohup kubectl port-forward -n nuvolaris registry-0 5000:5000 > /tmp/pf-registry.log 2>&1 &
+REGISTRY_PID=$!
+echo $REGISTRY_PID > /tmp/pf-registry.pid
+echo "$(date): Registry PID: $REGISTRY_PID" >> /tmp/pf-start.log
+
+nohup kubectl -n nuvolaris port-forward couchdb-0 5984:5984 > /tmp/pf-couchdb.log 2>&1 &
+COUCHDB_PID=$!
+echo $COUCHDB_PID > /tmp/pf-couchdb.pid
+echo "$(date): CouchDB PID: $COUCHDB_PID" >> /tmp/pf-start.log
+
+# Disown the processes so they don't get killed when the script exits
+disown -a
+
+# Wait a moment for port forwards to establish
+sleep 2
+
+# Verify processes are still running
+if ps -p $REGISTRY_PID > /dev/null 2>&1; then
+    echo "$(date): Registry port-forward is running" >> /tmp/pf-start.log
+else
+    echo "$(date): WARNING: Registry port-forward died!" >> /tmp/pf-start.log
+fi
+
+if ps -p $COUCHDB_PID > /dev/null 2>&1; then
+    echo "$(date): CouchDB port-forward is running" >> /tmp/pf-start.log
+else
+    echo "$(date): WARNING: CouchDB port-forward died!" >> /tmp/pf-start.log
+fi
+
+echo "Port forwards started: registry=$REGISTRY_PID, couchdb=$COUCHDB_PID"

.vscode/tasks.json

@@ -0,0 +1,48 @@
+{
+    "version": "2.0.0",
+    "tasks": [
+        {
+            "label": "Start Port Forwards",
+            "type": "shell",
+            "command": "${workspaceFolder}/.vscode/start-port-forwards.sh",
+            "isBackground": true,
+            "problemMatcher": {
+                "pattern": {
+                    "regexp": "^Port forwards started: (.*)$",
+                    "message": 1
+                },
+                "background": {
+                    "activeOnStart": true,
+                    "beginsPattern": "^.*$",
+                    "endsPattern": "^Port forwards started:.*$"
+                }
+            },
+            "presentation": {
+                "echo": false,
+                "reveal": "never",
+                "focus": false,
+                "panel": "shared",
+                "showReuseMessage": false,
+                "clear": false
+            }
+        },
+        {
+            "label": "Stop Port Forwards",
+            "type": "shell",
+            "command": "test -f /tmp/pf-registry.pid && kill $(cat /tmp/pf-registry.pid) 2>/dev/null; test -f /tmp/pf-couchdb.pid && kill $(cat /tmp/pf-couchdb.pid) 2>/dev/null; rm -f /tmp/pf-registry.pid /tmp/pf-registry.log /tmp/pf-couchdb.pid /tmp/pf-couchdb.log; pkill -f 'kubectl port-forward -n nuvolaris registry-0' 2>/dev/null; pkill -f 'kubectl -n nuvolaris port-forward couchdb-0' 2>/dev/null; echo 'Port forwards stopped'; exit 0",
+            "presentation": {
+                "echo": true,
+                "reveal": "silent",
+                "focus": false,
+                "panel": "dedicated",
+                "showReuseMessage": false
+            },
+            "options": {
+                "shell": {
+                    "executable": "/bin/bash",
+                    "args": ["-c"]
+                }
+            }
+        }
+    ]
+}

.vscode/test-task.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+echo "Testing from VSCode task..."
+echo "PATH: $PATH"
+echo "SHELL: $SHELL"
+echo "PWD: $PWD"
+which kubectl
+kubectl version --client --short 2>/dev/null || kubectl version --client 2>/dev/null

TaskfileDev.yml

@@ -52,7 +52,25 @@ tasks:
       - sed -i '' 's/^KUBERNETES_SERVICE_PORT=.*/KUBERNETES_SERVICE_PORT={{.KUB_SERVICE_PORT}}/' .env
       - sed -i '' 's/^REGISTRY_PASS=.*/REGISTRY_PASS={{.REGISTRY_PASS}}/' .env
       - sed -i '' 's/^COUCHDB_ADMIN_PASSWORD=.*/COUCHDB_ADMIN_PASSWORD={{.COUCHDB_PASS}}/' .env
+      - task: setup-secret
 
+  setup-secret:
+    desc: "Create docker-registry secret for development using credentials from .env"
+    dotenv: ['.env']
+    cmds:
+      - kubectl delete secret registry-pull-secret-dev -n nuvolaris 2>/dev/null || true
+      - kubectl create secret docker-registry registry-pull-secret-dev -n nuvolaris --docker-server=${REGISTRY_HOST} --docker-username=${REGISTRY_USER} --docker-password=${REGISTRY_PASS}
+      - echo "Secret 'registry-pull-secret-dev' created successfully in namespace 'nuvolaris'"
+      - ops env add REGISTRY_SECRET=registry-pull-secret-dev
+      - echo "Environment variable REGISTRY_SECRET set to 'registry-pull-secret-dev'"
+      - |
+        if ops env list | rg REGISTRY_SECRET > /dev/null 2>&1; then
+          echo "✓ Verified: REGISTRY_SECRET is configured"
+          ops env list | rg REGISTRY_SECRET
+        else
+          echo "⚠ Warning: REGISTRY_SECRET not found in ops env list"
+        fi
+
   run:
     desc: |
       Run the admin api locally, using configuration from .env file

openserverless/common/kube_api_client.py

@@ -15,7 +15,7 @@
 # specific language governing permissions and limitations
 # under the License.
 #
-from datetime import time
+import time
 import requests as req
 import json
 import os
@@ -452,9 +452,10 @@ def delete_secret(self, secret_name: str, namespace="nuvolaris"):
             logging.error(f"delete_secret {ex}")
             return False
 
-    def get_jobs(self, name_filter: str = None, namespace="nuvolaris"):
+    def get_jobs(self, name_filter: str | None = None, namespace="nuvolaris"):
         """
         Get all Kubernetes jobs in a specific namespace.
+        :param name_filter: Optional filter to match job names.
         :param namespace: Namespace to list jobs from.
         :return: List of jobs or None if failed.
         """
@@ -512,7 +513,7 @@ def delete_job(self, job_name: str, namespace="nuvolaris"):
             logging.error(f"delete_job {ex}")
             return False
 
-    def post_job(self, job_manifest: json, namespace="nuvolaris"):
+    def post_job(self, job_manifest: dict, namespace="nuvolaris"):
         """
         Create a Kubernetes job.
         :param job_manifest: Dictionary containing the job manifest. 
@@ -601,4 +602,104 @@ def check_job_status(self, job_name: str, namespace="nuvolaris"):
                 return False
         except Exception as ex:
             logging.error(f"check_job_status {ex}")
-            return False
+            return False
+
+    def get_pod(self, pod_name: str, namespace="nuvolaris"):
+        """
+        Get pod details by name.
+        :param pod_name: Name of the pod.
+        :param namespace: Namespace where the pod is located.
+        :return: Pod object or None if not found.
+        """
+        url = f"{self.host}/api/v1/namespaces/{namespace}/pods/{pod_name}"
+        headers = {"Authorization": self.token}
+
+        try:
+            logging.info(f"GET request to {url}")
+            response = req.get(url, headers=headers, verify=self.ssl_ca_cert)
+
+            if response.status_code == 200:
+                logging.debug(
+                    f"GET to {url} succeeded with {response.status_code}. Body {response.text}"
+                )
+                return json.loads(response.text)
+
+            logging.error(
+                f"GET to {url} failed with {response.status_code}. Body {response.text}"
+            )
+            return None
+        except Exception as ex:
+            logging.error(f"get_pod {ex}")
+            return None
+
+    def wait_for_init_container_completion(self, job_name: str, init_container_name: str, namespace="nuvolaris", timeout_seconds=300):
+        """
+        Wait for a specific init container in a job's pod to complete (successfully or with error).
+        :param job_name: Name of the job.
+        :param init_container_name: Name of the init container to wait for.
+        :param namespace: Namespace where the job is located.
+        :param timeout_seconds: Maximum time to wait in seconds (default: 300 = 5 minutes).
+        :return: True if init container completed (success or error), False if timeout or other failure.
+        """
+        import time
+
+        start_time = time.time()
+        pod_name = None
+
+        logging.info(f"Waiting for init container '{init_container_name}' in job '{job_name}' to complete")
+
+        # First, wait for the pod to be created
+        while time.time() - start_time < timeout_seconds:
+            pod_name = self.get_pod_by_job_name(job_name, namespace)
+            if pod_name:
+                logging.info(f"Found pod '{pod_name}' for job '{job_name}'")
+                break
+            logging.debug(f"Pod for job '{job_name}' not yet created, waiting...")
+            time.sleep(2)
+
+        if not pod_name:
+            logging.error(f"Timeout waiting for pod to be created for job '{job_name}'")
+            return False
+
+        # Now wait for the init container to complete
+        while time.time() - start_time < timeout_seconds:
+            pod = self.get_pod(pod_name, namespace)
+
+            if not pod:
+                logging.error(f"Failed to get pod '{pod_name}'")
+                return False
+
+            # Check init container status
+            init_container_statuses = pod.get("status", {}).get("initContainerStatuses", [])
+
+            for status in init_container_statuses:
+                if status.get("name") == init_container_name:
+                    state = status.get("state", {})
+
+                    # Check if terminated (completed or failed)
+                    if "terminated" in state:
+                        terminated = state["terminated"]
+                        exit_code = terminated.get("exitCode", -1)
+                        reason = terminated.get("reason", "Unknown")
+
+                        if exit_code == 0:
+                            logging.info(f"Init container '{init_container_name}' completed successfully")
+                        else:
+                            logging.warning(f"Init container '{init_container_name}' terminated with exit code {exit_code}, reason: {reason}")
+
+                        return True
+
+                    # Check if still running
+                    if "running" in state:
+                        logging.debug(f"Init container '{init_container_name}' is still running")
+
+                    # Check if waiting
+                    if "waiting" in state:
+                        waiting = state["waiting"]
+                        reason = waiting.get("reason", "Unknown")
+                        logging.debug(f"Init container '{init_container_name}' is waiting, reason: {reason}")
+
+            time.sleep(2)
+
+        logging.error(f"Timeout waiting for init container '{init_container_name}' to complete")
+        return False

openserverless/common/response_builder.py

@@ -19,36 +19,56 @@
 
 
 def build_error_message(
-    message: str, status_code=400, headers={"Content-Type": "application/json"}
+    message: str, status_code=400, headers=None
 ):
+    if headers is None:
+        headers = {"Content-Type": "application/json"}
     return make_response(
         jsonify({"message": message, "status": "ko"}), status_code, headers
     )
 
 
 def build_response_message(
-    message: str, status_code=200, headers={"Content-Type": "application/json"}
+    message: str, data=None, status_code=200, headers=None
 ):
+    if headers is None:
+        headers = {"Content-Type": "application/json"}
+
+    payload = {"message": message, "status": "ok"}
+    if data:
+        # If caller passed a dict, merge into payload. Otherwise attach under 'data'.
+        if isinstance(data, dict):
+            payload.update(data)
+        else:
+            payload["data"] = data
+
     return make_response(
-        jsonify({"message": message, "status": "ok"}), status_code, headers
+        jsonify(payload), status_code, headers
     )
 
 
 def build_response_with_data(
-    data, status_code=200, headers={"Content-Type": "application/json"}
+    data, status_code=200, headers=None
 ):
+    if headers is None:
+        headers = {"Content-Type": "application/json"}
+
     if isinstance(data, dict):
         return make_response(jsonify(data), status_code, headers)
     return make_response(data, status_code, headers)
 
 
 def build_response_raw(
-    message: str, status_code=200, headers={"Content-Type": "application/json"}
+    message: str, status_code=200, headers=None
 ):
+    if headers is None:
+        headers = {"Content-Type": "application/json"}
     return make_response(message, status_code, headers)
 
 
 def build_error_raw(
-    message: str, status_code=400, headers={"Content-Type": "application/json"}
+    message: str, status_code=400, headers=None
 ):
+    if headers is None:
+        headers = {"Content-Type": "application/json"}
     return make_response(message, status_code, headers)