Synergize Gaiaflow

.github/workflows/unittest.yml

@@ -34,10 +34,11 @@ jobs:
 
       - name: Install and Test
         run:
-          pixi run test --cov=gaiaflow --cov-report=xml
+          pixi run test --cov=gaiaflow --cov-branch --cov-report=xml
 
       - name: Upload coverage reports to Codecov
-        uses: codecov/codecov-action@v4
+        uses: codecov/codecov-action@v5
         with:
           verbose: true
-          token: ${{ secrets.CODECOV_TOKEN }}
+          token: ${{ secrets.CODECOV_TOKEN }}
+          slug: bcdev/gaiaflow

README.md

@@ -1,9 +1,11 @@
 # Gaiaflow
 
 ![PyPI - Version](https://img.shields.io/pypi/v/gaiaflow)
+[![codecov](https://codecov.io/gh/bcdev/gaiaflow/graph/badge.svg?token=pc9DfJx6bu)](https://codecov.io/gh/bcdev/gaiaflow)
 [![Pixi Badge](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/prefix-dev/pixi/main/assets/badge/v0.json)](https://pixi.sh)
 [![Ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/charliermarsh/ruff/main/assets/badge/v0.json)](https://github.com/charliermarsh/ruff)
 [![Docs](https://img.shields.io/badge/docs-mkdocs-blue)](https://bcdev.github.io/gaiaflow/)
+
 ![Static Badge](https://img.shields.io/badge/Airflow-3.0-8A2BE2?logo=apacheairflow)
 ![Static Badge](https://img.shields.io/badge/MLFlow-darkblue?logo=mlflow)
 ![Static Badge](https://img.shields.io/badge/MinIO-red?logo=minio)

pyproject.toml

@@ -31,12 +31,12 @@ build-backend = "hatchling.build"
 requires = ["hatchling"]
 
 [tool.hatch.build.targets.wheel]
-packages = ["src/gaiaflow", "src/docker_stuff"]
+packages = ["src/gaiaflow", "src/_docker"]
 
 [tool.hatch.build]
 include = [
   "src/gaiaflow",
-  "src/docker_stuff",
+  "src/_docker",
 ]
 
 [tool.mypy]
@@ -56,6 +56,7 @@ ruff = "*"
 
 [tool.pixi.tasks]
 test = "pytest tests"
+cov = "pytest --cov=gaiaflow --cov-report=html"
 format = "isort src/gaiaflow && ruff format src/gaiaflow"
 check = "ruff check src/gaiaflow"
 check_fix = "ruff check src/gaiaflow --fix"

src/docker_stuff/docker-compose/docker-compose.yml → src/_docker/docker-compose/docker-compose.yml

@@ -2,7 +2,7 @@ x-airflow-common:
   &airflow-common
   build:
     context: ../../
-    dockerfile: "./docker_stuff/airflow/Dockerfile"
+    dockerfile: "./_docker/airflow/Dockerfile"
   environment:
     &airflow-common-env
     # NOTE: The following secret can be commited to .git as it is only for local development.
@@ -54,6 +54,17 @@ x-airflow-common:
     - ml-network
 
 services:
+  docker-proxy:
+    image: alpine/socat
+    container_name: docker-proxy
+    command: "TCP4-LISTEN:2375,fork,reuseaddr UNIX-CONNECT:/var/run/docker.sock"
+    ports:
+      - "2376:2375"
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock
+    networks:
+      - ml-network
+
 
   postgres-mlflow:
     image: postgres:13
@@ -75,7 +86,7 @@ services:
   mlflow:
     build:
       context: ..
-      dockerfile: "../docker_stuff/mlflow/Dockerfile"
+      dockerfile: "../_docker/mlflow/Dockerfile"
     container_name: mlflow
     ports:
       - 5000:5000

src/gaiaflow/cli/commands/minikube.py

@@ -5,6 +5,7 @@
 import typer
 
 from gaiaflow.constants import DEFAULT_IMAGE_NAME
+from gaiaflow.managers.helpers import DockerHandlerMode
 
 app = typer.Typer()
 fs = fsspec.filesystem("file")
@@ -31,7 +32,6 @@ def load_imports():
 
 @app.command(help="Start Gaiaflow production-like services.")
 def start(
-    project_path: Path = typer.Option(..., "--path", "-p", help="Path to your project"),
     force_new: bool = typer.Option(
         False,
         "--force-new",
@@ -42,6 +42,7 @@ def start(
 ):
     """"""
     imports = load_imports()
+    project_path = Path.cwd()
     gaiaflow_path, user_project_path = imports.create_gaiaflow_context_path(
         project_path
     )
@@ -58,10 +59,9 @@ def start(
 
 
 @app.command(help="Stop Gaiaflow production-like services.")
-def stop(
-    project_path: Path = typer.Option(..., "--path", "-p", help="Path to your project"),
-):
+def stop():
     imports = load_imports()
+    project_path = Path.cwd()
     gaiaflow_path, user_project_path = imports.create_gaiaflow_context_path(
         project_path
     )
@@ -78,7 +78,6 @@ def stop(
 
 @app.command(help="Restart Gaiaflow production-like services.")
 def restart(
-    project_path: Path = typer.Option(..., "--path", "-p", help="Path to your project"),
     force_new: bool = typer.Option(
         False,
         "--force-new",
@@ -88,6 +87,7 @@ def restart(
     ),
 ):
     imports = load_imports()
+    project_path = Path.cwd()
     gaiaflow_path, user_project_path = imports.create_gaiaflow_context_path(
         project_path
     )
@@ -99,43 +99,92 @@ def restart(
         gaiaflow_path=gaiaflow_path,
         user_project_path=user_project_path,
         action=imports.BaseAction.RESTART,
+        force_new=force_new,
     )
 
 
 @app.command(
     help="Containerize your package into a docker image inside the minikube cluster."
 )
 def dockerize(
-    project_path: Path = typer.Option(..., "--path", "-p", help="Path to your project"),
     image_name: str = typer.Option(
         DEFAULT_IMAGE_NAME, "--image-name", "-i", help=("Name of your image.")
     ),
+    dockerfile_path: Path = typer.Option(
+        None, "--dockerfile-path", "-d", help=("Path to your custom Dockerfile")
+    ),
 ):
     imports = load_imports()
+    project_path = Path.cwd()
     gaiaflow_path, user_project_path = imports.create_gaiaflow_context_path(
         project_path
     )
     gaiaflow_path_exists = imports.gaiaflow_path_exists_in_state(gaiaflow_path, True)
     if not gaiaflow_path_exists:
         typer.echo("Please create a project with Gaiaflow before running this command.")
         return
+    if dockerfile_path:
+        docker_handler_mode = "minikube-user"
+    else:
+        docker_handler_mode = "minikube"
     imports.MinikubeManager.run(
         gaiaflow_path=gaiaflow_path,
         user_project_path=user_project_path,
         action=imports.ExtendedAction.DOCKERIZE,
-        local=False,
+        docker_handler_mode=docker_handler_mode,
         image_name=image_name,
+        dockerfile_path=dockerfile_path,
+    )
+
+
+@app.command(help="List all the docker images in your system")
+def list_images():
+    imports = load_imports()
+    project_path = Path.cwd()
+    gaiaflow_path, user_project_path = imports.create_gaiaflow_context_path(
+        project_path
+    )
+    gaiaflow_path_exists = imports.gaiaflow_path_exists_in_state(gaiaflow_path, True)
+    if not gaiaflow_path_exists:
+        typer.echo("Please create a project with Gaiaflow before running this command.")
+        return
+    imports.MinikubeManager.run(
+        gaiaflow_path=gaiaflow_path,
+        user_project_path=user_project_path,
+        action=imports.ExtendedAction.LIST_IMAGES,
+        docker_handler_mode=DockerHandlerMode.MINIKUBE,
+    )
+
+@app.command(help="Delete a docker image from your system")
+def remove_image(image_name: str = typer.Option(
+        DEFAULT_IMAGE_NAME, "--image-name", "-i", help=("Name of image "
+                                                        "to be deleted.")
+    ),):
+    imports = load_imports()
+    project_path = Path.cwd()
+    gaiaflow_path, user_project_path = imports.create_gaiaflow_context_path(
+        project_path
+    )
+    gaiaflow_path_exists = imports.gaiaflow_path_exists_in_state(gaiaflow_path, True)
+    if not gaiaflow_path_exists:
+        typer.echo("Please create a project with Gaiaflow before running this command.")
+        return
+    imports.MinikubeManager.run(
+        gaiaflow_path=gaiaflow_path,
+        user_project_path=user_project_path,
+        action=imports.ExtendedAction.REMOVE_IMAGE,
+        image_name=image_name,
+        docker_handler_mode=DockerHandlerMode.MINIKUBE,
     )
 
 
 @app.command(
     help="Create a config file for Airflow to talk to Kubernetes "
     "cluster. To be used only when debugging required."
 )
-def create_config(
-    project_path: Path = typer.Option(..., "--path", "-p", help="Path to your project"),
-):
+def create_config():
     imports = load_imports()
+    project_path = Path.cwd()
     gaiaflow_path, user_project_path = imports.create_gaiaflow_context_path(
         project_path
     )
@@ -152,13 +201,13 @@ def create_config(
 
 @app.command(help="Create secrets to provide to the production-like environment.")
 def create_secret(
-    project_path: Path = typer.Option(..., "--path", "-p", help="Path to your project"),
     name: str = typer.Option(..., "--name", help="Name of the secret"),
     data: list[str] = typer.Option(
         ..., "--data", help="Secret data as key=value pairs"
     ),
 ):
     imports = load_imports()
+    project_path = Path.cwd()
     secret_data = imports.parse_key_value_pairs(data)
     print(secret_data, name)
     gaiaflow_path, user_project_path = imports.create_gaiaflow_context_path(
@@ -179,12 +228,11 @@ def create_secret(
 
 @app.command(
     help="Clean Gaiaflow production-like services. This will only remove the "
-    "minikube speicifc things. To remove local docker stuff, use the dev mode."
+    "minikube specific things. To remove local docker stuff, use the dev mode."
 )
-def cleanup(
-    project_path: Path = typer.Option(..., "--path", "-p", help="Path to your project"),
-):
+def cleanup():
     imports = load_imports()
+    project_path = Path.cwd()
     gaiaflow_path, user_project_path = imports.create_gaiaflow_context_path(
         project_path
     )