Cell containerization extractor handler

.github/workflows/ci-pipeline.yml

@@ -25,6 +25,8 @@ env:
   DOCKER_FOLDER:  ${{ inputs.docker_folder }}
   TAG:  ${{ inputs.tag }}
   DOCKERHUB_IMAGE_ID: "qcdis/"
+  CELL_GITHUB: "https://github.com/QCDIS/NaaVRE-cells-test-3"
+  CELL_GITHUB_TOKEN: ${{ secrets.CELL_GITHUB_TOKEN }}
 
 jobs:
   build:
@@ -33,21 +35,40 @@ jobs:
     steps:
     - uses: actions/checkout@v4
 
+    - name: chkenv
+      run: python -c "import os; print(os.getenv('CELL_GITHUB')); print(os.getenv('BASE_PATH'))"
+
     - name: Build docker
       run: |
+          echo '-> envvar:'
+          env
           cd $DOCKER_FOLDER && docker build . --file Dockerfile -t $TAG --build-arg "NODE_ENV=${{ inputs.environment }}"
 
     - name: Run docker compose
-      uses: isbang/compose-action@v1.5.1
+      # uses: isbang/compose-action@v1.5.1
+      uses: hoverkraft-tech/compose-action@v2.0.1
       with:
         compose-file: "${{ inputs.docker_folder }}/docker-compose.yaml"
 
+    - name: chkenv2
+      run: python -c "import os; print(os.getenv('CELL_GITHUB')); print(os.getenv('BASE_PATH'))"
+
     - name: Wait for healthy app
       uses: raschmitt/wait-for-healthy-container/@v1
       with:
         container-name: app
         timeout: 120
 
+    - name: chkenv3
+      if: always()
+      run: python -c "import os; print(os.getenv('CELL_GITHUB')); print(os.getenv('BASE_PATH'))"
+
+    - name: Capture Docker Logs
+      if: failure()
+      run: |
+        echo '-> Capturing Docker logs ...'
+        docker logs app
+
     - name: Login to github Registry
       if: ${{ inputs.push }}
       uses: docker/login-action@v3

tilt/vreapis/Dockerfile

@@ -1,9 +1,11 @@
-FROM python:3.12.2-slim
+# FROM python:3.12.2-slim
+FROM python:3.11.9-slim
+# Dependency pytype [used by PyExtractor] complains 'Python versions > 3.11 are not yet supported'
 
 WORKDIR /app
 
 RUN apt update && \
-    apt install -y libpq-dev gcc gdal-bin curl net-tools iputils-ping
+    apt install -y libpq-dev gcc gdal-bin curl net-tools iputils-ping r-base
 
 # Install rysnc for Tilt syncback
 RUN apt install -y rsync
@@ -14,6 +16,9 @@ RUN /opt/venv/bin/pip install pip --upgrade
 COPY ./vreapis/requirements.txt /app
 RUN /opt/venv/bin/pip install -r requirements.txt
 
+# Set another directory for installation of R packages to avoid permission problem
+# RUN echo 'R_LIBS_USER="~/R/library"' >> ~/.Renviron
+
 COPY ./vreapis /app
 RUN chmod +x entrypoint.dev.sh
 

vreapis/.dockerignore

@@ -0,0 +1,6 @@
+.env
+export_VARS
+.idea
+.pytest_cache
+__pycache__
+NaaVRE_db.json

vreapis/Dockerfile

@@ -1,16 +1,19 @@
-FROM python:3.12.2-slim
+# FROM python:3.12.2-slim
+FROM python:3.11.9-slim
 
 WORKDIR /app
 
 RUN apt update && \
-    apt install -y libpq-dev gcc gdal-bin curl net-tools iputils-ping
+    apt install -y libpq-dev gcc gdal-bin curl net-tools iputils-ping r-base
 
 RUN python3 -m venv /opt/venv
 RUN /opt/venv/bin/pip install pip --upgrade
 
 COPY ./requirements.txt /app
 RUN /opt/venv/bin/pip install -r requirements.txt
 
+RUN Rscript -e "install.packages(c('rlang', 'lobstr', 'purrr', 'renv'), repos = 'https://cloud.r-project.org')"
+
 COPY . /app
 RUN chmod +x entrypoint.sh
 

vreapis/auth/Keycloak.py

@@ -0,0 +1,48 @@
+import os
+from requests.exceptions import RequestException
+from rest_framework.authentication import BaseAuthentication
+from rest_framework.exceptions import AuthenticationFailed
+from django.contrib.auth.models import User
+from django.http import HttpRequest
+from django.conf import settings
+import jwt
+import jwt.algorithms
+
+import common
+
+
+class KeycloakAuthentication(BaseAuthentication):
+    @staticmethod
+    def get_verif_inf() -> dict[str, any]:
+        return common.session.get(settings.KEYCLOAK_VERIF_URL, verify=settings.ALLOW_INSECURE_TLS).json()
+
+    def authenticate(self, request: HttpRequest):
+        received_token: str = request.headers.get('Authorization', '')
+        if received_token == '':
+            raise AuthenticationFailed('Authorization header not found. Login using Keycloak to get an access token.')
+        token_header: dict[str, any] = jwt.get_unverified_header(received_token)
+        try:
+            verif_inf: dict[str, any] = KeycloakAuthentication.get_verif_inf()
+        except RequestException as e:
+            common.logger.error(f'[{e.response.status_code}] Error while getting Keycloak public key.')
+            return None
+        kid: str = token_header.get('kid', '')
+        if kid == '':
+            common.logger.error(f'JWT Key ID (kid) not found:{os.linesep}{received_token}')
+            raise AuthenticationFailed(f'Invalid token format: JWT Key ID (kid) not found.')
+        verif_keys: list[dict[str, any]] = verif_inf.get('keys', [])
+        key = next((key for key in verif_keys if key['kid'] == kid), None)
+        if key is None:
+            common.logger.error(f'Keycloak public key not found.')
+            return None
+        pub_key = jwt.algorithms.RSAAlgorithm.from_jwk(key)
+        access_token: dict[str, any] = jwt.decode(received_token, pub_key, algorithms=key['alg'], audience='account')
+        username: str = access_token.get('preferred_username', '')
+        if username == '':
+            common.logger.error(f'Username (preferred_username) not found in token:{os.linesep}{received_token}')
+            raise AuthenticationFailed(f'Invalid token format: Username (preferred_username) not found.')
+        try:
+            user: User | None = User.objects.get(username=username)
+        except User.DoesNotExist:
+            user: User = User.objects.create_user(username)
+        return user, None

vreapis/auth/simple.py

@@ -0,0 +1,24 @@
+from rest_framework.authentication import BaseAuthentication
+from rest_framework.exceptions import AuthenticationFailed
+from django.contrib.auth.models import User
+from django.http import HttpRequest
+from django.conf import settings
+import string
+import random
+
+
+class StaticTokenAuthentication(BaseAuthentication):
+    dummy_user: User
+
+    def __init__(self):
+        try:
+            StaticTokenAuthentication.dummy_user = User.objects.get(username=settings.NAAVRE_API_TOKEN)
+        except User.DoesNotExist:
+            StaticTokenAuthentication.dummy_user = User.objects.create_user(settings.NAAVRE_API_TOKEN, password=''.join(random.choice(string.printable) for _ in range(32)))
+
+    def authenticate(self, request: HttpRequest):
+        access_token: str = request.headers.get('Authorization', '')
+        print(access_token == f'Token {settings.NAAVRE_API_TOKEN}')
+        if access_token != f'Token {settings.NAAVRE_API_TOKEN}':
+            raise AuthenticationFailed(f'Invalid NaaVRE API token')
+        return StaticTokenAuthentication.dummy_user, None

vreapis/auth/test_Keycloak.py

@@ -0,0 +1,18 @@
+from django.test import TestCase, RequestFactory
+from django.conf import settings
+from urllib.parse import urlencode
+from auth.Keycloak import KeycloakAuthentication
+import common
+
+
+class KeycloakAuthenticationTestCase(TestCase):
+    def test_authenticate(self):  # todo. what if authentication failed? what if refresh token is used?
+        post_data: dict[str, str] = {'client_id': settings.KEYCLOAK_CLIENT_ID, 'grant_type': 'password', 'scope': 'openid', 'username': 'u', 'password': 'u'}
+        response_body: dict[str, any] = common.session.post(settings.KEYCLOAK_LOGIN_URL, headers={'Content-Type': 'application/x-www-form-urlencoded'}, data=urlencode(post_data), verify=settings.ALLOW_INSECURE_TLS).json()
+        access_token: str = response_body['access_token']
+
+        request_factory = RequestFactory()
+        request = request_factory.post('', HTTP_AUTHORIZATION=access_token)
+        authenticator = KeycloakAuthentication()
+        user, _ = authenticator.authenticate(request)
+        assert user.username == post_data['username']

vreapis/auth/test_simple.py

@@ -0,0 +1,21 @@
+from django.test import TestCase, RequestFactory
+from django.conf import settings
+from rest_framework.exceptions import AuthenticationFailed
+
+from auth.simple import StaticTokenAuthentication
+
+
+class StaticTokenAuthenticationTestCase(TestCase):
+    tokens_accepted = [f'Token {settings.NAAVRE_API_TOKEN}', ]
+    tokens_not_accepted = [f'{settings.NAAVRE_API_TOKEN}', f'', f'0123456789abcdef0123456789abcdef01234567', ]
+
+    def test_authenticate(self):
+        request_factory = RequestFactory()
+        authenticator = StaticTokenAuthentication()
+        print(StaticTokenAuthenticationTestCase.tokens_accepted)
+        for token in StaticTokenAuthenticationTestCase.tokens_accepted:
+            request = request_factory.post('', HTTP_AUTHORIZATION=token)
+            user, _ = authenticator.authenticate(request)
+        for token in StaticTokenAuthenticationTestCase.tokens_not_accepted:
+            request = request_factory.post('', HTTP_AUTHORIZATION=token)
+            self.assertRaises(AuthenticationFailed, authenticator.authenticate, request)

vreapis/catalog/admin.py

@@ -0,0 +1,7 @@
+from django.contrib import admin
+
+from . import models
+
+# Register your models here.
+
+admin.site.register(models.Cell)

vreapis/cells/apps.py → vreapis/catalog/apps.py

@@ -1,6 +1,6 @@
 from django.apps import AppConfig
 
 
-class CellsConfig(AppConfig):
+class CatalogConfig(AppConfig):
     default_auto_field = 'django.db.models.BigAutoField'
-    name = 'cells'
+    name = 'catalog'

vreapis/catalog/management/commands/export-from-psql-pod-in-legacy-db-schema.ps1

@@ -0,0 +1,30 @@
+$cwd = $PWD
+
+$project_root = Resolve-Path "$PSScriptRoot/../../.."
+
+$vreapi_pod_inf = kubectl get pod | sls '^vrepaas-vreapi-\w+-\w+'
+$vreapi_pod_name = $vreapi_pod_inf.Matches.Value
+$psql_pod_inf = kubectl get pod -owide | sls '^vrepaas-postgresql-0'
+$psql_pod_IP = ($psql_pod_inf.Line | sls '\d+\.\d+\.\d+\.\d+').Matches.Value
+
+cd $project_root
+$var = Get-Content export_VARS
+$off_DB_HOST = ($var | sls '^export DB_HOST=').LineNumber - 1
+$off_DB_NAME = ($var | sls '^export DB_NAME=').LineNumber - 1
+$off_DB_USER = ($var | sls '^export DB_USER=').LineNumber - 1
+$off_DB_PASSWORD = ($var | sls '^export DB_PASSWORD=').LineNumber - 1
+$var[$off_DB_HOST] = "export DB_HOST=$psql_pod_IP"
+$var[$off_DB_NAME] = "export DB_NAME=vrepaas"
+$var[$off_DB_USER] = "export DB_USER=vrepaas"
+$var[$off_DB_PASSWORD] = "export DB_PASSWORD=vrepaas"
+
+$var > /tmp/export_VARS
+kubectl cp /tmp/export_VARS "${vreapi_pod_name}:/tmp"
+kubectl cp "$PSScriptRoot/export-from-psql-pod-in-legacy-db-schema.sh" "${vreapi_pod_name}:/tmp"
+
+kubectl cp ~/NaaVRE/NaaVRE_db.json "${vreapi_pod_name}:/app"
+kubectl exec -it $vreapi_pod_name -- bash "/tmp/export-from-psql-pod-in-legacy-db-schema.sh"
+
+kubectl cp "${vreapi_pod_name}:/app/NaaVRE_db.json" ~/NaaVRE/NaaVRE_db.json
+
+cd $cwd

vreapis/catalog/management/commands/export-from-psql-pod-in-legacy-db-schema.sh

@@ -0,0 +1,3 @@
+source /opt/venv/bin/activate
+source /tmp/export_VARS
+python manage.py export_in_legacy_db_schema Cell

vreapis/catalog/management/commands/export_in_legacy_db_schema.py

@@ -0,0 +1,39 @@
+import json
+
+from django.core.management import CommandParser
+from django.core.management.base import BaseCommand, CommandError
+from django.db.models import QuerySet
+
+import common
+from catalog.models import Cell
+from catalog.serializers import CellSerializer
+
+
+class Command(BaseCommand):
+    help: str = 'Export designated tables in legacy db schema'
+
+
+    def add_arguments(self, parser: CommandParser):
+        parser.add_argument('tables', type=str, nargs='+', help='Tables to export')
+
+    def handle(self, *args, **options):
+        for table in options['tables']:
+            match table:
+                case 'Cell':
+                    queryset: QuerySet = Cell.objects.all().order_by('task_name')
+                    print(f'Cell count: {queryset.count()}')
+                    no: int = 1
+                    cells: dict[str, dict] = {}
+                    for cell in queryset:
+                        serializer = CellSerializer(cell)
+                        print(f'task_name: {cell.task_name}')
+                        cells[str(no)] = serializer.data
+                        no += 1
+                    db_file: str = f'{common.project_root}/NaaVRE_db.json'
+                    with open(db_file) as f:
+                        db = json.load(f)
+                        db['cells'] = cells
+                    with open(db_file, 'w') as f:
+                        json.dump(db, f)
+                case _:
+                    raise CommandError(f"Table {table} is not supported")

vreapis/catalog/migrations/0001_initial.py

@@ -0,0 +1,37 @@
+# Generated by Django 5.0.3 on 2024-06-17 18:52
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = [
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='Cell',
+            fields=[
+                ('title', models.CharField(max_length=255)),
+                ('task_name', models.CharField(max_length=255)),
+                ('original_source', models.CharField(max_length=255)),
+                ('types', models.JSONField(blank=True, default=dict, null=True)),
+                ('base_image', models.JSONField(blank=True, null=True)),
+                ('inputs', models.JSONField(blank=True, default=list, null=True)),
+                ('outputs', models.JSONField(blank=True, default=list, null=True)),
+                ('params', models.JSONField(blank=True, default=list, null=True)),
+                ('param_values', models.JSONField(blank=True, null=True)),
+                ('confs', models.JSONField(blank=True, null=True)),
+                ('dependencies', models.JSONField(blank=True, null=True)),
+                ('chart_obj', models.JSONField(blank=True, null=True)),
+                ('node_id', models.CharField(max_length=255, primary_key=True, serialize=False)),
+                ('container_source', models.CharField(max_length=255)),
+                ('global_conf', models.JSONField(blank=True, null=True)),
+                ('kernel', models.CharField(max_length=255)),
+                ('notebook_dict', models.JSONField(blank=True, null=True)),
+                ('image_version', models.CharField(max_length=255)),
+            ],
+        ),
+    ]

vreapis/catalog/migrations/0002_alter_cell_container_source_alter_cell_image_version.py

@@ -0,0 +1,23 @@
+# Generated by Django 5.0.3 on 2024-06-17 19:03
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('catalog', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='cell',
+            name='container_source',
+            field=models.CharField(blank=True, max_length=255, null=True),
+        ),
+        migrations.AlterField(
+            model_name='cell',
+            name='image_version',
+            field=models.CharField(blank=True, max_length=255, null=True),
+        ),
+    ]