From 7aeaad299bd64a78613783b1a920455a97e9339e Mon Sep 17 00:00:00 2001 From: AndyBRoswell Date: Tue, 21 May 2024 19:57:37 +0200 Subject: [PATCH 001/149] preliminarily migrated class BaseImageTagsHandler(APIHandler) --- vreapis/containerizer/__init__.py | 0 vreapis/containerizer/admin.py | 3 +++ vreapis/containerizer/apps.py | 6 ++++++ vreapis/containerizer/models.py | 3 +++ vreapis/containerizer/tests.py | 3 +++ vreapis/containerizer/urls.py | 6 ++++++ vreapis/containerizer/views.py | 32 +++++++++++++++++++++++++++++++ vreapis/vreapis/settings/base.py | 4 +++- vreapis/vreapis/urls.py | 3 ++- 9 files changed, 58 insertions(+), 2 deletions(-) create mode 100644 vreapis/containerizer/__init__.py create mode 100644 vreapis/containerizer/admin.py create mode 100644 vreapis/containerizer/apps.py create mode 100644 vreapis/containerizer/models.py create mode 100644 vreapis/containerizer/tests.py create mode 100644 vreapis/containerizer/urls.py create mode 100644 vreapis/containerizer/views.py diff --git a/vreapis/containerizer/__init__.py b/vreapis/containerizer/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/vreapis/containerizer/admin.py b/vreapis/containerizer/admin.py new file mode 100644 index 00000000..8c38f3f3 --- /dev/null +++ b/vreapis/containerizer/admin.py @@ -0,0 +1,3 @@ +from django.contrib import admin + +# Register your models here. diff --git a/vreapis/containerizer/apps.py b/vreapis/containerizer/apps.py new file mode 100644 index 00000000..eb32c03b --- /dev/null +++ b/vreapis/containerizer/apps.py @@ -0,0 +1,6 @@ +from django.apps import AppConfig + + +class ContainerizerConfig(AppConfig): + default_auto_field = 'django.db.models.BigAutoField' + name = 'containerizer' diff --git a/vreapis/containerizer/models.py b/vreapis/containerizer/models.py new file mode 100644 index 00000000..71a83623 --- /dev/null +++ b/vreapis/containerizer/models.py @@ -0,0 +1,3 @@ +from django.db import models + +# Create your models here. diff --git a/vreapis/containerizer/tests.py b/vreapis/containerizer/tests.py new file mode 100644 index 00000000..7ce503c2 --- /dev/null +++ b/vreapis/containerizer/tests.py @@ -0,0 +1,3 @@ +from django.test import TestCase + +# Create your tests here. diff --git a/vreapis/containerizer/urls.py b/vreapis/containerizer/urls.py new file mode 100644 index 00000000..e67d7955 --- /dev/null +++ b/vreapis/containerizer/urls.py @@ -0,0 +1,6 @@ +from django.urls import path +from . import views + +urlpatterns = [ + path('baseimagetags/', views.get_base_images) +] diff --git a/vreapis/containerizer/views.py b/vreapis/containerizer/views.py new file mode 100644 index 00000000..38e44533 --- /dev/null +++ b/vreapis/containerizer/views.py @@ -0,0 +1,32 @@ +import os +import logging +import requests +import requests.adapters +import urllib3 +from rest_framework.response import Response +from rest_framework import status +from rest_framework.decorators import api_view + +logger = logging.getLogger(__name__) + +# customized requests.Session [w/ auto retry] +session = requests.Session() +retry_adapter = requests.adapters.HTTPAdapter(max_retries=urllib3.Retry(total=5, status_forcelist=[500])) +session.mount('http://', retry_adapter) +session.mount('https://', retry_adapter) + + +@api_view(['GET']) +def get_base_images(request): + url: str = os.getenv('BASE_IMAGE_TAGS_URL', 'https://github.com/QCDIS/NaaVRE-flavors/releases/latest/download/base_image_tags.json') + logger.debug(f'Base image tags URL: {url}') + try: + response = session.get(url) + response.raise_for_status() + dat: dict[str, dict[str, str]] = response.json() + except (requests.ConnectionError, requests.HTTPError, requests.JSONDecodeError,) as e: + msg: str = f'Error loading base image tags from {url}\n{e}' + logger.debug(msg) + return Response({'error': msg}, status=status.HTTP_500_INTERNAL_SERVER_ERROR) + Origin: str | None = request.headers.get('Origin', '*') + return Response(dat, headers={'Access-Control-Allow-Origin': Origin}) diff --git a/vreapis/vreapis/settings/base.py b/vreapis/vreapis/settings/base.py index d445320b..644e6819 100644 --- a/vreapis/vreapis/settings/base.py +++ b/vreapis/vreapis/settings/base.py @@ -22,7 +22,8 @@ ALLOWED_HOSTS = ['*'] -CSRF_TRUSTED_ORIGINS = [os.getenv('TRUSTED_ORIGINS')] +# CSRF_TRUSTED_ORIGINS = [os.getenv('TRUSTED_ORIGINS')] +CSRF_TRUSTED_ORIGINS = os.getenv('TRUSTED_ORIGINS').split(sep=';') CORS_ALLOWED_ORIGIN_REGEXES = [ r'^http:\/\/localhost:\d+$', @@ -39,6 +40,7 @@ 'virtual_labs', 'data_products', 'paas_configuration', + 'containerizer', 'rest_framework', 'rest_framework_gis', 'rest_framework.authtoken', diff --git a/vreapis/vreapis/urls.py b/vreapis/vreapis/urls.py index 16312f26..fdb019e8 100644 --- a/vreapis/vreapis/urls.py +++ b/vreapis/vreapis/urls.py @@ -38,7 +38,8 @@ urlpatterns = [ path('admin/', admin.site.urls), - path('api/', include(router.urls)) + path('api/', include(router.urls)), + path('api/containerizer/', include('containerizer.urls')), ] if BASE_PATH: From a533cfa12a76bbc9b567fe6aa07b29714a285dee Mon Sep 17 00:00:00 2001 From: AndyBRoswell Date: Sun, 26 May 2024 19:06:29 +0200 Subject: [PATCH 002/149] adding keycloak token auth --- vreapis/auth/KeycloakAuthentication.py | 42 ++++++++++++++++++++++++++ vreapis/common.py | 12 ++++++++ vreapis/containerizer/views.py | 22 ++++---------- vreapis/requirements.txt | 2 ++ vreapis/vreapis/settings/base.py | 8 +++++ 5 files changed, 70 insertions(+), 16 deletions(-) create mode 100644 vreapis/auth/KeycloakAuthentication.py create mode 100644 vreapis/common.py diff --git a/vreapis/auth/KeycloakAuthentication.py b/vreapis/auth/KeycloakAuthentication.py new file mode 100644 index 00000000..afe2bdd1 --- /dev/null +++ b/vreapis/auth/KeycloakAuthentication.py @@ -0,0 +1,42 @@ +import os +from requests.exceptions import RequestException +from rest_framework.authentication import BaseAuthentication +from django.contrib.auth.models import User +from django.http import HttpRequest +from django.conf import settings +import jwt +import jwt.algorithms + +import common + + +class KeycloakAuthentication(BaseAuthentication): + def authenticate(self, request: HttpRequest): + received_token: str = request.headers.get('Authorization', '') + if received_token == '': + return None + token_header: dict[str, any] = jwt.get_unverified_header(received_token) + try: + verif_body: dict[str, any] = common.session.get(settings.KEYCLOAK_VERIF_URL, verify=settings.ALLOW_INSECURE_TLS).json() + except RequestException as e: + common.logger.error(f'Error verifying token: {e}') + return None + kid: str = token_header.get('kid', '') + if kid == '': + common.logger.error(f'JWT Key ID [kid] not found:{os.linesep}{received_token}') + return None + verif_keys: list[dict[str, any]] = verif_body.get('keys', []) + key = next((key for key in verif_keys if key['kid'] == kid), None) + if key is None: + common.logger.error(f'Keycloak public key not found for received kid {kid}') + return None + pub_key = jwt.algorithms.RSAAlgorithm.from_jwk(key) + access_token: dict[str, any] = jwt.decode(received_token, pub_key, algorithms=key['alg'], audience='account') + username: str = access_token.get('preferred_username', '') + if username == '': + common.logger.error(f'Username (preferred_username) not found in token:{os.linesep}{received_token}') + return None + user: User | None = User.objects.get(username=username) + if user is None: + User.objects.create_user(username) + return user, None diff --git a/vreapis/common.py b/vreapis/common.py new file mode 100644 index 00000000..f3843084 --- /dev/null +++ b/vreapis/common.py @@ -0,0 +1,12 @@ +import logging +import urllib3 +import requests.adapters + +# customized requests.Session [w/ auto retry] +session = requests.Session() +retry_adapter = requests.adapters.HTTPAdapter(max_retries=urllib3.Retry(total=5, status_forcelist=[500])) +session.mount('http://', retry_adapter) +session.mount('https://', retry_adapter) + +# global logger +logger = logging.getLogger(__name__) diff --git a/vreapis/containerizer/views.py b/vreapis/containerizer/views.py index 38e44533..c46d2827 100644 --- a/vreapis/containerizer/views.py +++ b/vreapis/containerizer/views.py @@ -1,32 +1,22 @@ import os -import logging -import requests -import requests.adapters -import urllib3 from rest_framework.response import Response from rest_framework import status from rest_framework.decorators import api_view - -logger = logging.getLogger(__name__) - -# customized requests.Session [w/ auto retry] -session = requests.Session() -retry_adapter = requests.adapters.HTTPAdapter(max_retries=urllib3.Retry(total=5, status_forcelist=[500])) -session.mount('http://', retry_adapter) -session.mount('https://', retry_adapter) +import requests +import common @api_view(['GET']) def get_base_images(request): url: str = os.getenv('BASE_IMAGE_TAGS_URL', 'https://github.com/QCDIS/NaaVRE-flavors/releases/latest/download/base_image_tags.json') - logger.debug(f'Base image tags URL: {url}') + common.logger.debug(f'Base image tags URL: {url}') try: - response = session.get(url) + response = common.session.get(url) response.raise_for_status() dat: dict[str, dict[str, str]] = response.json() except (requests.ConnectionError, requests.HTTPError, requests.JSONDecodeError,) as e: msg: str = f'Error loading base image tags from {url}\n{e}' - logger.debug(msg) + common.logger.debug(msg) return Response({'error': msg}, status=status.HTTP_500_INTERNAL_SERVER_ERROR) - Origin: str | None = request.headers.get('Origin', '*') + Origin: str = request.headers.get('Origin', '*') return Response(dat, headers={'Access-Control-Allow-Origin': Origin}) diff --git a/vreapis/requirements.txt b/vreapis/requirements.txt index d6dd3d5f..f58fb174 100644 --- a/vreapis/requirements.txt +++ b/vreapis/requirements.txt @@ -11,3 +11,5 @@ django-cors-headers==4.3.1 django-extensions==3.2.3 psycopg2==2.9.9 django-probes==1.7.0 +pyjwt==2.8.0 +cryptography==42.0.7 diff --git a/vreapis/vreapis/settings/base.py b/vreapis/vreapis/settings/base.py index 644e6819..9c26bef1 100644 --- a/vreapis/vreapis/settings/base.py +++ b/vreapis/vreapis/settings/base.py @@ -186,3 +186,11 @@ if BASE_PATH: STATIC_URL = f'{BASE_PATH}/static/' STATIC_ROOT = BASE_DIR / "staticfiles" + +# CUSTOM VARS + +KEYCLOAK_URL: str = os.getenv('KEYCLOAK_URL', 'https://naavre-dev.minikube.test/auth') +KEYCLOAK_REALM: str = os.getenv('KEYCLOAK_REALM', 'vre') +KEYCLOAK_LOGIN_URL: str = f'{KEYCLOAK_URL}/realms/{KEYCLOAK_REALM}/protocol/openid-connect/token' +KEYCLOAK_VERIF_URL: str = f'{KEYCLOAK_URL}/realms/{KEYCLOAK_REALM}/protocol/openid-connect/certs' +KEYCLOAK_CLIENT_ID: str = os.getenv('KEYCLOAK_CLIENT_ID', 'myclient') From 16eb3189a51a5a9a84d9eded524de6e2ec7589d7 Mon Sep 17 00:00:00 2001 From: AndyBRoswell Date: Sun, 26 May 2024 19:36:30 +0200 Subject: [PATCH 003/149] keycloak token auth with unit test --- ...{KeycloakAuthentication.py => Keycloak.py} | 9 +++++---- vreapis/auth/test_KeycloakAuthentication.py | 19 +++++++++++++++++++ vreapis/containerizer/tests.py | 1 + 3 files changed, 25 insertions(+), 4 deletions(-) rename vreapis/auth/{KeycloakAuthentication.py => Keycloak.py} (87%) create mode 100644 vreapis/auth/test_KeycloakAuthentication.py diff --git a/vreapis/auth/KeycloakAuthentication.py b/vreapis/auth/Keycloak.py similarity index 87% rename from vreapis/auth/KeycloakAuthentication.py rename to vreapis/auth/Keycloak.py index afe2bdd1..2868f57b 100644 --- a/vreapis/auth/KeycloakAuthentication.py +++ b/vreapis/auth/Keycloak.py @@ -23,7 +23,7 @@ def authenticate(self, request: HttpRequest): return None kid: str = token_header.get('kid', '') if kid == '': - common.logger.error(f'JWT Key ID [kid] not found:{os.linesep}{received_token}') + common.logger.error(f'JWT Key ID (kid) not found:{os.linesep}{received_token}') return None verif_keys: list[dict[str, any]] = verif_body.get('keys', []) key = next((key for key in verif_keys if key['kid'] == kid), None) @@ -36,7 +36,8 @@ def authenticate(self, request: HttpRequest): if username == '': common.logger.error(f'Username (preferred_username) not found in token:{os.linesep}{received_token}') return None - user: User | None = User.objects.get(username=username) - if user is None: - User.objects.create_user(username) + try: + user: User | None = User.objects.get(username=username) + except User.DoesNotExist: + user: User = User.objects.create_user(username) return user, None diff --git a/vreapis/auth/test_KeycloakAuthentication.py b/vreapis/auth/test_KeycloakAuthentication.py new file mode 100644 index 00000000..3299bfa3 --- /dev/null +++ b/vreapis/auth/test_KeycloakAuthentication.py @@ -0,0 +1,19 @@ +from django.test import TestCase, RequestFactory +from django.conf import settings +from urllib.parse import urlencode +from auth.Keycloak import KeycloakAuthentication +from django.contrib.auth.models import User +import common + + +class KeycloakAuthenticationTestCase(TestCase): + def test_authenticate(self): + post_data: dict[str, str] = {'client_id': 'myclient', 'grant_type': 'password', 'scope': 'openid', 'username': 'u', 'password': 'u'} + response_body = common.session.post(settings.KEYCLOAK_LOGIN_URL, headers={'Content-Type': 'application/x-www-form-urlencoded'}, data=urlencode(post_data), verify=settings.ALLOW_INSECURE_TLS).json() + access_token = response_body['access_token'] + + request_factory = RequestFactory() + request = request_factory.post('', HTTP_AUTHORIZATION=access_token) + authenticator = KeycloakAuthentication() + user, _ = authenticator.authenticate(request) + assert user.username == post_data['username'] diff --git a/vreapis/containerizer/tests.py b/vreapis/containerizer/tests.py index 7ce503c2..0b4501e2 100644 --- a/vreapis/containerizer/tests.py +++ b/vreapis/containerizer/tests.py @@ -1,3 +1,4 @@ from django.test import TestCase # Create your tests here. + From 65311fcc028c27c2242c9327b3072af353edff83 Mon Sep 17 00:00:00 2001 From: AndyBRoswell Date: Sun, 26 May 2024 19:42:33 +0200 Subject: [PATCH 004/149] keycloak token auth with unit test --- vreapis/auth/test_Keycloak.py | 18 ++++++++++++++++++ vreapis/auth/test_KeycloakAuthentication.py | 19 ------------------- 2 files changed, 18 insertions(+), 19 deletions(-) create mode 100644 vreapis/auth/test_Keycloak.py delete mode 100644 vreapis/auth/test_KeycloakAuthentication.py diff --git a/vreapis/auth/test_Keycloak.py b/vreapis/auth/test_Keycloak.py new file mode 100644 index 00000000..b4565ee8 --- /dev/null +++ b/vreapis/auth/test_Keycloak.py @@ -0,0 +1,18 @@ +from django.test import TestCase, RequestFactory +from django.conf import settings +from urllib.parse import urlencode +from auth.Keycloak import KeycloakAuthentication +import common + + +class KeycloakAuthenticationTestCase(TestCase): + def test_authenticate(self): # todo. what if authentication failed? what if refresh token is used? + post_data: dict[str, str] = {'client_id': settings.KEYCLOAK_CLIENT_ID, 'grant_type': 'password', 'scope': 'openid', 'username': 'u', 'password': 'u'} + response_body: dict[str, any] = common.session.post(settings.KEYCLOAK_LOGIN_URL, headers={'Content-Type': 'application/x-www-form-urlencoded'}, data=urlencode(post_data), verify=settings.ALLOW_INSECURE_TLS).json() + access_token: str = response_body['access_token'] + + request_factory = RequestFactory() + request = request_factory.post('', HTTP_AUTHORIZATION=access_token) + authenticator = KeycloakAuthentication() + user, _ = authenticator.authenticate(request) + assert user.username == post_data['username'] diff --git a/vreapis/auth/test_KeycloakAuthentication.py b/vreapis/auth/test_KeycloakAuthentication.py deleted file mode 100644 index 3299bfa3..00000000 --- a/vreapis/auth/test_KeycloakAuthentication.py +++ /dev/null @@ -1,19 +0,0 @@ -from django.test import TestCase, RequestFactory -from django.conf import settings -from urllib.parse import urlencode -from auth.Keycloak import KeycloakAuthentication -from django.contrib.auth.models import User -import common - - -class KeycloakAuthenticationTestCase(TestCase): - def test_authenticate(self): - post_data: dict[str, str] = {'client_id': 'myclient', 'grant_type': 'password', 'scope': 'openid', 'username': 'u', 'password': 'u'} - response_body = common.session.post(settings.KEYCLOAK_LOGIN_URL, headers={'Content-Type': 'application/x-www-form-urlencoded'}, data=urlencode(post_data), verify=settings.ALLOW_INSECURE_TLS).json() - access_token = response_body['access_token'] - - request_factory = RequestFactory() - request = request_factory.post('', HTTP_AUTHORIZATION=access_token) - authenticator = KeycloakAuthentication() - user, _ = authenticator.authenticate(request) - assert user.username == post_data['username'] From 4047217e0941245676d49980d773de1bcc867940 Mon Sep 17 00:00:00 2001 From: AndyBRoswell Date: Sun, 26 May 2024 20:55:58 +0200 Subject: [PATCH 005/149] keycloak token auth with unit test --- vreapis/auth/Keycloak.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/vreapis/auth/Keycloak.py b/vreapis/auth/Keycloak.py index 2868f57b..d2793af6 100644 --- a/vreapis/auth/Keycloak.py +++ b/vreapis/auth/Keycloak.py @@ -1,6 +1,7 @@ import os from requests.exceptions import RequestException from rest_framework.authentication import BaseAuthentication +from rest_framework.exceptions import AuthenticationFailed from django.contrib.auth.models import User from django.http import HttpRequest from django.conf import settings @@ -14,28 +15,28 @@ class KeycloakAuthentication(BaseAuthentication): def authenticate(self, request: HttpRequest): received_token: str = request.headers.get('Authorization', '') if received_token == '': - return None + raise AuthenticationFailed('Authorization header not found. Login using Keycloak to get an access token.') token_header: dict[str, any] = jwt.get_unverified_header(received_token) try: verif_body: dict[str, any] = common.session.get(settings.KEYCLOAK_VERIF_URL, verify=settings.ALLOW_INSECURE_TLS).json() except RequestException as e: - common.logger.error(f'Error verifying token: {e}') + common.logger.error(f'Internal server error while getting Keycloak public key.') return None kid: str = token_header.get('kid', '') if kid == '': common.logger.error(f'JWT Key ID (kid) not found:{os.linesep}{received_token}') - return None + raise AuthenticationFailed(f'Invalid token format: JWT Key ID (kid) not found.') verif_keys: list[dict[str, any]] = verif_body.get('keys', []) key = next((key for key in verif_keys if key['kid'] == kid), None) if key is None: - common.logger.error(f'Keycloak public key not found for received kid {kid}') + common.logger.error(f'Keycloak public key not found.') return None pub_key = jwt.algorithms.RSAAlgorithm.from_jwk(key) access_token: dict[str, any] = jwt.decode(received_token, pub_key, algorithms=key['alg'], audience='account') username: str = access_token.get('preferred_username', '') if username == '': common.logger.error(f'Username (preferred_username) not found in token:{os.linesep}{received_token}') - return None + raise AuthenticationFailed(f'Invalid token format: Username (preferred_username) not found.') try: user: User | None = User.objects.get(username=username) except User.DoesNotExist: From 2cc5af4a1e18e08c6704b4b6b9e6d5613e3b2d2a Mon Sep 17 00:00:00 2001 From: AndyBRoswell Date: Mon, 27 May 2024 01:46:58 +0200 Subject: [PATCH 006/149] keycloak token auth with unit test --- vreapis/auth/Keycloak.py | 2 +- vreapis/common.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/vreapis/auth/Keycloak.py b/vreapis/auth/Keycloak.py index d2793af6..b4e140e2 100644 --- a/vreapis/auth/Keycloak.py +++ b/vreapis/auth/Keycloak.py @@ -20,7 +20,7 @@ def authenticate(self, request: HttpRequest): try: verif_body: dict[str, any] = common.session.get(settings.KEYCLOAK_VERIF_URL, verify=settings.ALLOW_INSECURE_TLS).json() except RequestException as e: - common.logger.error(f'Internal server error while getting Keycloak public key.') + common.logger.error(f'[{e.response.status_code}] Error while getting Keycloak public key.') return None kid: str = token_header.get('kid', '') if kid == '': diff --git a/vreapis/common.py b/vreapis/common.py index f3843084..bab5bf4b 100644 --- a/vreapis/common.py +++ b/vreapis/common.py @@ -4,7 +4,7 @@ # customized requests.Session [w/ auto retry] session = requests.Session() -retry_adapter = requests.adapters.HTTPAdapter(max_retries=urllib3.Retry(total=5, status_forcelist=[500])) +retry_adapter = requests.adapters.HTTPAdapter(max_retries=urllib3.Retry(total=10, status_forcelist=[500])) session.mount('http://', retry_adapter) session.mount('https://', retry_adapter) From c9b4ce854e04244eaf5284050bc0ef172116594e Mon Sep 17 00:00:00 2001 From: AndyBRoswell Date: Mon, 27 May 2024 11:09:56 +0200 Subject: [PATCH 007/149] keycloak token auth with unit test --- vreapis/auth/Keycloak.py | 8 ++++++-- vreapis/containerizer/tests.py | 17 +++++++++++++++-- vreapis/containerizer/views.py | 4 +++- vreapis/vreapis/settings/base.py | 2 ++ 4 files changed, 26 insertions(+), 5 deletions(-) diff --git a/vreapis/auth/Keycloak.py b/vreapis/auth/Keycloak.py index b4e140e2..158dc5a4 100644 --- a/vreapis/auth/Keycloak.py +++ b/vreapis/auth/Keycloak.py @@ -12,13 +12,17 @@ class KeycloakAuthentication(BaseAuthentication): + @staticmethod + def get_verif_inf() -> dict[str, any]: + return common.session.get(settings.KEYCLOAK_VERIF_URL, verify=settings.ALLOW_INSECURE_TLS).json() + def authenticate(self, request: HttpRequest): received_token: str = request.headers.get('Authorization', '') if received_token == '': raise AuthenticationFailed('Authorization header not found. Login using Keycloak to get an access token.') token_header: dict[str, any] = jwt.get_unverified_header(received_token) try: - verif_body: dict[str, any] = common.session.get(settings.KEYCLOAK_VERIF_URL, verify=settings.ALLOW_INSECURE_TLS).json() + verif_inf: dict[str, any] = KeycloakAuthentication.get_verif_inf() except RequestException as e: common.logger.error(f'[{e.response.status_code}] Error while getting Keycloak public key.') return None @@ -26,7 +30,7 @@ def authenticate(self, request: HttpRequest): if kid == '': common.logger.error(f'JWT Key ID (kid) not found:{os.linesep}{received_token}') raise AuthenticationFailed(f'Invalid token format: JWT Key ID (kid) not found.') - verif_keys: list[dict[str, any]] = verif_body.get('keys', []) + verif_keys: list[dict[str, any]] = verif_inf.get('keys', []) key = next((key for key in verif_keys if key['kid'] == kid), None) if key is None: common.logger.error(f'Keycloak public key not found.') diff --git a/vreapis/containerizer/tests.py b/vreapis/containerizer/tests.py index 0b4501e2..60ddde66 100644 --- a/vreapis/containerizer/tests.py +++ b/vreapis/containerizer/tests.py @@ -1,4 +1,17 @@ -from django.test import TestCase +import os +from django.test import TestCase, Client +from django.conf import settings +from urllib.parse import urlencode +import requests -# Create your tests here. +class ContainerizerTestCase(TestCase): + @staticmethod + def Keycloak_login() -> dict[str, any]: + return requests.post(settings.KEYCLOAK_LOGIN_URL, headers={'Content-Type': 'application/x-www-form-urlencoded'}, data=urlencode({'client_id': 'myclient', 'grant_type': 'password', 'scope': 'openid', 'username': 'u', 'password': 'u'}), verify=settings.ALLOW_INSECURE_TLS).json() + + def test_get_base_images(self): + client = Client() + + login_inf = ContainerizerTestCase.Keycloak_login() + response = client.post() \ No newline at end of file diff --git a/vreapis/containerizer/views.py b/vreapis/containerizer/views.py index c46d2827..3bd6aa89 100644 --- a/vreapis/containerizer/views.py +++ b/vreapis/containerizer/views.py @@ -1,12 +1,14 @@ import os from rest_framework.response import Response from rest_framework import status -from rest_framework.decorators import api_view +from rest_framework.decorators import api_view, authentication_classes import requests +from auth.Keycloak import KeycloakAuthentication import common @api_view(['GET']) +@authentication_classes([KeycloakAuthentication]) def get_base_images(request): url: str = os.getenv('BASE_IMAGE_TAGS_URL', 'https://github.com/QCDIS/NaaVRE-flavors/releases/latest/download/base_image_tags.json') common.logger.debug(f'Base image tags URL: {url}') diff --git a/vreapis/vreapis/settings/base.py b/vreapis/vreapis/settings/base.py index 9c26bef1..1b126ae0 100644 --- a/vreapis/vreapis/settings/base.py +++ b/vreapis/vreapis/settings/base.py @@ -189,6 +189,8 @@ # CUSTOM VARS +API_ENDPOINT: str = os.getenv('API_ENDPOINT', "https://naavre-dev.minikube.test/vre-api-test") + KEYCLOAK_URL: str = os.getenv('KEYCLOAK_URL', 'https://naavre-dev.minikube.test/auth') KEYCLOAK_REALM: str = os.getenv('KEYCLOAK_REALM', 'vre') KEYCLOAK_LOGIN_URL: str = f'{KEYCLOAK_URL}/realms/{KEYCLOAK_REALM}/protocol/openid-connect/token' From cc0ad39714098ddbc6a268f7c7322c60e2d0c65d Mon Sep 17 00:00:00 2001 From: AndyBRoswell Date: Tue, 28 May 2024 20:37:53 +0200 Subject: [PATCH 008/149] api/containerizer/baseimagetags test --- vreapis/containerizer/tests.py | 18 ++++++++++++++---- vreapis/containerizer/views.py | 2 +- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/vreapis/containerizer/tests.py b/vreapis/containerizer/tests.py index 60ddde66..c9094aed 100644 --- a/vreapis/containerizer/tests.py +++ b/vreapis/containerizer/tests.py @@ -1,6 +1,7 @@ -import os +import json from django.test import TestCase, Client from django.conf import settings +from django.contrib.auth.models import User from urllib.parse import urlencode import requests @@ -10,8 +11,17 @@ class ContainerizerTestCase(TestCase): def Keycloak_login() -> dict[str, any]: return requests.post(settings.KEYCLOAK_LOGIN_URL, headers={'Content-Type': 'application/x-www-form-urlencoded'}, data=urlencode({'client_id': 'myclient', 'grant_type': 'password', 'scope': 'openid', 'username': 'u', 'password': 'u'}), verify=settings.ALLOW_INSECURE_TLS).json() - def test_get_base_images(self): + def test_get_base_images_with_builtin_auth_bypassed(self): client = Client() + dummy_username = f'test' + try: + dummy_user: User = User.objects.get(username=dummy_username) + except User.DoesNotExist: + dummy_user = User.objects.create_user(dummy_username, password='0') + client.force_login(dummy_user) - login_inf = ContainerizerTestCase.Keycloak_login() - response = client.post() \ No newline at end of file + response = client.get('/api/containerizer/baseimagetags/') + self.assertEqual(response.status_code, 200) + images = response.json() + self.assertIsInstance(images, dict) + self.assertGreater(len(images), 0) diff --git a/vreapis/containerizer/views.py b/vreapis/containerizer/views.py index 3bd6aa89..d603aa09 100644 --- a/vreapis/containerizer/views.py +++ b/vreapis/containerizer/views.py @@ -8,7 +8,7 @@ @api_view(['GET']) -@authentication_classes([KeycloakAuthentication]) +# @authentication_classes([KeycloakAuthentication])e def get_base_images(request): url: str = os.getenv('BASE_IMAGE_TAGS_URL', 'https://github.com/QCDIS/NaaVRE-flavors/releases/latest/download/base_image_tags.json') common.logger.debug(f'Base image tags URL: {url}') From 1909e8698a23402ef136d151c9a1bae703f1995a Mon Sep 17 00:00:00 2001 From: AndyBRoswell Date: Wed, 29 May 2024 15:55:37 +0200 Subject: [PATCH 009/149] [bugfix] backoff factor of auto retry; [test] baseimagetags --- vreapis/common.py | 2 +- vreapis/containerizer/tests.py | 7 ++++--- vreapis/containerizer/views.py | 4 ++-- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/vreapis/common.py b/vreapis/common.py index bab5bf4b..bbcefdc7 100644 --- a/vreapis/common.py +++ b/vreapis/common.py @@ -4,7 +4,7 @@ # customized requests.Session [w/ auto retry] session = requests.Session() -retry_adapter = requests.adapters.HTTPAdapter(max_retries=urllib3.Retry(total=10, status_forcelist=[500])) +retry_adapter = requests.adapters.HTTPAdapter(max_retries=urllib3.Retry(total=10, backoff_factor=0.1, backoff_max=2, status_forcelist=[500, 502, 503, 504])) session.mount('http://', retry_adapter) session.mount('https://', retry_adapter) diff --git a/vreapis/containerizer/tests.py b/vreapis/containerizer/tests.py index c9094aed..11727093 100644 --- a/vreapis/containerizer/tests.py +++ b/vreapis/containerizer/tests.py @@ -11,14 +11,15 @@ class ContainerizerTestCase(TestCase): def Keycloak_login() -> dict[str, any]: return requests.post(settings.KEYCLOAK_LOGIN_URL, headers={'Content-Type': 'application/x-www-form-urlencoded'}, data=urlencode({'client_id': 'myclient', 'grant_type': 'password', 'scope': 'openid', 'username': 'u', 'password': 'u'}), verify=settings.ALLOW_INSECURE_TLS).json() - def test_get_base_images_with_builtin_auth_bypassed(self): + def test_get_base_images(self): client = Client() dummy_username = f'test' + dummy_password = '0' try: dummy_user: User = User.objects.get(username=dummy_username) except User.DoesNotExist: - dummy_user = User.objects.create_user(dummy_username, password='0') - client.force_login(dummy_user) + dummy_user = User.objects.create_user(dummy_username, password=dummy_password) + client.login(username=dummy_username, password=dummy_password) response = client.get('/api/containerizer/baseimagetags/') self.assertEqual(response.status_code, 200) diff --git a/vreapis/containerizer/views.py b/vreapis/containerizer/views.py index d603aa09..7dea0151 100644 --- a/vreapis/containerizer/views.py +++ b/vreapis/containerizer/views.py @@ -1,14 +1,14 @@ import os +from rest_framework.authentication import TokenAuthentication from rest_framework.response import Response from rest_framework import status from rest_framework.decorators import api_view, authentication_classes import requests -from auth.Keycloak import KeycloakAuthentication import common @api_view(['GET']) -# @authentication_classes([KeycloakAuthentication])e +# @authentication_classes([TokenAuthentication]) def get_base_images(request): url: str = os.getenv('BASE_IMAGE_TAGS_URL', 'https://github.com/QCDIS/NaaVRE-flavors/releases/latest/download/base_image_tags.json') common.logger.debug(f'Base image tags URL: {url}') From af97e95ce08324513c602fbb565c2ee5ddf2e7b0 Mon Sep 17 00:00:00 2001 From: AndyBRoswell Date: Wed, 29 May 2024 19:42:37 +0200 Subject: [PATCH 010/149] enable TokenAuthentication & IsAuthenticated --- vreapis/containerizer/views.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/vreapis/containerizer/views.py b/vreapis/containerizer/views.py index 7dea0151..0a063fa3 100644 --- a/vreapis/containerizer/views.py +++ b/vreapis/containerizer/views.py @@ -1,14 +1,16 @@ import os from rest_framework.authentication import TokenAuthentication +from rest_framework.permissions import IsAuthenticated from rest_framework.response import Response from rest_framework import status -from rest_framework.decorators import api_view, authentication_classes +from rest_framework.decorators import api_view, authentication_classes, permission_classes import requests import common @api_view(['GET']) -# @authentication_classes([TokenAuthentication]) +@authentication_classes([TokenAuthentication]) +@permission_classes([IsAuthenticated]) def get_base_images(request): url: str = os.getenv('BASE_IMAGE_TAGS_URL', 'https://github.com/QCDIS/NaaVRE-flavors/releases/latest/download/base_image_tags.json') common.logger.debug(f'Base image tags URL: {url}') From 3212e3656efb22f0ba7c42df5e77fd1e0952553d Mon Sep 17 00:00:00 2001 From: AndyBRoswell Date: Fri, 31 May 2024 00:14:03 +0200 Subject: [PATCH 011/149] StaticTokenAuthentication --- vreapis/auth/simple.py | 22 ++++++++++++++++++++++ vreapis/auth/test_simple.py | 21 +++++++++++++++++++++ vreapis/containerizer/views.py | 3 ++- vreapis/vreapis/settings/base.py | 2 ++ 4 files changed, 47 insertions(+), 1 deletion(-) create mode 100644 vreapis/auth/simple.py create mode 100644 vreapis/auth/test_simple.py diff --git a/vreapis/auth/simple.py b/vreapis/auth/simple.py new file mode 100644 index 00000000..53ba79e4 --- /dev/null +++ b/vreapis/auth/simple.py @@ -0,0 +1,22 @@ +from rest_framework.authentication import BaseAuthentication +from rest_framework.exceptions import AuthenticationFailed +from django.contrib.auth.models import User +from django.http import HttpRequest +from django.conf import settings + + +class StaticTokenAuthentication(BaseAuthentication): + dummy_user: User + + def __init__(self): + try: + StaticTokenAuthentication.dummy_user = User.objects.get(username=settings.NAAVRE_API_TOKEN) + except User.DoesNotExist: + StaticTokenAuthentication.dummy_user = User.objects.create_user(settings.NAAVRE_API_TOKEN, password='0') + + def authenticate(self, request: HttpRequest): + access_token: str = request.headers.get('Authorization', '') + print(access_token == f'Token {settings.NAAVRE_API_TOKEN}') + if access_token != f'Token {settings.NAAVRE_API_TOKEN}': + raise AuthenticationFailed(f'Invalid token') + return StaticTokenAuthentication.dummy_user, None diff --git a/vreapis/auth/test_simple.py b/vreapis/auth/test_simple.py new file mode 100644 index 00000000..99789e06 --- /dev/null +++ b/vreapis/auth/test_simple.py @@ -0,0 +1,21 @@ +from django.test import TestCase, RequestFactory +from django.conf import settings +from rest_framework.exceptions import AuthenticationFailed + +from auth.simple import StaticTokenAuthentication + + +class StaticTokenAuthenticationTestCase(TestCase): + tokens_accepted = [f'Token {settings.NAAVRE_API_TOKEN}', ] + tokens_not_accepted = [f'{settings.NAAVRE_API_TOKEN}', f'', f'0123456789abcdef0123456789abcdef01234567', ] + + def test_authenticate(self): + request_factory = RequestFactory() + authenticator = StaticTokenAuthentication() + print(StaticTokenAuthenticationTestCase.tokens_accepted) + for token in StaticTokenAuthenticationTestCase.tokens_accepted: + request = request_factory.post('', HTTP_AUTHORIZATION=token) + user, _ = authenticator.authenticate(request) + for token in StaticTokenAuthenticationTestCase.tokens_not_accepted: + request = request_factory.post('', HTTP_AUTHORIZATION=token) + self.assertRaises(AuthenticationFailed, authenticator.authenticate, request) diff --git a/vreapis/containerizer/views.py b/vreapis/containerizer/views.py index 0a063fa3..46e930fb 100644 --- a/vreapis/containerizer/views.py +++ b/vreapis/containerizer/views.py @@ -6,10 +6,11 @@ from rest_framework.decorators import api_view, authentication_classes, permission_classes import requests import common +from auth.simple import StaticTokenAuthentication @api_view(['GET']) -@authentication_classes([TokenAuthentication]) +@authentication_classes([StaticTokenAuthentication]) @permission_classes([IsAuthenticated]) def get_base_images(request): url: str = os.getenv('BASE_IMAGE_TAGS_URL', 'https://github.com/QCDIS/NaaVRE-flavors/releases/latest/download/base_image_tags.json') diff --git a/vreapis/vreapis/settings/base.py b/vreapis/vreapis/settings/base.py index 1b126ae0..654a7f3c 100644 --- a/vreapis/vreapis/settings/base.py +++ b/vreapis/vreapis/settings/base.py @@ -196,3 +196,5 @@ KEYCLOAK_LOGIN_URL: str = f'{KEYCLOAK_URL}/realms/{KEYCLOAK_REALM}/protocol/openid-connect/token' KEYCLOAK_VERIF_URL: str = f'{KEYCLOAK_URL}/realms/{KEYCLOAK_REALM}/protocol/openid-connect/certs' KEYCLOAK_CLIENT_ID: str = os.getenv('KEYCLOAK_CLIENT_ID', 'myclient') + +NAAVRE_API_TOKEN: str = os.getenv('NAAVRE_API_TOKEN') From 4ef15fba8b10fbc482455b8195bb392d642d4d38 Mon Sep 17 00:00:00 2001 From: AndyBRoswell Date: Fri, 31 May 2024 10:39:15 +0200 Subject: [PATCH 012/149] [StaticTokenAuthentication] remove hardcoded password --- vreapis/auth/simple.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/vreapis/auth/simple.py b/vreapis/auth/simple.py index 53ba79e4..5cc7ce6b 100644 --- a/vreapis/auth/simple.py +++ b/vreapis/auth/simple.py @@ -3,6 +3,8 @@ from django.contrib.auth.models import User from django.http import HttpRequest from django.conf import settings +import string +import random class StaticTokenAuthentication(BaseAuthentication): @@ -12,7 +14,7 @@ def __init__(self): try: StaticTokenAuthentication.dummy_user = User.objects.get(username=settings.NAAVRE_API_TOKEN) except User.DoesNotExist: - StaticTokenAuthentication.dummy_user = User.objects.create_user(settings.NAAVRE_API_TOKEN, password='0') + StaticTokenAuthentication.dummy_user = User.objects.create_user(settings.NAAVRE_API_TOKEN, password=''.join(random.choice(string.printable) for _ in range(32))) def authenticate(self, request: HttpRequest): access_token: str = request.headers.get('Authorization', '') From fb944936c2f234e6dd0c9dcf0419304b4dc434da Mon Sep 17 00:00:00 2001 From: AndyBRoswell Date: Wed, 5 Jun 2024 00:36:02 +0200 Subject: [PATCH 013/149] porting ExtractorHandler removed JupyterLab dependencies resolved Python dependencies resolving R dependencies --- vreapis/containerizer/tests.py | 115 +++++ vreapis/containerizer/views.py | 124 ++++- vreapis/db/__init__.py | 0 vreapis/db/catalog.py | 183 ++++++++ vreapis/db/cell.py | 172 +++++++ vreapis/db/repository.py | 9 + vreapis/db/sdia_credentials.py | 9 + vreapis/requirements.txt | 6 + vreapis/services/converter.py | 185 ++++++++ vreapis/services/extractor/__init__.py | 0 .../extractor/cell_header.schema.json | 198 ++++++++ vreapis/services/extractor/extractor.py | 56 +++ vreapis/services/extractor/headerextractor.py | 245 ++++++++++ vreapis/services/extractor/pyextractor.py | 279 ++++++++++++ vreapis/services/extractor/rextractor.py | 431 ++++++++++++++++++ vreapis/utils/cors.py | 7 + 16 files changed, 2015 insertions(+), 4 deletions(-) create mode 100644 vreapis/db/__init__.py create mode 100644 vreapis/db/catalog.py create mode 100644 vreapis/db/cell.py create mode 100644 vreapis/db/repository.py create mode 100644 vreapis/db/sdia_credentials.py create mode 100644 vreapis/services/converter.py create mode 100644 vreapis/services/extractor/__init__.py create mode 100644 vreapis/services/extractor/cell_header.schema.json create mode 100644 vreapis/services/extractor/extractor.py create mode 100644 vreapis/services/extractor/headerextractor.py create mode 100644 vreapis/services/extractor/pyextractor.py create mode 100644 vreapis/services/extractor/rextractor.py create mode 100644 vreapis/utils/cors.py diff --git a/vreapis/containerizer/tests.py b/vreapis/containerizer/tests.py index 11727093..7db068e6 100644 --- a/vreapis/containerizer/tests.py +++ b/vreapis/containerizer/tests.py @@ -1,9 +1,20 @@ +import glob +import os import json +import uuid + from django.test import TestCase, Client from django.conf import settings from django.contrib.auth.models import User from urllib.parse import urlencode import requests +import nbformat +from slugify import slugify + +from services.extractor.pyextractor import PyExtractor +from services.extractor.rextractor import RExtractor +from services.converter import ConverterReactFlowChart +from db.cell import Cell class ContainerizerTestCase(TestCase): @@ -26,3 +37,107 @@ def test_get_base_images(self): images = response.json() self.assertIsInstance(images, dict) self.assertGreater(len(images), 0) + + +class ExtractorTestCase(TestCase): + # Reference parameter values for `test_param_values_*.json` + param_values_ref = { + 'param_float': '1.1', + 'param_int': '1', + 'param_list': '[1, 2, 3]', + 'param_string': 'param_string value', + 'param_string_with_comment': 'param_string value', + } + + def __init__(self): + if os.path.exists('resources'): + base_path = 'resources' + elif os.path.exists('jupyterlab_vre/tests/resources/'): + base_path = 'jupyterlab_vre/tests/resources/' + + def create_cell(self, payload_path=None): + with open(payload_path, 'r') as file: + payload = json.load(file) + + cell_index = payload['cell_index'] + notebook = nbformat.reads(json.dumps(payload['notebook']), nbformat.NO_CONVERT) + source = notebook.cells[cell_index].source + if payload['kernel'] == "IRkernel": + extractor = RExtractor(notebook, source) + else: + extractor = PyExtractor(notebook, source) + + title = source.partition('\n')[0] + title = slugify(title) if title and title[ + 0] == "#" else "Untitled" + + if 'JUPYTERHUB_USER' in os.environ: + title += '-' + slugify(os.environ['JUPYTERHUB_USER']) + + ins = {} + outs = {} + params = {} + confs = [] + dependencies = [] + + # Check if cell is code. If cell is for example markdown we get execution from 'extractor.infere_cell_inputs(source)' + if notebook.cells[cell_index].cell_type == 'code': + ins = extractor.infer_cell_inputs() + outs = extractor.infer_cell_outputs() + + confs = extractor.extract_cell_conf_ref() + dependencies = extractor.infer_cell_dependencies(confs) + + node_id = str(uuid.uuid4())[:7] + cell = Cell( + node_id=node_id, + title=title, + task_name=slugify(title.lower()), + original_source=source, + inputs=ins, + outputs=outs, + params=params, + confs=confs, + dependencies=dependencies, + container_source="" + ) + if notebook.cells[cell_index].cell_type == 'code': + cell.integrate_configuration() + params = extractor.extract_cell_params(cell.original_source) + cell.add_params(params) + cell.add_param_values(params) + + return cell + + def extract_cell(self, payload_path): + # Check if file exists + if os.path.exists(payload_path): + cell = self.create_cell(payload_path) + + node = ConverterReactFlowChart.get_node(cell.node_id, cell.title, cell.inputs, cell.outputs, cell.params, ) + + chart = {'offset': {'x': 0, 'y': 0, }, 'scale': 1, 'nodes': {cell.node_id: node}, 'links': {}, 'selected': {}, 'hovered': {}, } + + cell.chart_obj = chart + return cell.toJSON() + return None + + def test_extract_cell(self): + notebooks_json_path = os.path.join(self.base_path, 'notebooks') + notebooks_files = glob.glob( + os.path.join(notebooks_json_path, "*.json") + ) + for notebook_file in notebooks_files: + cell = self.extract_cell(notebook_file) + if cell: + cell = json.loads(cell) + for conf_name in (cell['confs']): + self.assertFalse('conf_' in cell['confs'][conf_name].split('=')[1], 'conf_ values should not contain conf_ prefix in ''assignment') + # All params should have matching values + for param_name in cell['params']: + self.assertTrue(param_name in cell['param_values']) + + # For notebook_file test_param_values_*.json, extracted params should match with self.param_values_ref + if os.path.basename(notebook_file) in ['test_param_values_Python.json', 'test_param_values_R.json', ]: + for param_name in cell['params']: + self.assertTrue(cell['param_values'][param_name] == self.param_values_ref[param_name]) diff --git a/vreapis/containerizer/views.py b/vreapis/containerizer/views.py index 46e930fb..4455d680 100644 --- a/vreapis/containerizer/views.py +++ b/vreapis/containerizer/views.py @@ -1,12 +1,31 @@ import os -from rest_framework.authentication import TokenAuthentication +import json +import traceback +import copy +import hashlib + from rest_framework.permissions import IsAuthenticated from rest_framework.response import Response +from rest_framework.request import Request +from rest_framework.views import APIView +from rest_framework.status import HTTP_400_BAD_REQUEST from rest_framework import status from rest_framework.decorators import api_view, authentication_classes, permission_classes import requests -import common +import nbformat +import jsonschema +from slugify import slugify + +from services.extractor.extractor import DummyExtractor +from services.extractor.headerextractor import HeaderExtractor +from services.extractor.pyextractor import PyExtractor +from services.extractor.rextractor import RExtractor +from services.converter import ConverterReactFlowChart +import utils.cors from auth.simple import StaticTokenAuthentication +from db.catalog import Catalog + +import common @api_view(['GET']) @@ -23,5 +42,102 @@ def get_base_images(request): msg: str = f'Error loading base image tags from {url}\n{e}' common.logger.debug(msg) return Response({'error': msg}, status=status.HTTP_500_INTERNAL_SERVER_ERROR) - Origin: str = request.headers.get('Origin', '*') - return Response(dat, headers={'Access-Control-Allow-Origin': Origin}) + return Response(dat, headers=utils.cors.get_CORS_headers(request)) + + +class ExtractorHandler(APIView): + def extract_cell_by_index(self, notebook, cell_index): + new_nb = copy.deepcopy(notebook) + if cell_index < len(notebook.cells): + new_nb.cells = [notebook.cells[cell_index]] + return new_nb + + def set_notebook_kernel(self, notebook, kernel): + new_nb = copy.deepcopy(notebook) + # Replace kernel name in the notebook metadata + new_nb.metadata.kernelspec.name = kernel + new_nb.metadata.kernelspec.display_name = kernel + new_nb.metadata.kernelspec.language = kernel + return new_nb + + def get(self, request: Request): + msg_json = dict(title="Operation not supported.") + return Response(msg_json) + + def post(self, request: Request): + payload = request.data + common.logger.debug('ExtractorHandler. payload: ' + json.dumps(payload, indent=4)) + kernel = payload['kernel'] + cell_index = payload['cell_index'] + notebook = nbformat.reads(json.dumps(payload['notebook']), nbformat.NO_CONVERT) + + source = notebook.cells[cell_index].source + + if notebook.cells[cell_index].cell_type != 'code': + # dummy extractor for non-code cells (e.g. markdown) + extractor = DummyExtractor(notebook, source) + else: + # extractor based on the cell header + try: + extractor = HeaderExtractor(notebook, source) + except jsonschema.ValidationError as e: + return Response({'message': f"Error in cell header: {e}", 'reason': None, 'traceback': traceback.format_exception(e), }, status=HTTP_400_BAD_REQUEST) + + # Extractor based on code analysis. Used if the cell has no header, or if some values are not specified in the header + if not extractor.is_complete(): + if kernel == "IRkernel": + code_extractor = RExtractor(notebook, source) + else: + code_extractor = PyExtractor(notebook, source) + extractor.add_missing_values(code_extractor) + + extracted_nb = self.extract_cell_by_index(notebook, cell_index) + if kernel == "IRkernel": + extracted_nb = self.set_notebook_kernel(extracted_nb, 'R') + else: + extracted_nb = self.set_notebook_kernel(extracted_nb, 'python3') + + # initialize variables + title = source.partition('\n')[0].strip() + title = slugify(title) if title and title[0] == "#" else "Untitled" + + if 'JUPYTERHUB_USER' in os.environ: + title += '-' + slugify(os.environ['JUPYTERHUB_USER']) + + # If any of these change, we create a new cell in the catalog. This matches the cell properties saved in workflows. + cell_identity_dict = {'title': title, 'params': extractor.params, 'inputs': extractor.ins, 'outputs': extractor.outs, } + cell_identity_str = json.dumps(cell_identity_dict, sort_keys=True) + node_id = hashlib.sha1(cell_identity_str.encode()).hexdigest()[:7] + + cell = Cell( + node_id=node_id, + title=title, + task_name=slugify(title.lower()), + original_source=source, + inputs=extractor.ins, + outputs=extractor.outs, + params={}, + confs=extractor.confs, + dependencies=extractor.dependencies, + container_source="", + kernel=kernel, + notebook_dict=extracted_nb.dict() + ) + cell.integrate_configuration() + extractor.params = extractor.extract_cell_params(cell.original_source) + cell.add_params(extractor.params) + cell.add_param_values(extractor.params) + + node = ConverterReactFlowChart.get_node( + node_id, + title, + set(extractor.ins), + set(extractor.outs), + extractor.params, + ) + + chart = {'offset': {'x': 0, 'y': 0, }, 'scale': 1, 'nodes': {node_id: node}, 'links': {}, 'selected': {}, 'hovered': {}, } + + cell.chart_obj = chart + Catalog.editor_buffer = copy.deepcopy(cell) + return Response(cell.toJSON()) diff --git a/vreapis/db/__init__.py b/vreapis/db/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/vreapis/db/catalog.py b/vreapis/db/catalog.py new file mode 100644 index 00000000..746dff35 --- /dev/null +++ b/vreapis/db/catalog.py @@ -0,0 +1,183 @@ +import logging +import os +from pathlib import Path + +from tinydb import TinyDB, where + +from .cell import Cell +from .repository import Repository +from .sdia_credentials import SDIACredentials + +logger = logging.getLogger(__name__) +logger.setLevel(logging.DEBUG) + + +class Catalog: + naa_vre_path = os.path.join(str(Path.home()), 'NaaVRE') + + if not os.path.exists(naa_vre_path): + os.mkdir(naa_vre_path) + + db_path = os.path.join(naa_vre_path, 'NaaVRE_db.json') + db = TinyDB(db_path) + + cells = db.table('cells') + workflows = db.table('workflows') + repositories = db.table('repositories') + gh_credentials = db.table('gh_credentials') + registry_credentials = db.table('registry_credentials') + search_entry = db.table('search_entries') + + editor_buffer: Cell + + @classmethod + def add_search_entry(cls, query: dict): + cls.search_entry.insert(query) + + @classmethod + def update_cell(cls, cell: Cell): + cls.cells.update(cell.__dict__, where('node_id') == cell.node_id) + + + @classmethod + def get_search_entries(cls): + return cls.search_entry.all() + + @classmethod + def delete_all_search_entries(cls): + return cls.search_entry.truncate() + + @classmethod + def add_cell(cls, cell: Cell): + cls.cells.insert(cell.__dict__) + + @classmethod + def delete_cell_from_task_name(cls, task_name: str): + cell = cls.cells.search(where('task_name') == task_name) + cls.cells.remove(where('task_name') == task_name) + + @classmethod + def delete_cell_from_title(cls, title: str): + cell = cls.cells.search(where('title') == title) + cls.cells.remove(where('title') == title) + + @classmethod + def get_all_cells(cls): + return cls.cells.all() + + @classmethod + def get_registry_credentials(cls): + credentials = cls.registry_credentials.all() + return credentials + + @classmethod + def get_repository_credentials(cls): + credentials = cls.repository.all() + return credentials + + @classmethod + def get_registry_credentials_from_name(cls, name: str): + res = cls.registry_credentials.search(where('name') == name) + if res: + return res[0] + + @classmethod + def add_registry_credentials(cls, cred: Repository): + cls.registry_credentials.insert(cred.__dict__) + + @classmethod + def add_repository_credentials(cls, cred: Repository): + cls.repositories.insert(cred.__dict__) + + @classmethod + def get_gh_credentials(cls): + credentials = cls.gh_credentials.all() + if len(credentials) > 0: + return credentials[0] + + @classmethod + def delete_all_gh_credentials(cls): + cls.gh_credentials.truncate() + + @classmethod + def delete_all_cells(cls): + cls.cells.truncate() + + @classmethod + def delete_all_repository_credentials(cls): + cls.repositories.truncate() + credentials = cls.repositories.all() + ids = [] + for credential in credentials: + ids.append(credential.doc_id) + cls.repositories.remove(doc_ids=ids) + cls.repositories.truncate() + + @classmethod + def delete_all_registry_credentials(cls): + # Looks bad but for now I could not find a way to remove all + credentials = cls.registry_credentials.all() + ids = [] + for credential in credentials: + ids.append(credential.doc_id) + cls.registry_credentials.remove(doc_ids=ids) + cls.registry_credentials.truncate() + + @classmethod + def add_gh_credentials(cls, cred: Repository): + cls.repositories.insert(cred.__dict__) + cls.gh_credentials.insert(cred.__dict__) + + @classmethod + def delete_gh_credentials(cls, url: str): + cls.gh_credentials.remove(where('url') == url) + + @classmethod + def get_credentials_from_username(cls, cred_username) -> SDIACredentials: + res = cls.sdia_credentials.search(where('username') == cred_username) + if res: + return res[0] + + @classmethod + def get_sdia_credentials(cls): + return cls.sdia_credentials.all() + + @classmethod + def get_cell_from_og_node_id(cls, og_node_id) -> Cell: + res = cls.cells.search(where('node_id') == og_node_id) + if res: + return res[0] + else: + logger.warning('Cell not found for og_node_id: ' + og_node_id) + + @classmethod + def get_repositories(cls) -> list: + res = cls.repositories.all() + return res + + @classmethod + def get_repository_from_name(cls, name: str) -> Repository: + res = cls.repositories.search(where('name') == name) + if res: + return res[0] + + @classmethod + def cast_document_to_cell(cls, cell_document): + if not cell_document: + return None + + return Cell( + title=cell_document.get('title', ''), + task_name=cell_document.get('task_name', ''), + original_source=cell_document.get('original_source', ''), + inputs=cell_document.get('inputs', []), + outputs=cell_document.get('outputs', []), + params=cell_document.get('params', []), + confs=cell_document.get('confs', {}), + dependencies=cell_document.get('dependencies', []), + container_source=cell_document.get('container_source', ''), + chart_obj=cell_document.get('chart_obj', {}), + node_id=cell_document.get('node_id', ''), + kernel=cell_document.get('kernel', ''), + notebook_dict=cell_document.get('notebook_dict', {}) + ) diff --git a/vreapis/db/cell.py b/vreapis/db/cell.py new file mode 100644 index 00000000..784ae1d1 --- /dev/null +++ b/vreapis/db/cell.py @@ -0,0 +1,172 @@ +import json +import logging +import re + +from slugify import slugify + +logger = logging.getLogger(__name__) +logger.setLevel(logging.DEBUG) + + +class Cell: + logger = logging.getLogger(__name__) + logger.setLevel(logging.DEBUG) + + title: str + task_name: str + original_source: str + base_image: dict + inputs: list + outputs: list + params: list + param_values: dict + confs: dict + dependencies: list + chart_obj: dict + node_id: str + container_source: str + global_conf: dict + kernel: str + notebook_json: dict + image_version: str + + def __init__( + self, + title, + task_name, + original_source, + inputs, + outputs, + params, + confs, + dependencies, + container_source, + chart_obj=None, + node_id='', + kernel='', + notebook_dict=None, + image_version=None + ) -> None: + + self.title = slugify(title.strip()) + self.task_name = slugify(task_name) + self.original_source = original_source + self.types = dict() + self.add_inputs(inputs) + self.add_outputs(outputs) + self.add_params(params) + self.add_param_values(params) + self.confs = confs + self.all_inputs = list(inputs) + list(params) + self.dependencies = list(sorted(dependencies, key=lambda x: x['name'])) + self.chart_obj = chart_obj + self.node_id = node_id + self.container_source = container_source + self.kernel = kernel + self.notebook_dict = notebook_dict + + def _extract_types(self, vars_dict): + """ Extract types to self.types and return list of var names + + :param vars_dict: {'var1': {'name: 'var1', 'type': 'str'}, 'var2': ...} + :return: ['var1', 'var2', ...] + """ + names = [] + for var_props in vars_dict.values(): + var_type = var_props['type'] + var_name = var_props['name'] + self.types[var_name] = var_type + names.append(var_name) + return names + + def add_inputs(self, inputs): + if isinstance(inputs, dict): + inputs = self._extract_types(inputs) + self.inputs = inputs + + def add_outputs(self, outputs): + if isinstance(outputs, dict): + outputs = self._extract_types(outputs) + self.outputs = outputs + + def add_params(self, params): + if isinstance(params, dict): + params = self._extract_types(params) + self.params = params + + def set_image_version(self, image_version): + self.image_version = image_version + + + def add_param_values(self, params): + self.param_values = {} + if isinstance(params, dict): + for param_props in params.values(): + if 'value' in param_props: + self.param_values[param_props['name']] = param_props['value'] + + def concatenate_all_inputs(self): + self.all_inputs = list(self.inputs) + list(self.params) + + def clean_code(self): + indices_to_remove = [] + lines = self.original_source.splitlines() + self.original_source = "" + + for line_i in range(0, len(lines)): + line = lines[line_i] + # Do not remove line that startswith param_ if not in the self.params + if line.startswith('param_'): + # clean param name + pattern = r"\b(param_\w+)\b" + param_name = re.findall(pattern, line)[0] + if param_name in self.params: + indices_to_remove.append(line_i) + if re.match('^\s*(#|import|from)', line): + indices_to_remove.append(line_i) + + for ir in sorted(indices_to_remove, reverse=True): + lines.pop(ir) + + self.original_source = "\n".join(lines) + + def clean_task_name(self): + self.task_name = slugify(self.task_name) + + def clean_title(self): + self.title = slugify(self.title) + + def integrate_configuration(self): + lines = self.original_source.splitlines() + self.original_source = "" + for idx, conf in enumerate(self.generate_configuration()): + lines.insert(idx, conf) + self.original_source = "\n".join(lines) + + def generate_dependencies(self): + resolves = [] + for d in self.dependencies: + resolve_to = "import %s" % d['name'] + if d['module']: + resolve_to = "from %s %s" % (d['module'], resolve_to) + if d['asname']: + resolve_to += " as %s" % d['asname'] + resolves.append(resolve_to) + return resolves + + def generate_configuration(self): + resolves = [] + for c in self.confs: + resolves.append(self.confs[c]) + return resolves + + def generate_configuration_dict(self): + resolves = [] + for c in self.confs: + assignment = self.confs[c].split('=')[1].replace('=', '').strip() + conf = {c: assignment} + resolves.append(conf) + return resolves + + def toJSON(self): + return json.dumps(self, default=lambda o: o.__dict__, sort_keys=True, indent=4) diff --git a/vreapis/db/repository.py b/vreapis/db/repository.py new file mode 100644 index 00000000..cb20d86e --- /dev/null +++ b/vreapis/db/repository.py @@ -0,0 +1,9 @@ +class Repository: + name: str + url: str + token: str + + def __init__(self, name, url, token) -> None: + self.name = name + self.url = url + self.token = token diff --git a/vreapis/db/sdia_credentials.py b/vreapis/db/sdia_credentials.py new file mode 100644 index 00000000..71ab83fe --- /dev/null +++ b/vreapis/db/sdia_credentials.py @@ -0,0 +1,9 @@ +class SDIACredentials: + username: str + password: str + endpoint: str + + def __init__(self, username, password, endpoint) -> None: + self.username = username + self.password = password + self.endpoint = endpoint diff --git a/vreapis/requirements.txt b/vreapis/requirements.txt index f58fb174..115da79a 100644 --- a/vreapis/requirements.txt +++ b/vreapis/requirements.txt @@ -13,3 +13,9 @@ psycopg2==2.9.9 django-probes==1.7.0 pyjwt==2.8.0 cryptography==42.0.7 +nbformat +python-slugify +colorhash +rpy2==3.5.11 +pyflakes +pytype diff --git a/vreapis/services/converter.py b/vreapis/services/converter.py new file mode 100644 index 00000000..8294bc76 --- /dev/null +++ b/vreapis/services/converter.py @@ -0,0 +1,185 @@ +import uuid + +from colorhash import ColorHash + + +class ConverterReactFlowChart: + + @staticmethod + def get_node(node_id, title, ins, outs, params): + + node = {} + position = {} + ports = {} + properties = {} + + node['id'] = node_id + node['type'] = 'input-output' + position['x'] = 35 + position['y'] = 15 + node['position'] = position + properties['title'] = title + properties['vars'] = list() + properties['params'] = list(params) + properties['inputs'] = list(ins) + properties['outputs'] = list(outs) + properties['og_node_id'] = node_id + node['properties'] = properties + + for i in ins: + ports[i] = {} + ports[i]['properties'] = {} + ports[i]['id'] = i + ports[i]['type'] = 'left' + ports[i]['properties']['color'] = ColorHash(i).hex + properties['vars'].append({ + 'name': i, + 'direction': 'input', + 'type': 'datatype', + 'color': ports[i]['properties']['color'] + }) + + for o in outs: + ports[o] = {} + ports[o]['properties'] = {} + ports[o]['id'] = o + ports[o]['type'] = 'right' + ports[o]['properties']['color'] = ColorHash(o).hex + properties['vars'].append({ + 'name': o, + 'direction': 'output', + 'type': 'datatype', + 'color': ports[o]['properties']['color'] + }) + + node['ports'] = ports + + return node + + +class ConverterReactFlow: + + @staticmethod + def get_input_nodes(ins): + nodes = [] + idx = 0 + + for i in ins: + i_node = {} + i_node['data'] = {} + i_node['position'] = {} + i_node['id'] = i + i_node['type'] = 'input' + i_node['data']['label'] = i + i_node['position']['x'] = 10 + idx * 200 + i_node['position']['y'] = 10 + nodes.append(i_node) + idx += 1 + + return nodes + + @staticmethod + def get_output_nodes(outs): + nodes = [] + idx = 0 + + for o in outs: + o_node = {} + o_node['data'] = {} + o_node['position'] = {} + o_node['id'] = o + o_node['type'] = 'output' + o_node['data']['label'] = o + o_node['position']['x'] = 10 + idx * 200 + o_node['position']['y'] = 200 + nodes.append(o_node) + idx += 1 + + return nodes + + @staticmethod + def get_default_node(title, uuid): + d_node = {} + d_node['data'] = {} + d_node['position'] = {} + d_node['id'] = uuid + d_node['data']['label'] = title + d_node['position']['x'] = 100 + d_node['position']['y'] = 100 + + return d_node + + @staticmethod + def get_edges(d_node_id, ins, outs): + edges = [] + + for i in ins: + i_edge = {} + i_edge['id'] = "%s-%s" % (i, d_node_id) + i_edge['source'] = i + i_edge['target'] = d_node_id + i_edge['animated'] = True + edges.append(i_edge) + + for o in outs: + o_edge = {} + o_edge['id'] = "%s-%s" % (d_node_id, o) + o_edge['source'] = d_node_id + o_edge['target'] = o + o_edge['animated'] = True + edges.append(o_edge) + + return edges + + +class ConverterFlume: + + @staticmethod + def get_ports(ins, outs): + colors = [ + 'yellow', + 'orange', + 'red', + 'pink', + 'purple', + 'blue', + 'green', + 'grey' + ] + + ports = set() + ports_types = [] + + ports.update(ins) + ports.update(outs) + + color_i = 0 + for port in ports: + p_type = {} + p_type['type'] = port + p_type['name'] = port + p_type['label'] = port + p_type['color'] = colors[color_i], + color_i = (color_i + 1) % len(colors) + + ports_types.append(p_type) + + return ports_types + + @staticmethod + def get_node(source, ports, ins, outs): + title = source.partition('\n')[0] + short_uuid = str(uuid.uuid4())[:7] + + n_type = {} + n_type['type'] = short_uuid + n_type['label'] = title if title[0] == "#" else "Untitled %s" % short_uuid + n_type['description'] = short_uuid + + ports_in = [p for p in ports if p['name'] in ins] + ports_out = [p for p in ports if p['name'] in outs] + + n_type['inputs'] = ports_in + n_type['outputs'] = ports_out + + return n_type diff --git a/vreapis/services/extractor/__init__.py b/vreapis/services/extractor/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/vreapis/services/extractor/cell_header.schema.json b/vreapis/services/extractor/cell_header.schema.json new file mode 100644 index 00000000..7e84d04e --- /dev/null +++ b/vreapis/services/extractor/cell_header.schema.json @@ -0,0 +1,198 @@ +{ + "$schema": "http://json-schema.org/draft/2020-12/schema", + "$ref": "#/definitions/CellHeader", + "definitions": { + "CellHeader": { + "title": "Cell header", + "type": "object", + "additionalProperties": true, + "properties": { + "NaaVRE": { + "$ref": "#/definitions/NaaVRECellMetadata" + } + }, + "required": [ + "NaaVRE" + ] + }, + "NaaVRECellMetadata": { + "title": "NaaVRE cell metadata", + "type": "object", + "additionalProperties": false, + "properties": { + "cell": { + "$ref": "#/definitions/Cell" + } + }, + "required": [ + "cell" + ] + }, + "Cell": { + "title": "Cell", + "type": "object", + "additionalProperties": false, + "properties": { + "inputs": { + "type": "array", + "items": { + "$ref": "#/definitions/IOElement" + } + }, + "outputs": { + "type": "array", + "items": { + "$ref": "#/definitions/IOElement" + } + }, + "params": { + "type": "array", + "items": { + "$ref": "#/definitions/ParamElement" + } + }, + "confs": { + "type": "array", + "items": { + "$ref": "#/definitions/ConfElement" + } + }, + "dependencies": { + "type": "array", + "items": { + "$ref": "#/definitions/DependencyElement" + } + } + } + }, + "VarType": { + "title": "Variable type", + "type": "string", + "enum": [ + "Integer", + "Float", + "String", + "List" + ] + }, + "VarDefaultValue": { + "title": "Param variable default_value", + "description": "Used to pre-fill workflow parameters", + "type": ["string", "number", "array"] + }, + "VarAssignation": { + "title": "Conf variable assignation property", + "description": "Full variable assignation, e.g. 'conf_answer = 42'", + "type": "string" + }, + "ElementVarName": { + "title": "Variable name", + "description": "simple variable name, as string ('name')", + "type": "string" + }, + "ElementVarNameType": { + "title": "Variable name and type", + "description": "simple variable name and type, as object ({'name': 'type'})", + "type": "object", + "additionalProperties": false, + "patternProperties": { + "^.*$": { + "$ref": "#/definitions/VarType" + } + } + }, + "IOElementVarDict": { + "title": "Full i/o variable description", + "type": "object", + "additionalProperties": false, + "minProperties": 1, + "maxProperties": 1, + "patternProperties": { + "^.*$": { + "type": "object", + "additionalProperties": false, + "properties": { + "type": {"$ref": "#/definitions/VarType"} + } + } + } + }, + "ParamElementVarDict": { + "title": "Full param variable description", + "type": "object", + "additionalProperties": false, + "minProperties": 1, + "maxProperties": 1, + "patternProperties": { + "^param_.*$": { + "type": "object", + "additionalProperties": false, + "properties": { + "type": {"$ref": "#/definitions/VarType"}, + "default_value": {"$ref": "#/definitions/VarDefaultValue"} + } + } + } + }, + "ConfElementVarDict": { + "title": "Full param variable description", + "type": "object", + "additionalProperties": false, + "minProperties": 1, + "maxProperties": 1, + "patternProperties": { + "^conf_.*$": { + "type": "object", + "additionalProperties": false, + "properties": { + "assignation": {"$ref": "#/definitions/VarAssignation"} + }, + "required": ["assignation"] + } + } + }, + "DependencyElementVarDict": { + "title": "Full param variable description", + "type": "object", + "additionalProperties": false, + "properties": { + "name": {"type": "string"}, + "asname": {"type": ["string", "null"]}, + "module": {"type": "string"} + }, + "required": ["name"] + }, + "IOElement": { + "title": "I/O element", + "description": "description input and output variable elements", + "anyOf": [ + {"$ref": "#/definitions/ElementVarName"}, + {"$ref": "#/definitions/ElementVarNameType"}, + {"$ref": "#/definitions/IOElementVarDict"} + ] + }, + "ParamElement": { + "title": "Param element", + "description": "description param variable elements", + "anyOf": [ + {"$ref": "#/definitions/ElementVarName"}, + {"$ref": "#/definitions/ElementVarNameType"}, + {"$ref": "#/definitions/ParamElementVarDict"} + ] + }, + "ConfElement": { + "title": "Conf element", + "description": "description conf variable elements", + "anyOf": [ + {"$ref": "#/definitions/ConfElementVarDict"} + ] + }, + "DependencyElement": { + "title": "Dependency element", + "description": "description dependency elements", + "anyOf": [ + {"$ref": "#/definitions/DependencyElementVarDict"} + ] + } + } +} diff --git a/vreapis/services/extractor/extractor.py b/vreapis/services/extractor/extractor.py new file mode 100644 index 00000000..dcd3fb86 --- /dev/null +++ b/vreapis/services/extractor/extractor.py @@ -0,0 +1,56 @@ +import abc + + +class Extractor(abc.ABC): + ins: dict + outs: dict + params: dict + confs: list + dependencies: list + + def __init__(self, notebook, cell_source): + self.notebook = notebook + self.cell_source = cell_source + + self.ins = self.infer_cell_inputs() + self.outs = self.infer_cell_outputs() + self.params = self.extract_cell_params(cell_source) + self.confs = self.extract_cell_conf_ref() + self.dependencies = self.infer_cell_dependencies(self.confs) + + @abc.abstractmethod + def infer_cell_inputs(self): + pass + + @abc.abstractmethod + def infer_cell_outputs(self): + pass + + @abc.abstractmethod + def extract_cell_params(self, source): + pass + + @abc.abstractmethod + def extract_cell_conf_ref(self): + pass + + @abc.abstractmethod + def infer_cell_dependencies(self, confs): + pass + + +class DummyExtractor(Extractor): + def infer_cell_inputs(self): + return {} + + def infer_cell_outputs(self): + return {} + + def extract_cell_params(self, source): + return {} + + def extract_cell_conf_ref(self): + return [] + + def infer_cell_dependencies(self, confs): + return [] diff --git a/vreapis/services/extractor/headerextractor.py b/vreapis/services/extractor/headerextractor.py new file mode 100644 index 00000000..38429c80 --- /dev/null +++ b/vreapis/services/extractor/headerextractor.py @@ -0,0 +1,245 @@ +import json +import logging +import os +import re +from typing import Literal, Union + +import jsonschema +import yaml + +from .extractor import Extractor + + +class HeaderExtractor(Extractor): + """ Extracts cells using information defined by the user in its header + + Cells should contain a comment with a yaml block defining inputs, outputs, + params and confs. Eg: + + # My cell + # --- + # NaaVRE: + # cell: + # inputs: + # - my_input: + # type: String + # - my_other_input: + # type: Integer + # outputs: + # - my_output: + # type: List + # params: + # - param_something: + # type: String + # default_value: "my default value" + # confs: + # - conf_something_else: + # assignation: "conf_something_else = 'my other value'" + # ... + [cell code] + + The document is validated with the schema `cell_header.schema.json` + + """ + ins: Union[dict, None] + outs: Union[dict, None] + params: Union[dict, None] + confs: Union[list, None] + dependencies: Union[list, None] + + def __init__(self, notebook, cell_source): + self.re_yaml_doc_in_comment = re.compile( + (r"^(?:.*\n)*" + r"\s*#\s*---\s*\n" + r"((?:\s*#.*\n)+?)" + r"\s*#\s*\.\.\.\s*\n" + ), + re.MULTILINE) + self.schema = self._load_schema() + self.cell_header = self._extract_header(cell_source) + self._external_extract_cell_params = None + + super().__init__(notebook, cell_source) + + @staticmethod + def _load_schema(): + filename = os.path.join( + os.path.dirname(__file__), + 'cell_header.schema.json') + with open(filename) as f: + schema = json.load(f) + return schema + + def enabled(self): + return self.cell_header is not None + + def is_complete(self): + return ( + (self.ins is not None) + and (self.outs is not None) + and (self.params is not None) + and (self.confs is not None) + and (self.dependencies is not None) + ) + + def _extract_header(self, cell_source): + # get yaml document from cell comments + m = self.re_yaml_doc_in_comment.match(cell_source) + if not (m and m.groups()): + return None + yaml_doc = m.group(1) + # remove comment symbol + yaml_doc = '\n'.join([ + line.lstrip().lstrip('#') + for line in yaml_doc.splitlines() + ]) + # parse yaml + header = yaml.safe_load(yaml_doc) + # validate schema + try: + jsonschema.validate(header, self.schema) + except jsonschema.ValidationError as e: + logging.getLogger().debug(f"Cell header validation error: {e}") + raise e + return header + + def add_missing_values(self, extractor: Extractor): + """ Add values not specified in the header from another extractor + (e.g. PyExtractor or RExtractor) + """ + if self.ins is None: + self.ins = extractor.ins + if self.outs is None: + self.outs = extractor.outs + if self.params is None: + self.params = extractor.params + # We store a reference to extractor.extract_cell_params because + # self.extract_cell_params is called after self.add_missing_values + # in component_containerizer.handlers.ExtractorHandler.post() + self._external_extract_cell_params = extractor.extract_cell_params + if self.confs is None: + self.confs = extractor.confs + if self.dependencies is None: + self.dependencies = extractor.dependencies + + @staticmethod + def _parse_inputs_outputs_param_items( + item: Union[str, dict], + item_type: Literal['inputs', 'outputs', 'params'], + ) -> dict: + """ Parse inputs, outputs, or params items from the header + + They can have either format + - ElementVarName: 'my_name' + - ElementVarNameType {'my_name': 'my_type'} + - IOElementVarDict {'my_name': {'type': 'my_type'}} + or ParamElementVarDict {'my_name': {'type': 'my_type', + 'default_value': 'my_value'}} + + Returns + - if item_type is 'inputs' or 'outputs': + {'name': 'my_name', 'type': 'my_type'} + - if item_type is 'params': + {'name': 'my_name', 'type': 'my_type', 'value': 'my_value'} + """ + var_dict = {} + + # ElementVarName + if isinstance(item, str): + var_dict = { + 'name': item, + 'type': None, + 'value': None, + } + elif isinstance(item, dict): + if len(item.keys()) != 1: + # this should have been caught by the schema validation + raise ValueError(f"Unexpected item in {item_type}: {item}") + var_name = list(item.keys())[0] + var_props = item[var_name] + # ElementVarNameType + if isinstance(var_props, str): + var_dict = { + 'name': var_name, + 'type': var_props, + 'value': None, + } + # IOElementVarDict or ParamElementVarDict + elif isinstance(var_props, dict): + var_dict = { + 'name': var_name, + 'type': var_props.get('type'), + 'value': var_props.get('default_value'), + } + + # Convert types + types_conversion = { + 'Integer': 'int', + 'Float': 'float', + 'String': 'str', + 'List': 'list', + None: None, + } + var_dict['type'] = types_conversion[var_dict['type']] + + # 'value' should only be kept for params + if item_type not in ['params']: + del var_dict['value'] + + return var_dict + + def _infer_cell_inputs_outputs_params( + self, + header: Union[dict, None], + item_type: Literal['inputs', 'outputs', 'params'], + ) -> Union[dict, None]: + if header is None: + return None + items = header['NaaVRE']['cell'].get(item_type) + if items is None: + return None + items = [self._parse_inputs_outputs_param_items(it, item_type) + for it in items] + return {it['name']: it for it in items} + + def infer_cell_inputs(self): + return self._infer_cell_inputs_outputs_params( + self.cell_header, + 'inputs', + ) + + def infer_cell_outputs(self): + return self._infer_cell_inputs_outputs_params( + self.cell_header, + 'outputs', + ) + + def extract_cell_params(self, source): + if self._external_extract_cell_params is not None: + return self._external_extract_cell_params(source) + return self._infer_cell_inputs_outputs_params( + self._extract_header(source), + 'params', + ) + + def extract_cell_conf_ref(self): + if self.cell_header is None: + return None + items = self.cell_header['NaaVRE']['cell'].get('confs') + if items is None: + return None + return {k: v['assignation'] for it in items for k, v in it.items()} + + def infer_cell_dependencies(self, confs): + if self.cell_header is None: + return None + items = self.cell_header['NaaVRE']['cell'].get('dependencies') + if items is None: + return None + return [ + { + 'name': it.get('name'), + 'asname': it.get('asname', None), + 'module': it.get('module', ''), + } + for it in items] diff --git a/vreapis/services/extractor/pyextractor.py b/vreapis/services/extractor/pyextractor.py new file mode 100644 index 00000000..e374c4a3 --- /dev/null +++ b/vreapis/services/extractor/pyextractor.py @@ -0,0 +1,279 @@ +import ast +import logging +import re +from functools import lru_cache + +from pyflakes import reporter as pyflakes_reporter, api as pyflakes_api +from pytype import config as pytype_config +from pytype.tools.annotate_ast import annotate_ast + +from .extractor import Extractor + + +class PyExtractor(Extractor): + sources: list + imports: dict + configurations: dict + global_params: dict + undefined: dict + + def __init__(self, notebook, cell_source): + # If cell_type is code and not starting with '!' + self.sources = [nbcell.source for nbcell in notebook.cells if + nbcell.cell_type == 'code' and len(nbcell.source) > 0 and nbcell.source[0] != '!'] + self.notebook_names = self.__extract_cell_names( + '\n'.join(self.sources), + infer_types=True, + ) + self.imports = self.__extract_imports(self.sources) + self.configurations = self.__extract_configurations(self.sources) + self.global_params = self.__extract_params(self.sources) + self.undefined = dict() + for source in self.sources: + self.undefined.update(self.__extract_cell_undefined(source)) + + super().__init__(notebook, cell_source) + + def __extract_imports(self, sources): + imports = {} + for s in sources: + tree = ast.parse(s) + for node in ast.walk(tree): + if isinstance(node, (ast.Import, ast.ImportFrom,)): + for n in node.names: + key = n.asname if n.asname else n.name + if key not in imports: + imports[key] = { + 'name': n.name, + 'asname': n.asname or None, + 'module': node.module if isinstance(node, ast.ImportFrom) else "" + } + return imports + + def __extract_configurations(self, sources): + configurations = {} + for s in sources: + lines = s.splitlines() + tree = ast.parse(s) + for node in ast.walk(tree): + if isinstance(node, ast.Assign): + target = node.targets[0] + if hasattr(target, 'id'): + name = node.targets[0].id + prefix = name.split('_')[0] + if prefix == 'conf' and name not in configurations: + conf_line = '' + for line in lines[node.lineno - 1:node.end_lineno]: + conf_line += line.strip() + configurations[name] = conf_line + return self.__resolve_configurations(configurations) + + def __extract_params(self, sources): + params = dict() + for s in sources: + lines = s.splitlines() + tree = ast.parse(s) + for node in ast.walk(tree): + if isinstance(node, ast.Assign) and hasattr(node.targets[0], 'id'): + name = node.targets[0].id + prefix = name.split('_')[0] + if prefix == 'param': + param_line = '' + for line in lines[node.lineno - 1:node.end_lineno]: + param_line += line.strip() + param_value = ast.unparse(node.value) + try: + # remove quotes around strings + param_value = str(ast.literal_eval(param_value)) + except ValueError: + # when param_value can't safely be parsed, + pass + params[name] = { + 'name': name, + 'type': self.notebook_names[name]['type'], + 'value': param_value, + } + return params + + def infer_cell_outputs(self): + cell_names = self.__extract_cell_names(self.cell_source) + return { + name: properties + for name, properties in cell_names.items() + if name not in self.__extract_cell_undefined(self.cell_source) + and name not in self.imports + and name in self.undefined + and name not in self.configurations + and name not in self.global_params + } + + def infer_cell_inputs(self): + cell_undefined = self.__extract_cell_undefined(self.cell_source) + return { + und: properties + for und, properties in cell_undefined.items() + if und not in self.imports + and und not in self.configurations + and und not in self.global_params + } + + def infer_cell_dependencies(self, confs): + dependencies = [] + names = self.__extract_cell_names(self.cell_source) + + for ck in confs: + names.update(self.__extract_cell_names(confs[ck])) + + for name in names: + if name in self.imports: + dependencies.append(self.imports.get(name)) + + return dependencies + + def infer_cell_conf_dependencies(self, confs): + dependencies = [] + for ck in confs: + for name in self.__extract_cell_names(confs[ck]): + if name in self.imports: + dependencies.append(self.imports.get(name)) + + return dependencies + + @staticmethod + @lru_cache + def __get_annotated_ast(cell_source): + return annotate_ast.annotate_source( + cell_source, ast, pytype_config.Options.create()) + + def __convert_type_annotation(self, type_annotation): + """ Convert type annotation to the ones supported for cell interfaces + + :param type_annotation: type annotation obtained by e.g. pytype + :return: converted type: 'int', 'float', 'str', 'list', or None + """ + if type_annotation is None: + return None + + patterns = { + 'int': [ + re.compile(r'^int$'), + ], + 'float': [ + re.compile(r'^float$'), + ], + 'str': [ + re.compile(r'^str$'), + ], + 'list': [ + re.compile(r'^List\['), + ], + None: [ + re.compile(r'^Any$'), + re.compile(r'^Dict'), + re.compile(r'^Callable'), + ] + } + for type_name, regs in patterns.items(): + for reg in regs: + if reg.match(type_annotation): + return type_name + + logging.getLogger(__name__).debug(f'Unmatched type: {type_annotation}') + return None + + def __extract_cell_names(self, cell_source, infer_types=False): + names = dict() + if infer_types: + tree = self.__get_annotated_ast(cell_source) + else: + tree = ast.parse(cell_source) + for module in ast.walk(tree): + if isinstance(module, (ast.Name,)): + var_name = module.id + if infer_types: + try: + var_type = self.__convert_type_annotation(module.resolved_annotation) + except AttributeError: + print('__extract_cell_names failed', var_name) + var_type = None + else: + var_type = self.notebook_names[var_name]['type'] + names[module.id] = { + 'name': var_name, + 'type': var_type, + } + return names + + def __extract_cell_undefined(self, cell_source): + + flakes_stdout = StreamList() + flakes_stderr = StreamList() + rep = pyflakes_reporter.Reporter( + flakes_stdout.reset(), + flakes_stderr.reset()) + pyflakes_api.check(cell_source, filename="temp", reporter=rep) + + if rep._stderr(): + raise SyntaxError("Flakes reported the following error:" + "\n{}".format('\t' + '\t'.join(rep._stderr()))) + p = r"'(.+?)'" + + out = rep._stdout() + undef_vars = dict() + + for line in filter(lambda a: a != '\n' and 'undefined name' in a, out): + var_search = re.search(p, line) + var_name = var_search.group(1) + undef_vars[var_name] = { + 'name': var_name, + 'type': self.notebook_names[var_name]['type'], + } + return undef_vars + + def extract_cell_params(self, cell_source): + params = {} + cell_unds = self.__extract_cell_undefined(cell_source) + param_unds = [und for und in cell_unds if und in self.global_params] + for u in param_unds: + if u not in params: + params[u] = self.global_params[u] + return params + + def extract_cell_conf_ref(self): + confs = {} + cell_unds = self.__extract_cell_undefined(self.cell_source) + conf_unds = [und for und in cell_unds if und in self.configurations] + for u in conf_unds: + if u not in confs: + confs[u] = self.configurations[u] + return confs + + def __resolve_configurations(self, configurations): + resolved_configurations = {} + for k, assignment in configurations.items(): + while 'conf_' in assignment.split('=')[1]: + for conf_name, replacing_assignment in configurations.items(): + if conf_name in assignment.split('=')[1]: + assignment = assignment.replace( + conf_name, + replacing_assignment.split('=')[1], + ) + resolved_configurations[k] = assignment + configurations.update(resolved_configurations) + return configurations + + +class StreamList: + + def __init__(self): + self.out = list() + + def write(self, text): + self.out.append(text) + + def reset(self): + self.out = list() + return self + + def __call__(self): + return self.out diff --git a/vreapis/services/extractor/rextractor.py b/vreapis/services/extractor/rextractor.py new file mode 100644 index 00000000..a94f44ba --- /dev/null +++ b/vreapis/services/extractor/rextractor.py @@ -0,0 +1,431 @@ +import os +import re +import tempfile + +import rpy2.rinterface as rinterface +import rpy2.robjects as robjects +import rpy2.robjects.packages as rpackages +from rpy2.robjects.packages import importr + +from .extractor import Extractor + +# Create an R environment +r_env = robjects.globalenv + +# install R packages +packnames = ('rlang', 'lobstr', 'purrr',) +for p in packnames: + if not rpackages.isinstalled(p): + robjects.r(f'install.packages("{p}")') + +# This R code is used to obtain all assignment variables (source https://adv-r.hadley.nz/expressions.html) +r_env["result"] = robjects.r(""" +library(rlang) +library(lobstr) +library(purrr) + +expr_type <- function(x) { + if (rlang::is_syntactic_literal(x)) { + "constant" + } else if (is.symbol(x)) { + "symbol" + } else if (is.call(x)) { + "call" + } else if (is.pairlist(x)) { + "pairlist" + } else { + typeof(x) + } +} + +switch_expr <- function(x, ...) { + switch(expr_type(x), + ..., + stop("Don't know how to handle type ", typeof(x), call. = FALSE) + ) +} + +recurse_call <- function(x) { + switch_expr(x, + # Base cases + symbol = , + constant = , + + # Recursive cases + call = , + pairlist = + ) +} + +logical_abbr_rec <- function(x) { + switch_expr(x, + constant = FALSE, + symbol = as_string(x) %in% c("F", "T") + ) +} + +logical_abbr <- function(x) { + logical_abbr_rec(enexpr(x)) +} + +find_assign_rec <- function(x) { + switch_expr(x, + constant = , + symbol = character() + ) +} +find_assign <- function(x) unique(find_assign_rec(enexpr(x))) + +flat_map_chr <- function(.x, .f, ...) { + purrr::flatten_chr(purrr::map(.x, .f, ...)) +} + +find_assign_rec <- function(x) { + switch_expr(x, + # Base cases + constant = , + symbol = character(), + + # Recursive cases + pairlist = flat_map_chr(as.list(x), find_assign_rec), + call = { + if (is_call(x, "<-") || is_call(x, "=")) { # TODO: also added is_call(x, "=") here + if (typeof(x[[2]]) == "symbol"){ # TODO: added the type check here + as_string(x[[2]]) + } + } else { + flat_map_chr(as.list(x), find_assign_rec) + } + } + ) +} +""") + +# Load the base R package for parsing and evaluation +base = importr('base') + + +# TODO: create an interface such that it can be easily extended to other kernels + +class RExtractor(Extractor): + sources: list + imports: dict + configurations: dict + global_params: dict + undefined: dict + + def __init__(self, notebook, cell_source): + self.sources = [nbcell.source for nbcell in notebook.cells if + nbcell.cell_type == 'code' and len(nbcell.source) > 0] + + self.imports = self.__extract_imports(self.sources) + self.configurations = self.__extract_configurations(self.sources) + self.global_params = self.__extract_params(self.sources) + self.undefined = dict() + for source in self.sources: + self.undefined.update(self.__extract_cell_undefined(source)) + + super().__init__(notebook, cell_source) + + def __extract_imports(self, sources): + imports = {} + for s in sources: + packages = [] + + ''' Approach 1: Simple regex. + this matches the following cases: require(pkg), library(pkg), library("pkg"), library(package=pkg), library(package="pkg") + ''' + # packages = re.findall(r'(?:library|require)\((?:package=)?(?:")?(\w+)(?:")?\)', s) + + ''' Approach 2: Static analysis using 'renv' package. + this approach is more safe as it covers more cases and checks comments + ''' + with tempfile.NamedTemporaryFile(delete=False, suffix='.R') as tmp_file: + tmp_file.write(s.encode()) + tmp_file.flush() + renv = rpackages.importr('renv') + function_list = renv.dependencies(tmp_file.name) + + # transpose renv dependencies to readable dependencies + transposed_list = list(map(list, zip(*function_list))) + packages = [row[1] for row in transposed_list] + tmp_file.close() + os.remove(tmp_file.name) + + # format the packages + for package in packages: + imports[package] = { + # asname and module are specific to Python packages. So you can probably leave them out here + 'name': package, + 'asname': '', + 'module': '' + } + return imports + + def __extract_configurations(self, sources): + configurations = {} + for s in sources: + parsed_expr = base.parse(text=s, keep_source=True) + parsed_expr_py = robjects.conversion.rpy2py(parsed_expr) + lines = s.splitlines() + + # loop through all assignment variables + assignment_variables = self.assignment_variables(s) + for variable in assignment_variables: + + # the prefix should be 'conf' + if not (variable.split("_")[0] == "conf"): + continue + + # find the line of the assignment. (TODO) this approach assumes that there is only one expression in one line. + # this might not work when we have something like: a <- 3; b = 7 + for line in lines: + matches = re.findall(r'{}\s*(=|<-)'.format(variable), line) + + if len(matches) > 0 and variable not in configurations: + configurations[variable] = line + break + return configurations + + def __extract_params(self, sources): # check source https://adv-r.hadley.nz/expressions.html) + params = {} + for s in sources: + lines = s.splitlines() + + '''Approach 1: Naive way + Find all variable assignments with a prefix of "param"''' + # pattern = r"param_[a-zA-Z0-9_]{0,}" + # matches = re.findall(pattern, s) + # Extract the variable names from the matches + # for match in matches: + # params.add(match) + + '''Approach 2: Look at the AST''' + assignment_variables = self.assignment_variables(s) + for variable in assignment_variables: + + # the prefix should be 'param' + if not (variable.split("_")[0] == "param"): + continue + + # find the line of the assignment. (TODO) this approach assumes that there is only one expression in one line. + # this might not work when we have something like: a <- 3; b = 7 + param_value = '' + for line in lines: + m = re.match(r'{}\s*(?:=|<-)(.*?)\s*(#.*?)?$'.format(variable), line) + if m: + param_value = m.group(1).strip(" \"' ") + + params[variable] = { + 'name': variable, + 'type': None, + 'value': param_value, + } + return params + + def infer_cell_outputs(self): + cell_names = self.__extract_cell_names(self.cell_source) + return { + name: properties + for name, properties in cell_names.items() + if name not in self.__extract_cell_undefined(self.cell_source) + and name not in self.imports + and name in self.undefined + and name not in self.configurations + and name not in self.global_params + } + + def infer_cell_inputs(self): + cell_undefined = self.__extract_cell_undefined(self.cell_source) + return { + und: properties + for und, properties in cell_undefined.items() + if und not in self.imports + and und not in self.configurations + and und not in self.global_params + } + + def infer_cell_dependencies(self, confs): + # TODO: check this code, you have removed logic. + # we probably like to only use dependencies that are necessary to execute the cell + # however this is challenging in R as functions are non-scoped + dependencies = [] + for name in self.imports: + dependencies.append(self.imports.get(name)) + return dependencies + + def infer_cell_conf_dependencies(self, confs): + dependencies = [] + for ck in confs: + for name in self.__extract_cell_names(confs[ck]): + if name in self.imports: + dependencies.append(self.imports.get(name)) + + return dependencies + + def get_function_parameters(self, cell_source): + result = [] + + # Approach 1: Naive Regex + functions = re.findall(r'function\s*\((.*?)\)', cell_source) + for params in functions: + result.extend(re.findall(r'\b\w+\b', params)) + + # Approach 2: AST based + # TODO + + return list(set(result)) + + def get_iterator_variables(self, cell_source): + result = [] + + # Approach 1: Naive Regex. This means that iterator variables are in the following format: + # for ( .....) + result = re.findall(r'for\s*\(\s*([a-zA-Z0-9.]+)\s+in', cell_source) + + # Approach 2: Parse AST. Much cleaner option as iterator variables can appear in differen syntaxes. + # TODO + + return result + + def __extract_cell_names(self, cell_source): + parsed_r = robjects.r['parse'](text=cell_source) + vars_r = robjects.r['all.vars'](parsed_r) + + # Challenge 1: Function Parameters + function_parameters = self.get_function_parameters(cell_source) + vars_r = list(filter(lambda x: x not in function_parameters, vars_r)) + + # Challenge 2: Built-in Constants + built_in_cons = ["T", "F", "pi", "is.numeric", "mu", "round"] + vars_r = list(filter(lambda x: x not in built_in_cons, vars_r)) + + # Challenge 3: Iterator Variables + iterator_variables = self.get_iterator_variables(cell_source) + vars_r = list(filter(lambda x: x not in iterator_variables, vars_r)) + + # Challenge 4: Apply built-in functions + # MANUALLY SOLVED + + # Challenge 5: Libraries + vars_r = list(filter(lambda x: x not in self.imports, vars_r)) + + # Challenge 6: Variable-based data access + # MANUALLY SOLVED + + vars_r = { + name: { + 'name': name, + 'type': None, + } + for name in vars_r + } + + return vars_r + + # This is a very inefficient approach to obtain all assignment variables (Solution 1) + def recursive_variables(self, my_expr, result): + if isinstance(my_expr, rinterface.LangSexpVector): + # check if there are enough data values. for an assignment there must be three namely VARIABLE SYMBOL VALUE. e.g. a = 3 + if len(my_expr) >= 3: + + # check for matches + c = str(my_expr[0]) + variable = my_expr[1] + + # Check if assignment. + if (c == "<-" or c == "="): + if isinstance(my_expr[1], rinterface.SexpSymbol): + result.add(str(variable)) + try: + for expr in my_expr: + result = self.recursive_variables(expr, result) + except Exception as e: + pass + return result + + def assignment_variables(self, text): + result = [] + + # Solution 1 (Native-Python): Write our own recursive function that in Python that parses the Abstract Syntax Tree of the R cell + # This is a very inefficient solution + # parsed_expr = base.parse(text=text, keep_source=True) + # parsed_expr_py = robjects.conversion.rpy2py(parsed_expr) + # result = list(self.recursive_variables(parsed_expr_py, set())) + + # Solution 2 (Native-R): Use built-in recursive cases of R (source https://adv-r.hadley.nz/expressions.html). This method is significantly faster. + output_r = robjects.r("""find_assign({ + %s + })""" % text) + result = re.findall(r'"([^"]*)"', str(output_r)) + + # Return the result + return result + + def __extract_cell_undefined(self, cell_source): + # Approach 1: get all vars and substract the ones with the approach as in + cell_names = self.__extract_cell_names(cell_source) + assignment_variables = self.assignment_variables(cell_source) + undef_vars = set(cell_names).difference(set(assignment_variables)) + + # Approach 2: (TODO) dynamic analysis approach. this is complex for R as functions + # as they are not scoped (which is the case in python). As such, we might have to include + # all the libraries to make sure that those functions work + + undef_vars = { + name: { + 'name': name, + 'type': None, + } + for name in undef_vars + } + + return undef_vars + + def extract_cell_params(self, cell_source): + params = {} + cell_unds = self.__extract_cell_undefined(cell_source) + param_unds = [und for und in cell_unds if und in self.global_params] + for u in param_unds: + if u not in params: + params[u] = self.global_params[u] + return params + + def extract_cell_conf_ref(self): + confs = {} + cell_unds = self.__extract_cell_undefined(self.cell_source) + conf_unds = [und for und in cell_unds if und in self.configurations] + for u in conf_unds: + if u not in confs: + confs[u] = self.configurations[u] + return confs + + def __resolve_configurations(self, configurations): + resolved_configurations = {} + for k, assignment in configurations.items(): + while 'conf_' in assignment.split('=')[1]: + for conf_name, replacing_assignment in configurations.items(): + if conf_name in assignment.split('=')[1]: + assignment = assignment.replace( + conf_name, + replacing_assignment.split('=')[1], + ) + resolved_configurations[k] = assignment + configurations.update(resolved_configurations) + return configurations + + +class StreamList: + + def __init__(self): + self.out = list() + + def write(self, text): + self.out.append(text) + + def reset(self): + self.out = list() + return self + + def __call__(self): + return self.out diff --git a/vreapis/utils/cors.py b/vreapis/utils/cors.py new file mode 100644 index 00000000..d926cb0f --- /dev/null +++ b/vreapis/utils/cors.py @@ -0,0 +1,7 @@ +import django.http +import rest_framework.request + + +def get_CORS_headers(request: django.http.HttpRequest | rest_framework.request.Request) -> dict[str, str]: + Origin: str = request.META.get('HTTP_ORIGIN') + return {'Access-Control-Allow-Origin': Origin} From 2392d86652228ca394f3910d037d9d1c5f4e94bd Mon Sep 17 00:00:00 2001 From: AndyBRoswell Date: Wed, 5 Jun 2024 00:41:22 +0200 Subject: [PATCH 014/149] porting ExtractorHandler removed JupyterLab dependencies resolved Python dependencies resolving R dependencies --- tilt/vreapis/Dockerfile | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tilt/vreapis/Dockerfile b/tilt/vreapis/Dockerfile index 3d9a92c8..4c2ffbb4 100644 --- a/tilt/vreapis/Dockerfile +++ b/tilt/vreapis/Dockerfile @@ -3,7 +3,7 @@ FROM python:3.12.2-slim WORKDIR /app RUN apt update && \ - apt install -y libpq-dev gcc gdal-bin curl net-tools iputils-ping + apt install -y libpq-dev gcc gdal-bin curl net-tools iputils-ping r-base # Install rysnc for Tilt syncback RUN apt install -y rsync @@ -14,6 +14,9 @@ RUN /opt/venv/bin/pip install pip --upgrade COPY ./vreapis/requirements.txt /app RUN /opt/venv/bin/pip install -r requirements.txt +# Set another directory for installation of R packages to avoid permission problem +RUN echo 'R_LIBS_USER="~/R/library"' >> ~/.Renviron + COPY ./vreapis /app RUN chmod +x entrypoint.dev.sh From 297747bfc637edf77ed809feed3a0e19c4a0122a Mon Sep 17 00:00:00 2001 From: AndyBRoswell Date: Wed, 5 Jun 2024 18:36:15 +0200 Subject: [PATCH 015/149] test_extractor.py migration preliminarily completed --- tilt/vreapis/Dockerfile | 2 +- vreapis/containerizer/tests.py | 12 +- vreapis/requirements.txt | 1 + vreapis/services/extractor/rextractor.py | 27 +- .../add1-dev-user-name-at-domain-com.json | 92 ++ .../add1-r-dev-user-name-at-domain-com.json | 74 + ...i-pattern-dev-user-name-at-domain-com.json | 104 ++ ...e-climwin-dev-user-name-at-domain-com.json | 133 ++ ...eate-file-dev-user-name-at-domain-com.json | 93 ++ ...te-file-r-dev-user-name-at-domain-com.json | 70 + ...put-lists-dev-user-name-at-domain-com.json | 103 ++ ...t-lists-r-dev-user-name-at-domain-com.json | 83 + ...-packages-dev-user-name-at-domain-com.json | 130 ++ ...o-odim-h5-dev-user-name-at-domain-com.json | 93 ++ ...ile-lines-dev-user-name-at-domain-com.json | 87 ++ ...e-lines-r-dev-user-name-at-domain-com.json | 87 ++ ...-int-list-dev-user-name-at-domain-com.json | 87 ++ ...nt-list-r-dev-user-name-at-domain-com.json | 69 + ...loop-list-dev-user-name-at-domain-com.json | 69 + ...op-list-r-dev-user-name-at-domain-com.json | 69 + ...print-msg-dev-user-name-at-domain-com.json | 74 + ...ile-lines-dev-user-name-at-domain-com.json | 110 ++ ...e-lines-r-dev-user-name-at-domain-com.json | 89 ++ .../vol2bird-dev-user-name-at-domain-com.json | 93 ++ .../resources/notebooks/R-notebook.ipynb | 74 + .../notebooks/TraitsComputation_NaaVRE.json | 155 ++ .../dependency_with_submodule.notebook.json | 91 ++ .../tests/resources/notebooks/laserfarm.json | 178 +++ .../tests/resources/notebooks/r-notebook.json | 63 + vreapis/tests/resources/notebooks/test_!.json | 158 ++ .../notebooks/test_R_param_in_cell.json | 36 + .../notebooks/test_R_param_outside_cell.json | 46 + .../notebooks/test_cell_header_R.json | 87 ++ .../notebooks/test_conf_nesting.json | 53 + .../resources/notebooks/test_new_line.json | 48 + .../resources/notebooks/test_notebook.ipynb | 229 +++ .../test_param_in_cell_notebook.json | 159 ++ .../test_param_outside_cell_notebook.json | 159 ++ .../notebooks/test_param_values_Python.json | 56 + .../notebooks/test_param_values_R.json | 59 + vreapis/tests/resources/search/keyword.json | 3 + .../workflows/NaaVRE/468f918f414f96df.json | 160 ++ .../workflows/NaaVRE/test_R_workflow.json | 844 ++++++++++ .../workflows/NaaVRE/test_py_workflow.json | 1378 +++++++++++++++++ .../workflows/argo/argo_workflow.json | 37 + 45 files changed, 5916 insertions(+), 8 deletions(-) create mode 100644 vreapis/tests/resources/cells/add1-dev-user-name-at-domain-com.json create mode 100644 vreapis/tests/resources/cells/add1-r-dev-user-name-at-domain-com.json create mode 100644 vreapis/tests/resources/cells/anti-pattern-dev-user-name-at-domain-com.json create mode 100644 vreapis/tests/resources/cells/consume-climwin-dev-user-name-at-domain-com.json create mode 100644 vreapis/tests/resources/cells/create-file-dev-user-name-at-domain-com.json create mode 100644 vreapis/tests/resources/cells/create-file-r-dev-user-name-at-domain-com.json create mode 100644 vreapis/tests/resources/cells/input-lists-dev-user-name-at-domain-com.json create mode 100644 vreapis/tests/resources/cells/input-lists-r-dev-user-name-at-domain-com.json create mode 100644 vreapis/tests/resources/cells/install-and-load-the-climwinb-packages-dev-user-name-at-domain-com.json create mode 100644 vreapis/tests/resources/cells/knmi-vol-h5-to-odim-h5-dev-user-name-at-domain-com.json create mode 100644 vreapis/tests/resources/cells/loop-file-lines-dev-user-name-at-domain-com.json create mode 100644 vreapis/tests/resources/cells/loop-file-lines-r-dev-user-name-at-domain-com.json create mode 100644 vreapis/tests/resources/cells/loop-int-list-dev-user-name-at-domain-com.json create mode 100644 vreapis/tests/resources/cells/loop-int-list-r-dev-user-name-at-domain-com.json create mode 100644 vreapis/tests/resources/cells/loop-list-dev-user-name-at-domain-com.json create mode 100644 vreapis/tests/resources/cells/loop-list-r-dev-user-name-at-domain-com.json create mode 100644 vreapis/tests/resources/cells/print-msg-dev-user-name-at-domain-com.json create mode 100644 vreapis/tests/resources/cells/read-file-lines-dev-user-name-at-domain-com.json create mode 100644 vreapis/tests/resources/cells/read-file-lines-r-dev-user-name-at-domain-com.json create mode 100644 vreapis/tests/resources/cells/vol2bird-dev-user-name-at-domain-com.json create mode 100644 vreapis/tests/resources/notebooks/R-notebook.ipynb create mode 100644 vreapis/tests/resources/notebooks/TraitsComputation_NaaVRE.json create mode 100644 vreapis/tests/resources/notebooks/dependency_with_submodule.notebook.json create mode 100644 vreapis/tests/resources/notebooks/laserfarm.json create mode 100644 vreapis/tests/resources/notebooks/r-notebook.json create mode 100644 vreapis/tests/resources/notebooks/test_!.json create mode 100644 vreapis/tests/resources/notebooks/test_R_param_in_cell.json create mode 100644 vreapis/tests/resources/notebooks/test_R_param_outside_cell.json create mode 100644 vreapis/tests/resources/notebooks/test_cell_header_R.json create mode 100644 vreapis/tests/resources/notebooks/test_conf_nesting.json create mode 100644 vreapis/tests/resources/notebooks/test_new_line.json create mode 100644 vreapis/tests/resources/notebooks/test_notebook.ipynb create mode 100644 vreapis/tests/resources/notebooks/test_param_in_cell_notebook.json create mode 100644 vreapis/tests/resources/notebooks/test_param_outside_cell_notebook.json create mode 100644 vreapis/tests/resources/notebooks/test_param_values_Python.json create mode 100644 vreapis/tests/resources/notebooks/test_param_values_R.json create mode 100644 vreapis/tests/resources/search/keyword.json create mode 100644 vreapis/tests/resources/workflows/NaaVRE/468f918f414f96df.json create mode 100644 vreapis/tests/resources/workflows/NaaVRE/test_R_workflow.json create mode 100644 vreapis/tests/resources/workflows/NaaVRE/test_py_workflow.json create mode 100644 vreapis/tests/resources/workflows/argo/argo_workflow.json diff --git a/tilt/vreapis/Dockerfile b/tilt/vreapis/Dockerfile index 4c2ffbb4..cee789f9 100644 --- a/tilt/vreapis/Dockerfile +++ b/tilt/vreapis/Dockerfile @@ -15,7 +15,7 @@ COPY ./vreapis/requirements.txt /app RUN /opt/venv/bin/pip install -r requirements.txt # Set another directory for installation of R packages to avoid permission problem -RUN echo 'R_LIBS_USER="~/R/library"' >> ~/.Renviron +# RUN echo 'R_LIBS_USER="~/R/library"' >> ~/.Renviron COPY ./vreapis /app RUN chmod +x entrypoint.dev.sh diff --git a/vreapis/containerizer/tests.py b/vreapis/containerizer/tests.py index 7db068e6..dea9eded 100644 --- a/vreapis/containerizer/tests.py +++ b/vreapis/containerizer/tests.py @@ -32,7 +32,7 @@ def test_get_base_images(self): dummy_user = User.objects.create_user(dummy_username, password=dummy_password) client.login(username=dummy_username, password=dummy_password) - response = client.get('/api/containerizer/baseimagetags/') + response = client.get('/api/containerizer/baseimagetags/', headers={'Authorization': f'Token {settings.NAAVRE_API_TOKEN}'}) self.assertEqual(response.status_code, 200) images = response.json() self.assertIsInstance(images, dict) @@ -49,11 +49,13 @@ class ExtractorTestCase(TestCase): 'param_string_with_comment': 'param_string value', } - def __init__(self): + def setUp(self): + super().__init__() + self.base_path = '' if os.path.exists('resources'): - base_path = 'resources' - elif os.path.exists('jupyterlab_vre/tests/resources/'): - base_path = 'jupyterlab_vre/tests/resources/' + self.base_path = 'resources' + elif os.path.exists('tests/resources/'): + self.base_path = 'tests/resources/' def create_cell(self, payload_path=None): with open(payload_path, 'r') as file: diff --git a/vreapis/requirements.txt b/vreapis/requirements.txt index 115da79a..7c790334 100644 --- a/vreapis/requirements.txt +++ b/vreapis/requirements.txt @@ -19,3 +19,4 @@ colorhash rpy2==3.5.11 pyflakes pytype +tinydb diff --git a/vreapis/services/extractor/rextractor.py b/vreapis/services/extractor/rextractor.py index a94f44ba..9a4959c9 100644 --- a/vreapis/services/extractor/rextractor.py +++ b/vreapis/services/extractor/rextractor.py @@ -6,6 +6,10 @@ import rpy2.robjects as robjects import rpy2.robjects.packages as rpackages from rpy2.robjects.packages import importr +from rpy2.rinterface_lib.callbacks import logger as rpy2_logger +import logging + +rpy2_logger.setLevel(logging.WARNING) from .extractor import Extractor @@ -13,10 +17,29 @@ r_env = robjects.globalenv # install R packages -packnames = ('rlang', 'lobstr', 'purrr',) +robjects.r(''' +install_package_with_retry <- function(package_name, max_attempts = 5) { + for(i in 1:max_attempts) { + print(paste("Attempt", i, "to install", package_name)) + tryCatch({ + install.packages(package_name, quiet = TRUE) + print(paste(package_name, "installed successfully.")) + return(TRUE) + }, warning = function(w) { + print(paste("Warning while installing", package_name, ":", w)) + Sys.sleep(2) + }, error = function(e) { + print(paste("Failed to install", package_name, ":", e)) + Sys.sleep(2) + }) + } + return(FALSE) +} +''') +packnames = ('rlang', 'lobstr', 'purrr','renv',) for p in packnames: if not rpackages.isinstalled(p): - robjects.r(f'install.packages("{p}")') + robjects.r(f'install_package_with_retry("{p}")') # This R code is used to obtain all assignment variables (source https://adv-r.hadley.nz/expressions.html) r_env["result"] = robjects.r(""" diff --git a/vreapis/tests/resources/cells/add1-dev-user-name-at-domain-com.json b/vreapis/tests/resources/cells/add1-dev-user-name-at-domain-com.json new file mode 100644 index 00000000..adf237b8 --- /dev/null +++ b/vreapis/tests/resources/cells/add1-dev-user-name-at-domain-com.json @@ -0,0 +1,92 @@ +{ + "all_inputs": [ + "count" + ], + "base_image": {"build":"ghcr.io/qcdis/naavre/naavre-cell-build-python:latest","runtime":"ghcr.io/qcdis/naavre/naavre-cell-runtime-python:latest"}, + "chart_obj": { + "hovered": {}, + "links": {}, + "nodes": { + "49d5dda": { + "id": "49d5dda", + "ports": { + "a": { + "id": "a", + "properties": { + "color": "#77862d" + }, + "type": "right" + }, + "count": { + "id": "count", + "properties": { + "color": "#783d3a" + }, + "type": "left" + } + }, + "position": { + "x": 35, + "y": 15 + }, + "properties": { + "inputs": [ + "count" + ], + "og_node_id": "49d5dda", + "outputs": [ + "a" + ], + "params": [], + "title": "Add1-dev-user-name-at-domain-com", + "vars": [ + { + "color": "#783d3a", + "direction": "input", + "name": "count", + "type": "datatype" + }, + { + "color": "#77862d", + "direction": "output", + "name": "a", + "type": "datatype" + } + ] + }, + "type": "input-output" + } + }, + "offset": { + "x": 0, + "y": 0 + }, + "scale": 1, + "selected": {} + }, + "confs": {}, + "container_source": "", + "dependencies": [], + "inputs": [ + "count" + ], + "kernel": "ipython", + "node_id": "49d5dda", + "original_source": "\na = count + 1", + "outputs": [ + "a" + ], + "params": [], + "task_name": "add1-dev-user-name-at-domain-com", + "title": "Add1-dev-user-name-at-domain-com", + "types": { + "a": "int", + "count": "int" + }, + "example_inputs" : [ + "--id", + "0", + "--count", + "1" + ] +} diff --git a/vreapis/tests/resources/cells/add1-r-dev-user-name-at-domain-com.json b/vreapis/tests/resources/cells/add1-r-dev-user-name-at-domain-com.json new file mode 100644 index 00000000..42f74cac --- /dev/null +++ b/vreapis/tests/resources/cells/add1-r-dev-user-name-at-domain-com.json @@ -0,0 +1,74 @@ +{ + "all_inputs": [ + "count" + ], + "base_image": {"build":"ghcr.io/qcdis/naavre/naavre-cell-build-r:latest","runtime":"ghcr.io/qcdis/naavre/naavre-cell-runtime-r:latest"}, + "chart_obj": { + "hovered": {}, + "links": {}, + "nodes": { + "d9acfa2": { + "id": "d9acfa2", + "ports": { + "count": { + "id": "count", + "properties": { + "color": "#783d3a" + }, + "type": "left" + } + }, + "position": { + "x": 35, + "y": 15 + }, + "properties": { + "inputs": [ + "count" + ], + "og_node_id": "d9acfa2", + "outputs": [], + "params": [], + "title": "Add1 R-dev-user-name-at-domain-com", + "vars": [ + { + "color": "#783d3a", + "direction": "input", + "name": "count", + "type": "datatype" + } + ] + }, + "type": "input-output" + } + }, + "offset": { + "x": 0, + "y": 0 + }, + "scale": 1, + "selected": {} + }, + "confs": {}, + "container_source": "", + "dependencies": [], + "inputs": [ + "count" + ], + "kernel": "IRkernel", + "node_id": "d9acfa2", + "original_source": "\na = count + 1", + "outputs": [], + "params": [], + "task_name": "add1-r-dev-user-name-at-domain-com", + "title": "Add1 R-dev-user-name-at-domain-com", + "types": { + "count": "int" + }, + "example_inputs" : [ + "--id", + "0", + "--count", + "1" + ] +} diff --git a/vreapis/tests/resources/cells/anti-pattern-dev-user-name-at-domain-com.json b/vreapis/tests/resources/cells/anti-pattern-dev-user-name-at-domain-com.json new file mode 100644 index 00000000..7d005f14 --- /dev/null +++ b/vreapis/tests/resources/cells/anti-pattern-dev-user-name-at-domain-com.json @@ -0,0 +1,104 @@ +{ + "all_inputs": [ + "count", + "a" + ], + "base_image": {"build":"ghcr.io/qcdis/naavre/naavre-cell-build-python:latest","runtime":"ghcr.io/qcdis/naavre/naavre-cell-runtime-python:latest"}, + "chart_obj": { + "hovered": {}, + "links": {}, + "nodes": { + "44dec46": { + "id": "44dec46", + "ports": { + "a": { + "id": "a", + "properties": { + "color": "#77862d" + }, + "type": "left" + }, + "count": { + "id": "count", + "properties": { + "color": "#783d3a" + }, + "type": "left" + }, + "msg": { + "id": "msg", + "properties": { + "color": "#7bd279" + }, + "type": "right" + } + }, + "position": { + "x": 35, + "y": 15 + }, + "properties": { + "inputs": [ + "a", + "count" + ], + "og_node_id": "44dec46", + "outputs": [ + "msg" + ], + "params": [], + "title": "Anti-pattern-dev-user-name-at-domain-com", + "vars": [ + { + "color": "#77862d", + "direction": "input", + "name": "a", + "type": "datatype" + }, + { + "color": "#783d3a", + "direction": "input", + "name": "count", + "type": "datatype" + }, + { + "color": "#7bd279", + "direction": "output", + "name": "msg", + "type": "datatype" + } + ] + }, + "type": "input-output" + } + }, + "offset": { + "x": 0, + "y": 0 + }, + "scale": 1, + "selected": {} + }, + "confs": {}, + "container_source": "", + "dependencies": [], + "inputs": [ + "count", + "a" + ], + "kernel": "ipython", + "node_id": "44dec46", + "original_source": "some_list = range(count, a+1)\n\nmsg = '1'", + "outputs": [ + "msg" + ], + "params": [], + "task_name": "anti-pattern-dev-user-name-at-domain-com", + "title": "Anti-pattern-dev-user-name-at-domain-com", + "types": { + "a": "int", + "count": "int", + "msg": "str" + }, + "skip_exec": true +} diff --git a/vreapis/tests/resources/cells/consume-climwin-dev-user-name-at-domain-com.json b/vreapis/tests/resources/cells/consume-climwin-dev-user-name-at-domain-com.json new file mode 100644 index 00000000..21c7d451 --- /dev/null +++ b/vreapis/tests/resources/cells/consume-climwin-dev-user-name-at-domain-com.json @@ -0,0 +1,133 @@ +{ + "all_inputs": [ + "rolling_mean_temp_str", + "temperature_data_str" + ], + "base_image": {"build":"ghcr.io/qcdis/naavre/naavre-cell-build-lter-life-veluwe:latest","runtime":"ghcr.io/qcdis/naavre/naavre-cell-runtime-lter-life-veluwe:latest"}, + "chart_obj": { + "hovered": {}, + "links": {}, + "nodes": { + "dd373f8": { + "id": "dd373f8", + "ports": { + "rolling_mean_temp_str": { + "id": "rolling_mean_temp_str", + "properties": { + "color": "#c5a987" + }, + "type": "left" + }, + "temperature_data_str": { + "id": "temperature_data_str", + "properties": { + "color": "#aca953" + }, + "type": "left" + } + }, + "position": { + "x": 35, + "y": 15 + }, + "properties": { + "inputs": [ + "rolling_mean_temp_str", + "temperature_data_str" + ], + "og_node_id": "dd373f8", + "outputs": [], + "params": [], + "title": "consume climwin-dev-user-name-at-domain-com", + "vars": [ + { + "color": "#c5a987", + "direction": "input", + "name": "rolling_mean_temp_str", + "type": "datatype" + }, + { + "color": "#aca953", + "direction": "input", + "name": "temperature_data_str", + "type": "datatype" + } + ] + }, + "type": "input-output" + } + }, + "offset": { + "x": 0, + "y": 0 + }, + "scale": 1, + "selected": {} + }, + "confs": {}, + "container_source": "", + "dependencies": [ + { + "asname": "", + "module": "", + "name": "climwin" + }, + { + "asname": "", + "module": "", + "name": "zoo" + } + ], + "inputs": [ + "rolling_mean_temp_str", + "temperature_data_str" + ], + "kernel": "IRkernel", + "node_id": "dd373f8", + "notebook_dict": { + "cells": [ + { + "cell_type": "code", + "execution_count": 2, + "id": "84eee104-ceac-4ccb-8a66-d75b6715c4f8", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": "Original Temperature Data:\n 12.1976217672389, 13.8491125525836, 22.7935415707456, 15.3525419571229, 15.6464386758047, 23.5753249344164, 17.304581029946, 8.67469382696733, 11.5657357405324, 12.7716901495002, 21.1204089871973, 16.7990691352868, 17.0038572529703, 15.5534135797256, 12.2207943262296, 23.9345656840154, 17.4892523911462, 5.16691421685181, 18.5067795078184, 12.6360429613603, 9.66088147006577, 13.9101254267085, 9.8699777584638, 11.3555438535443, 11.8748036607537, 6.56653344628793, 19.1889352224726, 15.7668655891826, 9.30931531494026, 21.2690746053496, 17.1323211073841, 13.5246425850386, 19.4756283052251, 19.3906674376652, 19.1079054081874, 18.4432012705005, 17.7695882676879, 14.6904414471164, 13.4701866813004, 13.0976449949381, 11.5264651053974, 13.960413609902, 8.67301824215868, 25.8447798266926, 21.039809991525, 9.38445708398325, 12.9855758235046, 12.6667232318839, 18.8998255916816, 14.5831546676409, 16.2665925699738, 14.8572662232565, 14.7856477135434, 21.8430114200723, 13.8711450717037, 22.5823530221477, 7.25623597884889, 17.9230687481803, 15.6192712192231, 16.0797078437199, 16.8981974137994, 12.4883827344535, 13.3339630816529, 9.90712308446456, 9.64104386762211, 16.5176432070213, 17.2410488931471, 15.2650211336525, 19.6113373393987, 25.2504234281357, 12.5448441697173, 3.45415562179594, 20.0286926223113, 11.453996187088, 11.5599569176632, 20.1278568484835, 13.576134964745, 8.89641143872732, 15.9065173987458, 14.3055431878048, 15.0288209294994, 16.9264020056317, 13.146699841038, 18.2218827425942, 13.8975671909062, 16.6589098195785, 20.4841950657467, 17.175907454169, 13.3703420723439, 20.7440380922555, 19.9675192798106, 17.7419847975404, 16.1936586755572, 11.8604696198031, 21.80326224265, 11.9987020642644, 25.9366649650829, 22.6630531309259, 13.8214982044976, 9.8678954984661, 11.4479671815035, 16.2844185457826, 13.7665406076881, 13.2622870030113, 10.2419071636749, 14.7748613759554, 11.0754776527146, 6.66029031705932, 13.0988673985612, 19.5949830453038, 12.123265186958, 18.0398216111252, 6.91058645855418, 14.7221901723773, 17.5970360197173, 16.5057668108336, 15.5283809707447, 11.7964699584731, 10.7514782698321, 9.87935604697543, 15.5882329855006, 10.262626929076, 12.5472127814967, 13.7195390390088, 24.219310026161, 11.7402504915227, 16.1769328614243, 15.3898042478186, 10.1907168293494, 14.643459569382, 22.2227542921167, 17.2575202653961, 15.2061646099647, 12.8875158383019, 4.73376389229742, 20.6566860670709, 7.69679964537589, 18.6997375543867, 24.5455178460874, 7.780534195141, 18.5089216768736, 13.6890125529877, 7.13927920427256, 7.42666173109124, 6.99231913212703, 12.3454673891485, 7.6912220750205, 18.4395838648791, 25.5005447026284, 8.56484761982411, 18.9386942373759, 18.8452112050045, 16.6610128947506, 9.95811695861496, 14.4027369668467, 13.5980233241488, 17.8149476661024, 13.1378062194809, 19.8848669334281, 13.1270957111649, 20.2635573278967, 9.75411496666967, 8.69922377620944, 31.205199674712, 12.9157120591978, 16.4911379577036, 18.1828483701692, 12.5810968714563, 17.584310221568, 16.8448226369254, 13.9230974617915, 15.3264651676266, 14.8296637313077, 25.6422594950809, 11.2933195186359, 9.52001866462668, 15.1889419958554, 16.5524037472157, 17.1826173945509, 12.7081733364445, 9.68336933014404, 21.3159258804475, 13.2517480602322, 10.6724356867331, 13.8186021552945, 14.0141205282572, 20.5496014485682, 15.423686460986, 18.7702689259226, 12.5035399141387, 16.072226547908, 13.3765704425458, 15.4729176408679, 10.5231832101123, 8.44599233336014, 24.9860669237398, 18.0035441183621, 8.74364319187528, 11.9441704165979, 9.07259957701345, 25.9940517444186, 21.5620648821676, 13.6742747165182, 17.7159702961604, 12.9283002604057, 12.6187655269221, 11.0569858107488, 12.0269136627024, 23.2545373366834, 14.7298593745728, 15.5962261821379, 16.2184371479955, 21.1623793924267, 12.4196808452761, 10.0374642480398, 23.3784846620159, 12.7941839154736, 11.3846701503006, 8.81863440558353, 8.57642138841102, 12.1301326035101, 18.0899290858326, 20.5492406946486, 18.5379417691779, 13.1817135145237, 15.298749686923, 11.4770176815996, 11.4139091921299, 19.4232524948846, 9.92203710698228, 24.7764698274623, 14.5484020301707, 16.0726941331461, 11.3073614763021, 12.1280565511837, 8.41491933847379, 14.0853730581364, 17.0949120246223, 16.6215217208069, 11.0923175647262, 11.05689014573, 12.4890064082857, 22.4803033492318, 9.31348189667128, 14.104742028099, 24.5118091083946, 14.495125573356, 8.20079648089303, 11.6761528236297, 17.4272998945244, 13.1219856416511, 12.1906181822511, 13.2804138293577, 15.4524832356961, 22.9925438557291, 14.5571744393056, 20.4039974807576, 18.1537705782528, 14.4318005224693, 7.33548998554701, 12.3944134122374, 12.5506477343076, 15.2357721638076, 21.5009933883341, 26.4653948691555, 22.7379052949188, 14.3342451783553, 6.21736302221181, 13.0561006796413, 15.4460361153665, 19.2250650203372, 19.8126398424214, 18.4215471470823, 8.02362825100266, 19.2482152281668, 12.7672139178639, 15.8740135008063, 15.3727558858687, 17.1408338248525, 15.1233749141307, 6.66262451207169, 18.6824798238672, 16.9301328417484, 13.6717418736089, 15.5907225552334, 15.6701932268423, 16.105097342805, 23.2042308298874, 13.9047481053326, 15.8403269194233, 20.8419193653455, 20.2709051168846, 20.7263155519018, 12.1126599947022, 25.0124136514641, 15.3335043546509, 24.3342592235343, 8.24548656984644, 15.1049179317712, 21.2495728548461, 11.423789063886, 11.2365551589113, 10.3073064819655, 9.73743360330632, 12.814202334098, 16.6558958647949, 4.9289475103964, 16.0599021668615, 21.1833752320829, 25.1878700912022, 21.5058799610029, 18.7838738189798, 6.36634800442835, 11.9924664599661, 13.2397677170869, 18.5176195137845, 14.4716433299811, 8.70675685969914, 23.4221785404706, 19.5569564589798, 16.1871513624551, 21.0905430516291, 8.30612856382514, 18.304101488949, 12.3854381184329, 18.4187276092536, 14.6958902266996, 18.1648035651573, 21.677588075297, 15.0364504515845, 20.0877931847605, 9.05782982426011, 11.3919777978199, 22.5960885569409, 16.8869398651196, 4.73888589783135, 8.17981273958812, 13.9960949220544, 19.3288970216724, 14.4905837214239, 18.1209373601033, 19.7950268889391, 23.3552741443147, 15.2800836663748, 14.7400904690955, 6.23381320428865, 15.4966379704391, 12.1407497105222, 10.1299520859795, 14.1004688447623, 20.0747158637183, 5.03625755655709, 12.8636035639729, 15.5831864179135, 10.5339621497252, 16.6695147124961, 17.0571496030787, 14.83481920362, 2.67050903119986, 27.8572907293332, 13.973503712659, 18.2559664079384, 16.3688324551827, 20.1233661740917, 19.0882972318704 \n\nRolling Mean Temperature in Moving Windows:\n 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14.764481219841, 14.9289711978458, 14.918155532261, 14.8075584234103, 14.942162606095, 15.0575448305078, 14.8864740417106, 14.9019742829687, 15.1024992036403, 15.1659809016659, 15.1768460631805, 14.8570479337872, 14.762426082941, 14.4847314492473, 14.8277769908128, 15.121744179656, 14.636740559655, 14.4866180074002, 14.7366116412346, 14.7497131773634, 14.8146169009061, 15.034807270903, 15.066378630788, 15.2302342959573, 15.5798165481749, 15.6463612618732, 16.1802219144019, 15.7824652729477, 15.854338711581, 16.0646705750571, 15.8916916830028, 15.8838875598833, 15.8493455648638, 15.6446233907447, 15.328505245638, 15.0129431942858, 14.9487579255032, 14.9311399463518, 14.9502926025697, 15.1549976245063, 15.5600902389462, 15.5940362077569, 15.2438276081534, 15.6223500874918, 15.142657299505, 14.8266621970429, 15.1847755225263, 15.2044608272343, 15.0787837674624, 14.9790068276978, 14.9697531117033, 14.9284940570208, 14.9974652497667, 14.9428336540165, 14.8221293647672, 14.823010102074, 14.6255619953217, 15.0664939648849, 15.0415885884179, 14.9666242835219, 15.1221019584731, 15.2244126873401, 15.3995327561097, 15.4948559425732, 15.5599674937511, 15.9653747729187, 15.8147434014935, 16.104597270558, 16.3511983371338, 16.1582036993038, 15.6454527683148, 15.608890202041, 16.0365656328405, 15.8278272323531, 15.8881035928839, 15.8441686010843, 15.6657354186667, 15.5823801749323, 15.5078428042101, 15.4142544708706, 15.5905691327872, 15.4937172747025, 15.5308312615523, 15.3229608154695, 15.2063043964623, 15.3296200240893, 15.3245152571311, 15.1593214539644, 14.9800068707745, 14.8927114106908, 14.5305553425148, 14.3845791327045, 14.1352672037557, 14.013719007287, 14.0756879879272, 14.1562229140442, 14.1476078616195, 13.8222834581642, 13.5798418287273, 13.458815782889, 13.6180012519195, 13.9771608222733, 14.0095975462604, 14.057585013003, 14.045092640846, 13.8614878651334, 14.0575486881706, 13.944926087926, 14.3462409958369, 14.7277960107544, 14.3339810490823, 14.5468362654128, 14.4018092968083, 14.4094323883322, 14.166248106956, 13.8127575440363, 13.6740808966468, 13.4128422667893, 13.6342793970029, 14.1259149447628, 14.082097997191, 14.1937800389202, 14.4798661814512, 14.616992851893, 14.4916121158798, 14.1643930139027, 14.2263187749902, 14.2809192684795, 14.2058526675349, 14.5289910043375, 14.478445542397, 14.413138976923, 14.1630254669654, 13.9461274391736, 14.5567169003872, 14.8294485059506, 14.690596902305, 15.0401318597981, 14.8361771703671, 14.6041369162165, 14.906279864276, 14.7534190571066, 14.8080008109278, 15.064346961829, 15.671533553962, 15.8149002335123, 15.7207186093616, 15.9706426067227, 15.9077366028006, 15.630472359198, 15.7685832164187, 15.4600723861777, 15.5424295420258, 15.4287873808751, 15.4525980051458, 15.4331268447607, 15.4469967515643, 15.5381518776465, 15.6143478856967, 15.5771946187798, 15.5564094255456, 15.416698399546, 15.5374469154085, 15.7632367108971, 15.0738361620771, 14.9248455045492, 15.2080098034171, 15.2020329950235, 15.0741178723708, 14.8861132122051, 14.6270391102081, 15.0294042529623, 15.2372575767803, 15.1987446096207, 14.9345349696567, 14.9890343277157, 15.0923258897922, 14.9545940169553, 14.8037443474715, 15.006141678876, 15.0735312134802, 15.27062644188, 15.1007101507983, 15.3643978618714, 15.4226393671562, 15.2966014369144, 15.6087469080397, 15.3502329902699, 15.215599113247, 14.883877962569, 14.7529740117114, 14.6215708802315, 14.7786828350077, 14.9478936034671, 15.2150522221026, 15.3729095948081, 15.0499990202475, 14.8324481390221, 14.9214570056973, 15.1707597416401, 15.1990743259724, 15.1584882620739, 14.9246995003407, 15.0046468142283, 14.791026520233, 14.7643517299256, 14.624223523644, 14.7251697652236, 14.8941030439542, 14.673002523425, 14.5517511297635, 14.4004399285499, 14.2761255705595, 14.3200563691197, 14.2165164041662, 14.3520923301682, 14.3898698117141, 14.4465678669769, 14.3404387446633, 14.4356893585982, 14.7307186421353, 14.7637804100733, 14.5671367132873, 14.3248424844442, 14.2219938666615, 14.5490215447017, 14.5243023697811, 14.8218683630864, 15.0465304092905, 14.8801486768766, 14.7939304394954, 14.3811952256546, 14.3146034157925, 14.2867060168146, 14.6264937472156, 15.1044050244814, 15.5818378896962, 15.5901336270368, 15.2275486602898, 15.108701292251, 15.253825243939, 15.5260977397592, 15.7702188542304, 15.6349269808254, 15.5919318593031, 15.763380965972, 15.371894459621, 15.417857390536, 15.6569227040352, 15.8390787374093, 15.7622812380629, 15.5469692004102, 15.7633645884641, 15.8850218888771, 15.8256638434742, 15.578936466791, 15.6160370930422, 15.4727404217771, 15.6410890968316, 15.6235206829271, 15.9070152473896, 16.1885987791599, 16.4459406919125, 16.6289588048489, 16.3160143583945, 16.2675816511381, 16.0207682864626, 16.3541020879685, 16.421706206223, 16.4900001146273, 16.6834513392767, 16.423408807395, 16.1375393179446, 15.8670646291074, 15.9241914741842, 15.7097243777152, 15.8393471092796, 15.4745115762659, 15.497416452299, 15.63216783254, 15.9676510051091, 16.4624261867401, 16.4658059865772, 16.1136798253332, 16.0577039782118, 15.9793388169402, 16.074253026505, 16.0198045594108, 15.5365554270712, 15.8538031082425, 15.977690759561, 15.8225318261314, 15.8498530906228, 15.435846857687, 15.6422282408285, 15.2213290563941, 15.3241698315476, 15.0028908649864, 15.3335347648301, 15.5526237696143, 15.3455196895056, 15.6343198268681, 15.5616956490464, 15.5978513595748, 16.0264731913627, 16.1622311090634, 15.7649974434979, 15.8733596178043, 15.8045660429774, 15.7427501026304, 15.3861738903045, 15.2733424702745, 15.3070475726064, 15.8733451106027, 15.9829323508163, 16.0329431092166, 15.6234828989, 15.6576493869153, 15.7721158152761, 15.3290416001264, 15.1471586796525, 15.2767441630279, 14.7416013131922, 14.8935171465304, 14.8028199774959, 14.7411041118723, 14.6827970153137, 14.761505661193, 14.6505061824751, 14.0169368810052, 14.4442982235968, 14.2404885745268, 14.5470931273161, 14.7129882825615, 14.6305642031332, 14.7039427820249, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 \n" + } + ], + "source": "# consume climwin\n\ncat(\"Original Temperature Data:\\n\", head(temperature_data_str), \"\\n\\n\")\ncat(\"Rolling Mean Temperature in Moving Windows:\\n\", head(coredata(rolling_mean_temp_str)), \"\\n\")\n" + } + ], + "metadata": { + "kernelspec": { + "display_name": "R", + "language": "R", + "name": "R" + }, + "language_info": { + "codemirror_mode": "r", + "file_extension": ".r", + "mimetype": "text/x-r-source", + "name": "R", + "pygments_lexer": "r", + "version": "4.3.2" + } + }, + "nbformat": 4, + "nbformat_minor": 5 + }, + "original_source": "\ncat(\"Original Temperature Data:\\n\", head(temperature_data_str), \"\\n\\n\")\ncat(\"Rolling Mean Temperature in Moving Windows:\\n\", head(coredata(rolling_mean_temp_str)), \"\\n\")", + "outputs": [], + "param_values": {}, + "params": [], + "task_name": "consume-climwin-dev-user-name-at-domain-com", + "title": "consume climwin-dev-user-name-at-domain-com", + "types": { + "rolling_mean_temp_str": "str", + "temperature_data_str": "str" + }, + "skip_exec": false +} diff --git a/vreapis/tests/resources/cells/create-file-dev-user-name-at-domain-com.json b/vreapis/tests/resources/cells/create-file-dev-user-name-at-domain-com.json new file mode 100644 index 00000000..f8241ad8 --- /dev/null +++ b/vreapis/tests/resources/cells/create-file-dev-user-name-at-domain-com.json @@ -0,0 +1,93 @@ +{ + "all_inputs": [], + "base_image": {"build":"ghcr.io/qcdis/naavre/naavre-cell-build-python:latest","runtime":"ghcr.io/qcdis/naavre/naavre-cell-runtime-python:latest"}, + "chart_obj": { + "hovered": {}, + "links": {}, + "nodes": { + "5a918c8": { + "id": "5a918c8", + "ports": { + "file_path": { + "id": "file_path", + "properties": { + "color": "#663a78" + }, + "type": "right" + } + }, + "position": { + "x": 35, + "y": 15 + }, + "properties": { + "inputs": [], + "og_node_id": "5a918c8", + "outputs": [ + "file_path" + ], + "params": [], + "title": "Create file-dev-user-name-at-domain-com", + "vars": [ + { + "color": "#663a78", + "direction": "output", + "name": "file_path", + "type": "datatype" + } + ] + }, + "type": "input-output" + } + }, + "offset": { + "x": 0, + "y": 0 + }, + "scale": 1, + "selected": {} + }, + "confs": { + "conf_data_folder": "conf_data_folder = os.path.join('/tmp','data')" + }, + "container_source": "", + "dependencies": [ + { + "asname": null, + "module": "os.path", + "name": "isfile" + }, + { + "asname": null, + "module": "os.path", + "name": "join" + }, + { + "asname": null, + "module": "os", + "name": "listdir" + }, + { + "asname": null, + "module": "", + "name": "os" + } + ], + "inputs": [], + "kernel": "ipython", + "node_id": "5a918c8", + "original_source": "conf_data_folder = os.path.join('/tmp','data')\n\nL = [\"a\\n\", \"b\\n\", \"c\\n\"]\nfile_path = os.path.join(conf_data_folder,'hello.txt')\nfp = open(file_path, 'w')\nfp.writelines(L)\nfp.close()\n\nonlyfiles = [f for f in listdir(conf_data_folder) if isfile(join(conf_data_folder, f))]\n\nprint(onlyfiles)", + "outputs": [ + "file_path" + ], + "params": [], + "task_name": "create-file-dev-user-name-at-domain-com", + "title": "Create file-dev-user-name-at-domain-com", + "types": { + "file_path": "str" + }, + "example_inputs" : [ + "--id", + "0" + ] +} diff --git a/vreapis/tests/resources/cells/create-file-r-dev-user-name-at-domain-com.json b/vreapis/tests/resources/cells/create-file-r-dev-user-name-at-domain-com.json new file mode 100644 index 00000000..0873cf67 --- /dev/null +++ b/vreapis/tests/resources/cells/create-file-r-dev-user-name-at-domain-com.json @@ -0,0 +1,70 @@ +{ + "all_inputs": [], + "base_image": {"build":"ghcr.io/qcdis/naavre/naavre-cell-build-r:latest","runtime":"ghcr.io/qcdis/naavre/naavre-cell-runtime-r:latest"}, + "chart_obj": { + "hovered": {}, + "links": {}, + "nodes": { + "c45fb67": { + "id": "c45fb67", + "ports": { + "file_path": { + "id": "file_path", + "properties": { + "color": "#663a78" + }, + "type": "right" + } + }, + "position": { + "x": 35, + "y": 15 + }, + "properties": { + "inputs": [], + "og_node_id": "c45fb67", + "outputs": [ + "file_path" + ], + "params": [], + "title": "Create file R-dev-user-name-at-domain-com", + "vars": [ + { + "color": "#663a78", + "direction": "output", + "name": "file_path", + "type": "datatype" + } + ] + }, + "type": "input-output" + } + }, + "offset": { + "x": 0, + "y": 0 + }, + "scale": 1, + "selected": {} + }, + "confs": {}, + "container_source": "", + "dependencies": [], + "inputs": [], + "kernel": "IRkernel", + "node_id": "c45fb67", + "original_source": "\nL <- c(\"a\", \"b\", \"c\")\n\nconf_data_folder <- \"/tmp/data\"\nfile_path <- file.path(conf_data_folder, \"hello.txt\")\n\nwriteLines(L, file_path)\n\nonlyfiles <- list.files(conf_data_folder, full.names = TRUE)\n\nprint(onlyfiles)", + "outputs": [ + "file_path" + ], + "params": [], + "task_name": "create-file-r-dev-user-name-at-domain-com", + "title": "Create file R-dev-user-name-at-domain-com", + "types": { + "file_path": "str" + }, + "example_inputs" : [ + "--id", + "0" + ] +} diff --git a/vreapis/tests/resources/cells/input-lists-dev-user-name-at-domain-com.json b/vreapis/tests/resources/cells/input-lists-dev-user-name-at-domain-com.json new file mode 100644 index 00000000..684c6e2a --- /dev/null +++ b/vreapis/tests/resources/cells/input-lists-dev-user-name-at-domain-com.json @@ -0,0 +1,103 @@ +{ + "all_inputs": [ + "msg" + ], + "base_image": {"build":"ghcr.io/qcdis/naavre/naavre-cell-build-python:latest","runtime":"ghcr.io/qcdis/naavre/naavre-cell-runtime-python:latest"}, + "chart_obj": { + "hovered": {}, + "links": {}, + "nodes": { + "3e4f548": { + "id": "3e4f548", + "ports": { + "list_of_ints": { + "id": "list_of_ints", + "properties": { + "color": "#5f1f93" + }, + "type": "right" + }, + "list_of_paths": { + "id": "list_of_paths", + "properties": { + "color": "#79c5d2" + }, + "type": "right" + }, + "msg": { + "id": "msg", + "properties": { + "color": "#7bd279" + }, + "type": "left" + } + }, + "position": { + "x": 35, + "y": 15 + }, + "properties": { + "inputs": [ + "msg" + ], + "og_node_id": "3e4f548", + "outputs": [ + "list_of_ints", + "list_of_paths" + ], + "params": [], + "title": "input lists-dev-user-name-at-domain-com", + "vars": [ + { + "color": "#7bd279", + "direction": "input", + "name": "msg", + "type": "datatype" + }, + { + "color": "#5f1f93", + "direction": "output", + "name": "list_of_ints", + "type": "datatype" + }, + { + "color": "#79c5d2", + "direction": "output", + "name": "list_of_paths", + "type": "datatype" + } + ] + }, + "type": "input-output" + } + }, + "offset": { + "x": 0, + "y": 0 + }, + "scale": 1, + "selected": {} + }, + "confs": {}, + "container_source": "", + "dependencies": [], + "inputs": [ + "msg" + ], + "kernel": "ipython", + "node_id": "3e4f548", + "original_source": "\nlist_of_paths = [\"/webdav/LAZ/targets_myname\",\"/webdav/LAZ/targets_myname\",\"/webdav/LAZ/targets_myname\",\"/webdav/LAZ/targets_myname\",\"/webdav/LAZ/targets_myname\",\"/webdav/LAZ/targets_myname\",\"/webdav/LAZ/targets_myname\",\"/webdav/LAZ/targets_myname\",\"/webdav/LAZ/targets_myname\",\"/webdav/LAZ/targets_myname\"]\nlist_of_ints = [1,2,35,6,65]\nprint(msg)", + "outputs": [ + "list_of_paths", + "list_of_ints" + ], + "params": [], + "task_name": "input-lists-dev-user-name-at-domain-com", + "title": "input lists-dev-user-name-at-domain-com", + "types": { + "list_of_ints": "list", + "list_of_paths": "list", + "msg": "str" + }, + "skip_exec": true +} diff --git a/vreapis/tests/resources/cells/input-lists-r-dev-user-name-at-domain-com.json b/vreapis/tests/resources/cells/input-lists-r-dev-user-name-at-domain-com.json new file mode 100644 index 00000000..6ed403d6 --- /dev/null +++ b/vreapis/tests/resources/cells/input-lists-r-dev-user-name-at-domain-com.json @@ -0,0 +1,83 @@ +{ + "all_inputs": [], + "base_image": {"build":"ghcr.io/qcdis/naavre/naavre-cell-build-r:latest","runtime":"ghcr.io/qcdis/naavre/naavre-cell-runtime-r:latest"}, + "chart_obj": { + "hovered": {}, + "links": {}, + "nodes": { + "3cfdc6f": { + "id": "3cfdc6f", + "ports": { + "list_of_ints": { + "id": "list_of_ints", + "properties": { + "color": "#5f1f93" + }, + "type": "right" + }, + "list_of_paths": { + "id": "list_of_paths", + "properties": { + "color": "#79c5d2" + }, + "type": "right" + } + }, + "position": { + "x": 35, + "y": 15 + }, + "properties": { + "inputs": [], + "og_node_id": "3cfdc6f", + "outputs": [ + "list_of_ints", + "list_of_paths" + ], + "params": [], + "title": "input lists R-dev-user-name-at-domain-com", + "vars": [ + { + "color": "#5f1f93", + "direction": "output", + "name": "list_of_ints", + "type": "datatype" + }, + { + "color": "#79c5d2", + "direction": "output", + "name": "list_of_paths", + "type": "datatype" + } + ] + }, + "type": "input-output" + } + }, + "offset": { + "x": 0, + "y": 0 + }, + "scale": 1, + "selected": {} + }, + "confs": {}, + "container_source": "", + "dependencies": [], + "inputs": [], + "kernel": "IRkernel", + "node_id": "3cfdc6f", + "original_source": "\nlist_of_paths <- c(\n \"/webdav/LAZ/targets_myname\",\n \"/webdav/LAZ/targets_myname\",\n \"/webdav/LAZ/targets_myname\",\n \"/webdav/LAZ/targets_myname\"\n)\n\nlist_of_ints <- c(1, 2, 35, 6, 65)\n\nprint(list_of_paths)\nprint(list_of_ints)", + "outputs": [ + "list_of_paths", + "list_of_ints" + ], + "params": [], + "task_name": "input-lists-r-dev-user-name-at-domain-com", + "title": "input lists R-dev-user-name-at-domain-com", + "types": { + "list_of_ints": "list", + "list_of_paths": "list" + }, + "skip_exec": true +} diff --git a/vreapis/tests/resources/cells/install-and-load-the-climwinb-packages-dev-user-name-at-domain-com.json b/vreapis/tests/resources/cells/install-and-load-the-climwinb-packages-dev-user-name-at-domain-com.json new file mode 100644 index 00000000..5e5f8b1d --- /dev/null +++ b/vreapis/tests/resources/cells/install-and-load-the-climwinb-packages-dev-user-name-at-domain-com.json @@ -0,0 +1,130 @@ +{ + "all_inputs": [], + "base_image": {"build":"ghcr.io/qcdis/naavre/naavre-cell-build-lter-life-veluwe:latest","runtime":"ghcr.io/qcdis/naavre/naavre-cell-runtime-lter-life-veluwe:latest"}, + "chart_obj": { + "hovered": {}, + "links": {}, + "nodes": { + "7b4271d": { + "id": "7b4271d", + "ports": { + "rolling_mean_temp_str": { + "id": "rolling_mean_temp_str", + "properties": { + "color": "#c5a987" + }, + "type": "right" + }, + "temperature_data_str": { + "id": "temperature_data_str", + "properties": { + "color": "#aca953" + }, + "type": "right" + } + }, + "position": { + "x": 35, + "y": 15 + }, + "properties": { + "inputs": [], + "og_node_id": "7b4271d", + "outputs": [ + "rolling_mean_temp_str", + "temperature_data_str" + ], + "params": [], + "title": "Install and load the climwinb packages-dev-user-name-at-domain-com", + "vars": [ + { + "color": "#c5a987", + "direction": "output", + "name": "rolling_mean_temp_str", + "type": "datatype" + }, + { + "color": "#aca953", + "direction": "output", + "name": "temperature_data_str", + "type": "datatype" + } + ] + }, + "type": "input-output" + } + }, + "offset": { + "x": 0, + "y": 0 + }, + "scale": 1, + "selected": {} + }, + "confs": {}, + "container_source": "", + "dependencies": [ + { + "asname": "", + "module": "", + "name": "climwin" + }, + { + "asname": "", + "module": "", + "name": "zoo" + } + ], + "inputs": [], + "kernel": "IRkernel", + "node_id": "7b4271d", + "notebook_dict": { + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "4681c280-3623-47c4-a020-fe8f34166536", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": "Loading required package: ggplot2\n\nLoading required package: gridExtra\n\nLoading required package: Matrix\n\n\nAttaching package: \u2018zoo\u2019\n\n\nThe following objects are masked from \u2018package:base\u2019:\n\n as.Date, as.Date.numeric\n\n\n" + } + ], + "source": "# Install and load the climwinb packages\nif (!requireNamespace(\"climwin\", quietly = TRUE)) {\n install.packages(\"climwin\",repos = \"http://cran.us.r-project.org\")\n}\nif (!requireNamespace(\"zoo\", quietly = TRUE)) {\n install.packages(\"zoo\",repos = \"http://cran.us.r-project.org\")\n}\n\nzoo = ''\nclimwin = ''\nlibrary(climwin)\nlibrary(zoo)\n\n# Generate example temperature data\nset.seed(123)\ntemperature_data <- rnorm(365, mean = 15, sd = 5)\n\n# Define the window size (e.g., 30 days)\nwindow_size <- 30\n\n# Convert temperature_data to a zoo object for rollmean\ntemperature_zoo <- zoo::zoo(temperature_data)\n\n# Calculate the rolling mean temperature within a moving window\nrolling_mean_temp <- rollmean(temperature_zoo, k = window_size, fill = 0.0)\n\ntemperature_zoo_str <- toString(temperature_zoo)\nrolling_mean_temp_str <- toString(rolling_mean_temp)\ntemperature_data_str <- toString(temperature_data)\n\n\n" + } + ], + "metadata": { + "kernelspec": { + "display_name": "R", + "language": "R", + "name": "R" + }, + "language_info": { + "codemirror_mode": "r", + "file_extension": ".r", + "mimetype": "text/x-r-source", + "name": "R", + "pygments_lexer": "r", + "version": "4.3.2" + } + }, + "nbformat": 4, + "nbformat_minor": 5 + }, + "original_source": "if (!requireNamespace(\"climwin\", quietly = TRUE)) {\n install.packages(\"climwin\",repos = \"http://cran.us.r-project.org\")\n}\nif (!requireNamespace(\"zoo\", quietly = TRUE)) {\n install.packages(\"zoo\",repos = \"http://cran.us.r-project.org\")\n}\n\nzoo = ''\nclimwin = ''\nlibrary(climwin)\nlibrary(zoo)\n\nset.seed(123)\ntemperature_data <- rnorm(365, mean = 15, sd = 5)\n\nwindow_size <- 30\n\ntemperature_zoo <- zoo::zoo(temperature_data)\n\nrolling_mean_temp <- rollmean(temperature_zoo, k = window_size, fill = 0.0)\n\ntemperature_zoo_str <- toString(temperature_zoo)\nrolling_mean_temp_str <- toString(rolling_mean_temp)\ntemperature_data_str <- toString(temperature_data)\n", + "outputs": [ + "rolling_mean_temp_str", + "temperature_data_str" + ], + "param_values": {}, + "params": [], + "task_name": "install-and-load-the-climwinb-packages-dev-user-name-at-domain-com", + "title": "Install and load the climwinb packages-dev-user-name-at-domain-com", + "types": { + "rolling_mean_temp_str": "str", + "temperature_data_str": "str" + }, + "skip_exec": false +} diff --git a/vreapis/tests/resources/cells/knmi-vol-h5-to-odim-h5-dev-user-name-at-domain-com.json b/vreapis/tests/resources/cells/knmi-vol-h5-to-odim-h5-dev-user-name-at-domain-com.json new file mode 100644 index 00000000..8c98afa4 --- /dev/null +++ b/vreapis/tests/resources/cells/knmi-vol-h5-to-odim-h5-dev-user-name-at-domain-com.json @@ -0,0 +1,93 @@ +{ + "all_inputs": [ + "a" + ], + "base_image": {"build":"ghcr.io/qcdis/naavre/naavre-cell-build-vol2bird:latest","runtime":"ghcr.io/qcdis/naavre/naavre-cell-runtime-vol2bird:latest"}, + "chart_obj": { + "hovered": {}, + "links": {}, + "nodes": { + "e25ec34": { + "id": "e25ec34", + "ports": { + "a": { + "id": "a", + "properties": { + "color": "#77862d" + }, + "type": "left" + }, + "msg": { + "id": "msg", + "properties": { + "color": "#7bd279" + }, + "type": "right" + } + }, + "position": { + "x": 35, + "y": 15 + }, + "properties": { + "inputs": [ + "a" + ], + "og_node_id": "e25ec34", + "outputs": [ + "msg" + ], + "params": [], + "title": "KNMI-vol-h5-to-ODIM-h5-dev-user-name-at-domain-com", + "vars": [ + { + "color": "#77862d", + "direction": "input", + "name": "a", + "type": "datatype" + }, + { + "color": "#7bd279", + "direction": "output", + "name": "msg", + "type": "datatype" + } + ] + }, + "type": "input-output" + } + }, + "offset": { + "x": 0, + "y": 0 + }, + "scale": 1, + "selected": {} + }, + "confs": {}, + "container_source": "", + "dependencies": [ + { + "asname": null, + "module": "", + "name": "os" + } + ], + "inputs": [ + "a" + ], + "kernel": "ipython", + "node_id": "e25ec34", + "original_source": "print(a)\ncmd = \"KNMI_vol_h5_to_ODIM_h5 \"\n\nmsg = os.system(cmd) # returns the exit code in unix", + "outputs": [ + "msg" + ], + "params": [], + "task_name": "knmi-vol-h5-to-odim-h5-dev-user-name-at-domain-com", + "title": "KNMI-vol-h5-to-ODIM-h5-dev-user-name-at-domain-com", + "types": { + "a": "int", + "msg": "int" + }, + "skip_exec": true +} diff --git a/vreapis/tests/resources/cells/loop-file-lines-dev-user-name-at-domain-com.json b/vreapis/tests/resources/cells/loop-file-lines-dev-user-name-at-domain-com.json new file mode 100644 index 00000000..d9a47f78 --- /dev/null +++ b/vreapis/tests/resources/cells/loop-file-lines-dev-user-name-at-domain-com.json @@ -0,0 +1,87 @@ +{ + "all_inputs": [ + "lines" + ], + "base_image": {"build":"ghcr.io/qcdis/naavre/naavre-cell-build-python:latest","runtime":"ghcr.io/qcdis/naavre/naavre-cell-runtime-python:latest"}, + "chart_obj": { + "hovered": {}, + "links": {}, + "nodes": { + "5d9de73": { + "id": "5d9de73", + "ports": { + "count": { + "id": "count", + "properties": { + "color": "#783d3a" + }, + "type": "right" + }, + "lines": { + "id": "lines", + "properties": { + "color": "#ac535c" + }, + "type": "left" + } + }, + "position": { + "x": 35, + "y": 15 + }, + "properties": { + "inputs": [ + "lines" + ], + "og_node_id": "5d9de73", + "outputs": [ + "count" + ], + "params": [], + "title": "loop file lines-dev-user-name-at-domain-com", + "vars": [ + { + "color": "#ac535c", + "direction": "input", + "name": "lines", + "type": "datatype" + }, + { + "color": "#783d3a", + "direction": "output", + "name": "count", + "type": "datatype" + } + ] + }, + "type": "input-output" + } + }, + "offset": { + "x": 0, + "y": 0 + }, + "scale": 1, + "selected": {} + }, + "confs": {}, + "container_source": "", + "dependencies": [], + "inputs": [ + "lines" + ], + "kernel": "ipython", + "node_id": "5d9de73", + "original_source": "\ncount = 0\nfor l in lines:\n count += 1\n print(\"Line{}: {}\".format(count, l.strip()))", + "outputs": [ + "count" + ], + "params": [], + "task_name": "loop-file-lines-dev-user-name-at-domain-com", + "title": "loop file lines-dev-user-name-at-domain-com", + "types": { + "count": "int", + "lines": "list" + }, + "skip_exec": true +} diff --git a/vreapis/tests/resources/cells/loop-file-lines-r-dev-user-name-at-domain-com.json b/vreapis/tests/resources/cells/loop-file-lines-r-dev-user-name-at-domain-com.json new file mode 100644 index 00000000..a20cf0df --- /dev/null +++ b/vreapis/tests/resources/cells/loop-file-lines-r-dev-user-name-at-domain-com.json @@ -0,0 +1,87 @@ +{ + "all_inputs": [ + "lines" + ], + "base_image": {"build":"ghcr.io/qcdis/naavre/naavre-cell-build-r:latest","runtime":"ghcr.io/qcdis/naavre/naavre-cell-runtime-r:latest"}, + "chart_obj": { + "hovered": {}, + "links": {}, + "nodes": { + "f893a78": { + "id": "f893a78", + "ports": { + "count": { + "id": "count", + "properties": { + "color": "#783d3a" + }, + "type": "right" + }, + "lines": { + "id": "lines", + "properties": { + "color": "#ac535c" + }, + "type": "left" + } + }, + "position": { + "x": 35, + "y": 15 + }, + "properties": { + "inputs": [ + "lines" + ], + "og_node_id": "f893a78", + "outputs": [ + "count" + ], + "params": [], + "title": "loop file lines R-dev-user-name-at-domain-com", + "vars": [ + { + "color": "#ac535c", + "direction": "input", + "name": "lines", + "type": "datatype" + }, + { + "color": "#783d3a", + "direction": "output", + "name": "count", + "type": "datatype" + } + ] + }, + "type": "input-output" + } + }, + "offset": { + "x": 0, + "y": 0 + }, + "scale": 1, + "selected": {} + }, + "confs": {}, + "container_source": "", + "dependencies": [], + "inputs": [ + "lines" + ], + "kernel": "IRkernel", + "node_id": "f893a78", + "original_source": "count <- 0\nfor (l in lines) {\n count <- count + 1\n cat(sprintf(\"Line %d: %s\\n\", count, trimws(l)))\n}", + "outputs": [ + "count" + ], + "params": [], + "task_name": "loop-file-lines-r-dev-user-name-at-domain-com", + "title": "loop file lines R-dev-user-name-at-domain-com", + "types": { + "count": "int", + "lines": "list" + }, + "skip_exec": true +} diff --git a/vreapis/tests/resources/cells/loop-int-list-dev-user-name-at-domain-com.json b/vreapis/tests/resources/cells/loop-int-list-dev-user-name-at-domain-com.json new file mode 100644 index 00000000..f2b2d69f --- /dev/null +++ b/vreapis/tests/resources/cells/loop-int-list-dev-user-name-at-domain-com.json @@ -0,0 +1,87 @@ +{ + "all_inputs": [ + "list_of_ints" + ], + "base_image": {"build":"ghcr.io/qcdis/naavre/naavre-cell-build-python:latest","runtime":"ghcr.io/qcdis/naavre/naavre-cell-runtime-python:latest"}, + "chart_obj": { + "hovered": {}, + "links": {}, + "nodes": { + "e1578ef": { + "id": "e1578ef", + "ports": { + "a": { + "id": "a", + "properties": { + "color": "#77862d" + }, + "type": "right" + }, + "list_of_ints": { + "id": "list_of_ints", + "properties": { + "color": "#5f1f93" + }, + "type": "left" + } + }, + "position": { + "x": 35, + "y": 15 + }, + "properties": { + "inputs": [ + "list_of_ints" + ], + "og_node_id": "e1578ef", + "outputs": [ + "a" + ], + "params": [], + "title": "loop int list-dev-user-name-at-domain-com", + "vars": [ + { + "color": "#5f1f93", + "direction": "input", + "name": "list_of_ints", + "type": "datatype" + }, + { + "color": "#77862d", + "direction": "output", + "name": "a", + "type": "datatype" + } + ] + }, + "type": "input-output" + } + }, + "offset": { + "x": 0, + "y": 0 + }, + "scale": 1, + "selected": {} + }, + "confs": {}, + "container_source": "", + "dependencies": [], + "inputs": [ + "list_of_ints" + ], + "kernel": "ipython", + "node_id": "e1578ef", + "original_source": "\nfor i in list_of_ints:\n a = i -1\n print(a)", + "outputs": [ + "a" + ], + "params": [], + "task_name": "loop-int-list-dev-user-name-at-domain-com", + "title": "loop int list-dev-user-name-at-domain-com", + "types": { + "a": "int", + "list_of_ints": "list" + }, + "skip_exec": true +} diff --git a/vreapis/tests/resources/cells/loop-int-list-r-dev-user-name-at-domain-com.json b/vreapis/tests/resources/cells/loop-int-list-r-dev-user-name-at-domain-com.json new file mode 100644 index 00000000..85ecec25 --- /dev/null +++ b/vreapis/tests/resources/cells/loop-int-list-r-dev-user-name-at-domain-com.json @@ -0,0 +1,69 @@ +{ + "all_inputs": [ + "list_of_ints" + ], + "base_image": {"build":"ghcr.io/qcdis/naavre/naavre-cell-build-r:latest","runtime":"ghcr.io/qcdis/naavre/naavre-cell-runtime-r:latest"}, + "chart_obj": { + "hovered": {}, + "links": {}, + "nodes": { + "3285abd": { + "id": "3285abd", + "ports": { + "list_of_ints": { + "id": "list_of_ints", + "properties": { + "color": "#5f1f93" + }, + "type": "left" + } + }, + "position": { + "x": 35, + "y": 15 + }, + "properties": { + "inputs": [ + "list_of_ints" + ], + "og_node_id": "3285abd", + "outputs": [], + "params": [], + "title": "loop int list R-dev-user-name-at-domain-com", + "vars": [ + { + "color": "#5f1f93", + "direction": "input", + "name": "list_of_ints", + "type": "datatype" + } + ] + }, + "type": "input-output" + } + }, + "offset": { + "x": 0, + "y": 0 + }, + "scale": 1, + "selected": {} + }, + "confs": {}, + "container_source": "", + "dependencies": [], + "inputs": [ + "list_of_ints" + ], + "kernel": "IRkernel", + "node_id": "3285abd", + "original_source": "\nfor (l in list_of_ints) {\n print(l)\n}", + "outputs": [], + "params": [], + "task_name": "loop-int-list-r-dev-user-name-at-domain-com", + "title": "loop int list R-dev-user-name-at-domain-com", + "types": { + "list_of_ints": "list" + }, + "skip_exec": true +} diff --git a/vreapis/tests/resources/cells/loop-list-dev-user-name-at-domain-com.json b/vreapis/tests/resources/cells/loop-list-dev-user-name-at-domain-com.json new file mode 100644 index 00000000..eff08db5 --- /dev/null +++ b/vreapis/tests/resources/cells/loop-list-dev-user-name-at-domain-com.json @@ -0,0 +1,69 @@ +{ + "all_inputs": [ + "list_of_paths" + ], + "base_image": {"build":"ghcr.io/qcdis/naavre/naavre-cell-build-python:latest","runtime":"ghcr.io/qcdis/naavre/naavre-cell-runtime-python:latest"}, + "chart_obj": { + "hovered": {}, + "links": {}, + "nodes": { + "6ce40db": { + "id": "6ce40db", + "ports": { + "list_of_paths": { + "id": "list_of_paths", + "properties": { + "color": "#79c5d2" + }, + "type": "left" + } + }, + "position": { + "x": 35, + "y": 15 + }, + "properties": { + "inputs": [ + "list_of_paths" + ], + "og_node_id": "6ce40db", + "outputs": [], + "params": [], + "title": "loop list-dev-user-name-at-domain-com", + "vars": [ + { + "color": "#79c5d2", + "direction": "input", + "name": "list_of_paths", + "type": "datatype" + } + ] + }, + "type": "input-output" + } + }, + "offset": { + "x": 0, + "y": 0 + }, + "scale": 1, + "selected": {} + }, + "confs": {}, + "container_source": "", + "dependencies": [], + "inputs": [ + "list_of_paths" + ], + "kernel": "ipython", + "node_id": "6ce40db", + "original_source": "\nfor l in list_of_paths:\n print(l)", + "outputs": [], + "params": [], + "task_name": "loop-list-dev-user-name-at-domain-com", + "title": "loop list-dev-user-name-at-domain-com", + "types": { + "list_of_paths": "list" + }, + "skip_exec": true +} diff --git a/vreapis/tests/resources/cells/loop-list-r-dev-user-name-at-domain-com.json b/vreapis/tests/resources/cells/loop-list-r-dev-user-name-at-domain-com.json new file mode 100644 index 00000000..1f4796cb --- /dev/null +++ b/vreapis/tests/resources/cells/loop-list-r-dev-user-name-at-domain-com.json @@ -0,0 +1,69 @@ +{ + "all_inputs": [ + "list_of_paths" + ], + "base_image": {"build":"ghcr.io/qcdis/naavre/naavre-cell-build-r:latest","runtime":"ghcr.io/qcdis/naavre/naavre-cell-runtime-r:latest"}, + "chart_obj": { + "hovered": {}, + "links": {}, + "nodes": { + "943e143": { + "id": "943e143", + "ports": { + "list_of_paths": { + "id": "list_of_paths", + "properties": { + "color": "#79c5d2" + }, + "type": "left" + } + }, + "position": { + "x": 35, + "y": 15 + }, + "properties": { + "inputs": [ + "list_of_paths" + ], + "og_node_id": "943e143", + "outputs": [], + "params": [], + "title": "loop list R-dev-user-name-at-domain-com", + "vars": [ + { + "color": "#79c5d2", + "direction": "input", + "name": "list_of_paths", + "type": "datatype" + } + ] + }, + "type": "input-output" + } + }, + "offset": { + "x": 0, + "y": 0 + }, + "scale": 1, + "selected": {} + }, + "confs": {}, + "container_source": "", + "dependencies": [], + "inputs": [ + "list_of_paths" + ], + "kernel": "IRkernel", + "node_id": "943e143", + "original_source": "\nfor (l in list_of_paths) {\n print(l)\n}", + "outputs": [], + "params": [], + "task_name": "loop-list-r-dev-user-name-at-domain-com", + "title": "loop list R-dev-user-name-at-domain-com", + "types": { + "list_of_paths": "list" + }, + "skip_exec": true +} diff --git a/vreapis/tests/resources/cells/print-msg-dev-user-name-at-domain-com.json b/vreapis/tests/resources/cells/print-msg-dev-user-name-at-domain-com.json new file mode 100644 index 00000000..7741afc3 --- /dev/null +++ b/vreapis/tests/resources/cells/print-msg-dev-user-name-at-domain-com.json @@ -0,0 +1,74 @@ +{ + "all_inputs": [ + "msg" + ], + "base_image": {"build":"ghcr.io/qcdis/naavre/naavre-cell-build-python:latest","runtime":"ghcr.io/qcdis/naavre/naavre-cell-runtime-python:latest"}, + "chart_obj": { + "hovered": {}, + "links": {}, + "nodes": { + "a74dc70": { + "id": "a74dc70", + "ports": { + "msg": { + "id": "msg", + "properties": { + "color": "#7bd279" + }, + "type": "left" + } + }, + "position": { + "x": 35, + "y": 15 + }, + "properties": { + "inputs": [ + "msg" + ], + "og_node_id": "a74dc70", + "outputs": [], + "params": [], + "title": "Print msg-dev-user-name-at-domain-com", + "vars": [ + { + "color": "#7bd279", + "direction": "input", + "name": "msg", + "type": "datatype" + } + ] + }, + "type": "input-output" + } + }, + "offset": { + "x": 0, + "y": 0 + }, + "scale": 1, + "selected": {} + }, + "confs": {}, + "container_source": "", + "dependencies": [], + "inputs": [ + "msg" + ], + "kernel": "ipython", + "node_id": "a74dc70", + "original_source": "print('msg:', str(msg))", + "outputs": [], + "params": [], + "task_name": "print-msg-dev-user-name-at-domain-com", + "title": "Print msg-dev-user-name-at-domain-com", + "types": { + "msg": "int" + }, + "example_inputs" : [ + "--id", + "0", + "--msg", + "1" + ] +} diff --git a/vreapis/tests/resources/cells/read-file-lines-dev-user-name-at-domain-com.json b/vreapis/tests/resources/cells/read-file-lines-dev-user-name-at-domain-com.json new file mode 100644 index 00000000..d1313f66 --- /dev/null +++ b/vreapis/tests/resources/cells/read-file-lines-dev-user-name-at-domain-com.json @@ -0,0 +1,110 @@ +{ + "all_inputs": [ + "file_path" + ], + "base_image": {"build":"ghcr.io/qcdis/naavre/naavre-cell-build-python:latest","runtime":"ghcr.io/qcdis/naavre/naavre-cell-runtime-python:latest"}, + "chart_obj": { + "hovered": {}, + "links": {}, + "nodes": { + "4deac70": { + "id": "4deac70", + "ports": { + "file_path": { + "id": "file_path", + "properties": { + "color": "#663a78" + }, + "type": "left" + }, + "lines": { + "id": "lines", + "properties": { + "color": "#ac535c" + }, + "type": "right" + } + }, + "position": { + "x": 35, + "y": 15 + }, + "properties": { + "inputs": [ + "file_path" + ], + "og_node_id": "4deac70", + "outputs": [ + "lines" + ], + "params": [], + "title": "read file lines-dev-user-name-at-domain-com", + "vars": [ + { + "color": "#663a78", + "direction": "input", + "name": "file_path", + "type": "datatype" + }, + { + "color": "#ac535c", + "direction": "output", + "name": "lines", + "type": "datatype" + } + ] + }, + "type": "input-output" + } + }, + "offset": { + "x": 0, + "y": 0 + }, + "scale": 1, + "selected": {} + }, + "confs": { + "conf_data_folder": "conf_data_folder = os.path.join('/tmp','data')" + }, + "container_source": "", + "dependencies": [ + { + "asname": null, + "module": "os.path", + "name": "isfile" + }, + { + "asname": null, + "module": "os.path", + "name": "join" + }, + { + "asname": null, + "module": "os", + "name": "listdir" + }, + { + "asname": null, + "module": "", + "name": "os" + } + ], + "inputs": [ + "file_path" + ], + "kernel": "ipython", + "node_id": "4deac70", + "original_source": "conf_data_folder = os.path.join('/tmp','data')\n\nonlyfiles = [f for f in listdir(conf_data_folder) if isfile(join(conf_data_folder, f))]\n\nprint(onlyfiles)\n\nf = open(file_path, 'r')\nlines = f.readlines()\nf.close()", + "outputs": [ + "lines" + ], + "params": [], + "task_name": "read-file-lines-dev-user-name-at-domain-com", + "title": "read file lines-dev-user-name-at-domain-com", + "types": { + "file_path": "str", + "lines": "list" + }, + "skip_exec": true +} diff --git a/vreapis/tests/resources/cells/read-file-lines-r-dev-user-name-at-domain-com.json b/vreapis/tests/resources/cells/read-file-lines-r-dev-user-name-at-domain-com.json new file mode 100644 index 00000000..347c164b --- /dev/null +++ b/vreapis/tests/resources/cells/read-file-lines-r-dev-user-name-at-domain-com.json @@ -0,0 +1,89 @@ +{ + "all_inputs": [ + "file_path" + ], + "base_image": {"build":"ghcr.io/qcdis/naavre/naavre-cell-build-r:latest","runtime":"ghcr.io/qcdis/naavre/naavre-cell-runtime-r:latest"}, + "chart_obj": { + "hovered": {}, + "links": {}, + "nodes": { + "47b2d97": { + "id": "47b2d97", + "ports": { + "file_path": { + "id": "file_path", + "properties": { + "color": "#663a78" + }, + "type": "left" + }, + "lines": { + "id": "lines", + "properties": { + "color": "#ac535c" + }, + "type": "right" + } + }, + "position": { + "x": 35, + "y": 15 + }, + "properties": { + "inputs": [ + "file_path" + ], + "og_node_id": "47b2d97", + "outputs": [ + "lines" + ], + "params": [], + "title": "read file lines R-dev-user-name-at-domain-com", + "vars": [ + { + "color": "#663a78", + "direction": "input", + "name": "file_path", + "type": "datatype" + }, + { + "color": "#ac535c", + "direction": "output", + "name": "lines", + "type": "datatype" + } + ] + }, + "type": "input-output" + } + }, + "offset": { + "x": 0, + "y": 0 + }, + "scale": 1, + "selected": {} + }, + "confs": { + "conf_data_folder": "conf_data_folder <- file.path('/tmp', 'data')" + }, + "container_source": "", + "dependencies": [], + "inputs": [ + "file_path" + ], + "kernel": "IRkernel", + "node_id": "47b2d97", + "original_source": "conf_data_folder <- file.path('/tmp', 'data')\n\nonlyfiles <- list.files(conf_data_folder, full.names = TRUE)\n\nprint(onlyfiles)\n\nf <- file(file_path, \"r\")\nlines <- readLines(f)\nclose(f)", + "outputs": [ + "lines" + ], + "params": [], + "task_name": "read-file-lines-r-dev-user-name-at-domain-com", + "title": "read file lines R-dev-user-name-at-domain-com", + "types": { + "file_path": "str", + "lines": "list" + }, + "skip_exec": true +} diff --git a/vreapis/tests/resources/cells/vol2bird-dev-user-name-at-domain-com.json b/vreapis/tests/resources/cells/vol2bird-dev-user-name-at-domain-com.json new file mode 100644 index 00000000..4fef8818 --- /dev/null +++ b/vreapis/tests/resources/cells/vol2bird-dev-user-name-at-domain-com.json @@ -0,0 +1,93 @@ +{ + "all_inputs": [ + "a" + ], + "base_image": {"build":"ghcr.io/qcdis/naavre/naavre-cell-build-vol2bird:latest","runtime":"ghcr.io/qcdis/naavre/naavre-cell-runtime-vol2bird:latest"}, + "chart_obj": { + "hovered": {}, + "links": {}, + "nodes": { + "ded63d7": { + "id": "ded63d7", + "ports": { + "a": { + "id": "a", + "properties": { + "color": "#77862d" + }, + "type": "left" + }, + "msg": { + "id": "msg", + "properties": { + "color": "#7bd279" + }, + "type": "right" + } + }, + "position": { + "x": 35, + "y": 15 + }, + "properties": { + "inputs": [ + "a" + ], + "og_node_id": "ded63d7", + "outputs": [ + "msg" + ], + "params": [], + "title": "vol2bird-dev-user-name-at-domain-com", + "vars": [ + { + "color": "#77862d", + "direction": "input", + "name": "a", + "type": "datatype" + }, + { + "color": "#7bd279", + "direction": "output", + "name": "msg", + "type": "datatype" + } + ] + }, + "type": "input-output" + } + }, + "offset": { + "x": 0, + "y": 0 + }, + "scale": 1, + "selected": {} + }, + "confs": {}, + "container_source": "", + "dependencies": [ + { + "asname": null, + "module": "", + "name": "os" + } + ], + "inputs": [ + "a" + ], + "kernel": "ipython", + "node_id": "ded63d7", + "original_source": "print(a)\ncmd = \"vol2bird --version\"\n\nmsg = os.system(cmd) # returns the exit code in unix\n", + "outputs": [ + "msg" + ], + "params": [], + "task_name": "vol2bird-dev-user-name-at-domain-com", + "title": "vol2bird-dev-user-name-at-domain-com", + "types": { + "a": "int", + "msg": "int" + }, + "skip_exec": true +} diff --git a/vreapis/tests/resources/notebooks/R-notebook.ipynb b/vreapis/tests/resources/notebooks/R-notebook.ipynb new file mode 100644 index 00000000..36ff7417 --- /dev/null +++ b/vreapis/tests/resources/notebooks/R-notebook.ipynb @@ -0,0 +1,74 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "59b2ee0b-958d-431f-a396-b4b34c77d540", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "# input\n", + "a = 2 \n", + "numbers <- c(a, 4, 6, 8, 10)" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "0a83d716-23c8-4eb2-b246-9a8f4cdf9ac2", + "metadata": {}, + "outputs": [], + "source": [ + "#csddfv\n", + "average <- mean(numbers)" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "73a1f742-2a8e-4c38-a1c0-5ff8bb372579", + "metadata": { + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[1] 6\n" + ] + } + ], + "source": [ + "print(average)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "528b97bb-396a-4520-8be0-9aeaed7e4715", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "R [conda env:jupyterlab] *", + "language": "R", + "name": "conda-env-jupyterlab-r" + }, + "language_info": { + "codemirror_mode": "r", + "file_extension": ".r", + "mimetype": "text/x-r-source", + "name": "R", + "pygments_lexer": "r", + "version": "4.2.3" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/vreapis/tests/resources/notebooks/TraitsComputation_NaaVRE.json b/vreapis/tests/resources/notebooks/TraitsComputation_NaaVRE.json new file mode 100644 index 00000000..ad056c0d --- /dev/null +++ b/vreapis/tests/resources/notebooks/TraitsComputation_NaaVRE.json @@ -0,0 +1,155 @@ +{ + "save": false, + "kernel": "IRkernel", + "cell_index": 1, + "notebook": { + "metadata": { + "kernelspec": { + "display_name": "R", + "language": "R", + "name": "ir" + }, + "language_info": { + "codemirror_mode": "r", + "file_extension": ".r", + "mimetype": "text/x-r-source", + "name": "R", + "pygments_lexer": "r", + "version": "4.2.3" + } + }, + "nbformat_minor": 5, + "nbformat": 4, + "cells": [ + { + "cell_type": "markdown", + "source": "### DESCRIPTION\nScript to compute demographic and morphological traits such as biovolume (\u00b5m3), surface area (\u00b5m2), surface/volume ratio, density (cells/L), cell carbon content (pg), total biovolume (\u00b5m\u00b3/L), total carbon content (pg/L). \n### INPUT VARIABLES\n- datain: the input file\n- CalcType: the computation type simplified or advanced\n- CompTraits: one or more among biovolume,totalbiovolume,density,surfacearea,surfacevolumeratio,cellcarboncontent,totalcarboncontent\n- CountingStrategy: one among density0,density1,density2,density3 \n\n### OUTPUT \nfile in .csv format, containing the original input data and the new calculated traits", + "metadata": {}, + "id": "prepared-shield" + }, + { + "cell_type": "code", + "source": "# input parameters\ninstall.packages(\"dplyr\",repos = \"http://cran.us.r-project.org\")\ndplyr = ''\nlibrary(dplyr)\n\n\nparam_datain = 'input/Phytoplankton__Progetto_Strategico_2009_2012_Australia.csv'\n\nparam_CalcType = 'advanced'\n\nparam_biovolume = 1 # if 1 it is calculated, if 0 it is not calculated\nparam_totalbiovolume = 1\nparam_density = 1\nparam_surfacearea = 1\nparam_surfacevolumeratio = 1\nparam_cellcarboncontent = 1\nparam_totalcarboncontent = 1\nparam_CountingStrategy = ''\n\nif (param_density==1) {param_CountingStrategy <- 'density0'}\n\n# read input cvs\ndf.datain=read.csv(param_datain,stringsAsFactors=FALSE,sep = \";\", dec = \".\")\nmeasurementremarks = tolower(df.datain$measurementremarks) # eliminate capital letters\ndf.datain$measurementremarks <- tolower(df.datain$measurementremarks) # eliminate capital letters\nindex = c(1:nrow(df.datain))\ndf.datain$index <- c(1:nrow(df.datain)) # needed to restore rows order later\n\n# read support cvs file\ndf.operator<-read.csv('input/2_FILEinformativo_OPERATORE.csv',stringsAsFactors=FALSE,sep = \";\", dec = \".\") ## load internal database \ndf.operator[df.operator==('no')]<-NA\ndf.operator[df.operator==('see note')]<-NA\n\n# merge dataframes\ndf.merged <- merge(x = df.datain, y = df.operator, by = c(\"scientificname\",\"measurementremarks\"), all.x = TRUE)\n\n# check if mandatory fields are present\ndiameterofsedimentationchamber = 'diameterofsedimentationchamber'\ndiameteroffieldofview = 'diameteroffieldofview'\ntransectcounting = 'transectcounting'\nnumberofcountedfields = 'numberofcountedfields'\nnumberoftransects = 'numberoftransects'\nsettlingvolume = 'settlingvolume'\ndilutionfactor = 'dilutionfactor'\n\nif(!'diameterofsedimentationchamber'%in%names(df.merged))df.merged$diameterofsedimentationchamber=NA\nif(!'diameteroffieldofview'%in%names(df.merged))df.merged$diameteroffieldofview=NA\nif(!'transectcounting'%in%names(df.merged))df.merged$transectcounting=NA\nif(!'numberofcountedfields'%in%names(df.merged))df.merged$numberofcountedfields=df.merged$transectcounting\nif(!'numberoftransects'%in%names(df.merged))df.merged$numberoftransects==df.merged$transectcounting\nif(!'settlingvolume'%in%names(df.merged))df.merged$settlingvolume=NA\nif(!'dilutionfactor'%in%names(df.merged))df.merged$dilutionfactor=1\n\n# save merged dataframe as csv\noutput_dfmerged = 'output/dfmerged.csv'\noutput_dfdatain = 'output/dfdatain.csv'\nwrite.table(df.merged,paste(output_dfmerged,sep=''),row.names=FALSE,sep = \";\",dec = \".\",quote=FALSE)\nwrite.table(df.datain,paste(output_dfdatain,sep=''),row.names=FALSE,sep = \";\",dec = \".\",quote=FALSE) ", + "metadata": { + "tags": [] + }, + "execution_count": 2, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": "Updating HTML index of packages in '.Library'\n\nMaking 'packages.html' ...\n done\n\n" + }, + { + "ename": "ERROR", + "evalue": "Error in `$<-.data.frame`(`*tmp*`, measurementremarks, value = character(0)): replacement has 0 rows, data has 6\n", + "output_type": "error", + "traceback": [ + "Error in `$<-.data.frame`(`*tmp*`, measurementremarks, value = character(0)): replacement has 0 rows, data has 6\nTraceback:\n", + "1. `$<-`(`*tmp*`, measurementremarks, value = character(0))", + "2. `$<-.data.frame`(`*tmp*`, measurementremarks, value = character(0))", + "3. stop(sprintf(ngettext(N, \"replacement has %d row, data has %d\", \n . \"replacement has %d rows, data has %d\"), N, nrows), domain = NA)" + ] + } + ], + "id": "steady-metabolism" + }, + { + "cell_type": "code", + "source": "#Calculate \n\ndf.merged=read.csv(output_dfmerged,stringsAsFactors=FALSE,sep = \";\", dec = \".\")\nCalcType=param_CalcType\n\ndf.temp = '' # Because the Component cannot identify '<-' we need to initialize these variables so they are not showing up as inputs\nmd.form = ''\nmissingdimension = ''\ndf.temp2 = ''\nindex = ''\nformulaformissingdimensionsimplified = '' \nmd = ''\ndf.merged.concat = '' \nmd.formulas = ''\nmissingdimensionsimplified = ''\nformulaformissingdimension = '' \n\nif(CalcType=='advanced'){\n df.merged.concat <- df.merged[is.na(df.merged$formulaformissingdimension),]\n md.formulas = ''\n md.formulas <- unique(df.merged[!is.na(df.merged$formulaformissingdimension),]$formulaformissingdimension)\n for(md.form in md.formulas){\n df.temp <- subset(df.merged,formulaformissingdimension==md.form)\n for(md in unique(df.temp$missingdimension)){\n df.temp2 <- df.temp[df.temp$missingdimension==md,]\n df.temp2[[md]] <- round(with(df.temp2,eval(parse(text=md.form))),2)\n df.merged.concat <- rbind(df.merged.concat,df.temp2)\n }\n }\n df.merged.concat = ''\n df.merged.concat <- df.merged.concat[order(df.merged.concat$index),]\n df.merged = '' \n df.merged <- df.merged.concat\n} else if(CalcType=='simplified'){\n formulaformissingdimensionsimplified = ''\n df.merged.concat <- df.merged[is.na(df.merged$formulaformissingdimensionsimplified),]\n md.formulas <- unique(df.merged[!is.na(df.merged$formulaformissingdimensionsimplified),]$formulaformissingdimensionsimplified)\n for(md.form in md.formulas){\n df.temp <- subset(df.merged,formulaformissingdimensionsimplified==md.form)\n for(md in unique(df.temp$missingdimensionsimplified)){\n df.temp2 = '' \n df.temp2 <- df.temp[df.temp$missingdimensionsimplified==md,]\n df.temp2[[md]] <- round(with(df.temp2,eval(parse(text=md.form))),2)\n df.merged.concat <- rbind(df.merged.concat,df.temp2)\n }\n }\n df.merged.concat <- df.merged.concat[order(df.merged.concat$index),]\n df.merged <- df.merged.concat\n}\n \nwrite.table(df.merged,paste(output_dfmerged,sep=''),row.names=FALSE,sep = \";\",dec = \".\",quote=FALSE) \n", + "metadata": { + "tags": [] + }, + "execution_count": null, + "outputs": [], + "id": "reflected-cardiff" + }, + { + "cell_type": "code", + "source": "df.merged=read.csv('~/Unisalento/Lifewatch/Phyto_VRE/Script_R/Traits_Computation/dfmerged.csv',stringsAsFactors=FALSE,sep = \";\", dec = \".\")\nTraitBiovolume=param_biovolume\nCalcType=param_CalcType\n\nif(TraitBiovolume==1){\n \n if(CalcType=='advanced'){\n df.merged$biovolume <- rep(NA,length=nrow(df.merged))\n df.merged.concat <- df.merged[is.na(df.merged$formulaforbiovolume),]\n bv.formulas <- unique(df.merged[!is.na(df.merged$formulaforbiovolume),]$formulaforbiovolume)\n for(bv.form in bv.formulas){\n df.temp <- subset(df.merged,formulaforbiovolume==bv.form)\n df.temp$biovolume <- round(with(df.temp,eval(parse(text=bv.form))),2)\n df.merged.concat <- rbind(df.merged.concat,df.temp)\n }\n df.merged.concat <- df.merged.concat[order(df.merged.concat$index),]\n df.merged <- df.merged.concat\n BV_calc = df.merged$biovolume\n }\n else if(CalcType=='simplified'){\n df.merged$biovolume <- rep(NA,length=nrow(df.merged))\n df.merged.concat <- df.merged[is.na(df.merged$formulaforbiovolumesimplified),]\n bv.formulas <- unique(df.merged[!is.na(df.merged$formulaforbiovolumesimplified),]$formulaforbiovolumesimplified)\n for(bv.form in bv.formulas){\n df.temp <- subset(df.merged,formulaforbiovolumesimplified==bv.form)\n df.temp$biovolume <- round(with(df.temp,eval(parse(text=bv.form))),2)\n df.merged.concat <- rbind(df.merged.concat,df.temp)\n }\n df.merged.concat <- df.merged.concat[order(df.merged.concat$index),]\n df.merged <- df.merged.concat\n BV_calc = df.merged$biovolume\n }\n} \n\nwrite.table(df.merged,paste('~/Unisalento/Lifewatch/Phyto_VRE/Script_R/Traits_Computation/dfmerged.csv',sep=''),row.names=FALSE,sep = \";\",dec = \".\",quote=FALSE)", + "metadata": {}, + "execution_count": 16, + "outputs": [], + "id": "induced-trash" + }, + { + "cell_type": "code", + "source": "df.merged=read.csv('~/Unisalento/Lifewatch/Phyto_VRE/Script_R/Traits_Computation/dfmerged.csv',stringsAsFactors=FALSE,sep = \";\", dec = \".\")\n\nTraitCellcarboncontent=param_cellcarboncontent\nTraitBiovolume=param_biovolume\n\nif(TraitCellcarboncontent==1){\n \n df.merged$cellcarboncontent <- rep(NA,length=nrow(df.merged))\n if(TraitBiovolume==1){\n df.merged.concat <- df.merged[is.na(df.merged$biovolume),]\n df.cc <- df.merged[!is.na(df.merged$biovolume),]\n df.cc1 <- subset(df.cc,biovolume <= 3000)\n df.cc2 <- subset(df.cc,biovolume > 3000)\n cc.formulas1 <- unique(df.merged[!is.na(df.merged$formulaforweight1),]$formulaforweight1)\n for(cc.form in cc.formulas1){\n df.temp <- subset(df.cc1,formulaforweight1==cc.form)\n df.temp$cellcarboncontent <- round(with(df.temp,eval(parse(text=tolower(cc.form)))),2)\n df.merged.concat <- rbind(df.merged.concat,df.temp)\n }\n cc.formulas2 <- unique(df.merged[!is.na(df.merged$formulaforweight2),]$formulaforweight2)\n for(cc.form in cc.formulas2){\n df.temp <- subset(df.cc2,formulaforweight2==cc.form)\n df.temp$cellcarboncontent <- round(with(df.temp,eval(parse(text=tolower(cc.form)))),2)\n df.merged.concat <- rbind(df.merged.concat,df.temp)\n }\n df.merged.concat <- df.merged.concat[order(df.merged.concat$index),]\n df.merged <- df.merged.concat\n CC_calc = df.merged$cellcarboncontent\n }\n}\n \nwrite.table(df.merged,paste('~/Unisalento/Lifewatch/Phyto_VRE/Script_R/Traits_Computation/dfmerged.csv',sep=''),row.names=FALSE,sep = \";\",dec = \".\",quote=FALSE) ", + "metadata": {}, + "execution_count": 17, + "outputs": [], + "id": "peripheral-contest" + }, + { + "cell_type": "code", + "source": "df.merged=read.csv('~/Unisalento/Lifewatch/Phyto_VRE/Script_R/Traits_Computation/dfmerged.csv',stringsAsFactors=FALSE,sep = \";\", dec = \".\")\n\nTraitDensity=param_density\nCountingStrategy=param_CountingStrategy\n\nif(TraitDensity==1){\n df.merged$density <- rep(NA,length=nrow(df.merged))\n # default method to calculate the density\n if(CountingStrategy=='density0'){ \n df.merged.concat <- df.merged[(is.na(df.merged$volumeofsedimentationchamber)) & (is.na(df.merged$transectcounting)),]\n df.temp <- df.merged[!is.na(df.merged$volumeofsedimentationchamber) & !is.na(df.merged$transectcounting),]\n df.temp1 <- subset(df.temp,volumeofsedimentationchamber <= 5)\n df.temp1$density <- df.temp1$organismquantity/df.temp1$transectcounting*1000/0.001979\n df.merged.concat <- rbind(df.merged.concat,df.temp1)\n df.temp2 <- subset(df.temp,(volumeofsedimentationchamber > 5) & (volumeofsedimentationchamber <= 10))\n df.temp2$density <- df.temp2$organismquantity/df.temp2$transectcounting*1000/0.00365\n df.merged.concat <- rbind(df.merged.concat,df.temp2)\n df.temp3 <- subset(df.temp,(volumeofsedimentationchamber > 10) & (volumeofsedimentationchamber <= 25))\n df.temp3$density <- df.temp3$organismquantity/df.temp3$transectcounting*1000/0.010555\n df.merged.concat <- rbind(df.merged.concat,df.temp3)\n df.temp4 <- subset(df.temp,(volumeofsedimentationchamber > 25) & (volumeofsedimentationchamber <= 50))\n df.temp4$density <- df.temp4$organismquantity/df.temp4$transectcounting*1000/0.021703\n df.merged.concat <- rbind(df.merged.concat,df.temp4)\n df.temp5 <- subset(df.temp,volumeofsedimentationchamber > 50)\n df.temp5$density <- df.temp5$organismquantity/df.temp5$transectcounting*1000/0.041598\n df.merged.concat <- rbind(df.merged.concat,df.temp5)\n df.merged.concat <- df.merged.concat[order(df.merged.concat$index),]\n df.merged <- df.merged.concat\n D_calc <- round(df.merged$density,2)\n }\n # counts per random field\n else if(CountingStrategy=='density1'){\n df.merged$areaofsedimentationchamber <- ((df.merged$diameterofsedimentationchamber/2)^2)*pi\n df.merged$areaofcountingfield <- ((df.merged$diameteroffieldofview/2)^2)*pi\n df.merged$density <- round(df.merged$organismquantity*1000*df.merged$areaofsedimentationchamber/df.merged$numberofcountedfields*df.merged$areaofcountingfield*df.merged$settlingvolume,2)\n }\n # counts per diameter transects\n else if(CountingStrategy=='density2'){\n df.merged$density <- round(((df.merged$organismquantity/df.merged$numberoftransects)*(pi/4)*(df.merged$diameterofsedimentationchamber/df.merged$diameteroffieldofview))*1000/df.merged$settlingvolume,2)\n }\n # counting method for whole chamber\n else if(CountingStrategy=='density3'){\n df.merged$density <- round((df.merged$organismquantity*1000)/df.merged$settlingvolume,2)\n }\n D_calc = df.merged$density/df.merged$dilutionfactor\n}\n \n \nwrite.table(df.merged,paste('~/Unisalento/Lifewatch/Phyto_VRE/Script_R/Traits_Computation/dfmerged.csv',sep=''),row.names=FALSE,sep = \";\",dec = \".\",quote=FALSE)\n", + "metadata": {}, + "execution_count": 18, + "outputs": [], + "id": "cellular-constraint" + }, + { + "cell_type": "code", + "source": "df.merged=read.csv('~/Unisalento/Lifewatch/Phyto_VRE/Script_R/Traits_Computation/dfmerged.csv',stringsAsFactors=FALSE,sep = \";\", dec = \".\")\n\nTraitTotalbiovolume=param_totalbiovolume\nTraitDensity=param_density\nTraitBiovolume=param_biovolume\n\nif(TraitTotalbiovolume==1){\n if((TraitDensity==0) & (!'density'%in%names(df.merged))) df.merged$density<-NA\n if((TraitBiovolume==0) & (!'biovolume'%in%names(df.merged))) df.merged$biovolume<-NA\n TBV_calc = round(df.merged$density*df.merged$biovolume,2)\n}\n\nwrite.table(df.merged,paste('~/Unisalento/Lifewatch/Phyto_VRE/Script_R/Traits_Computation/dfmerged.csv',sep=''),row.names=FALSE,sep = \";\",dec = \".\",quote=FALSE)\n", + "metadata": {}, + "execution_count": 19, + "outputs": [], + "id": "intense-muscle" + }, + { + "cell_type": "code", + "source": "df.merged=read.csv('~/Unisalento/Lifewatch/Phyto_VRE/Script_R/Traits_Computation/dfmerged.csv',stringsAsFactors=FALSE,sep = \";\", dec = \".\")\n\nTraitSurfacearea=param_surfacearea\nCalcType=param_CalcType\n\nif(TraitSurfacearea==1){\n if(CalcType=='advanced'){\n df.merged$surfacearea <- rep(NA,length=nrow(df.merged))\n df.merged.concat <- df.merged[is.na(df.merged$formulaforsurface),]\n sa.formulas <- unique(df.merged[!is.na(df.merged$formulaforsurface),]$formulaforsurface)\n for(sa.form in sa.formulas){\n df.temp <- subset(df.merged,formulaforsurface==sa.form)\n df.temp$surfacearea <- round(with(df.temp,eval(parse(text=sa.form))),2)\n df.merged.concat <- rbind(df.merged.concat,df.temp)\n }\n df.merged.concat <- df.merged.concat[order(df.merged.concat$index),]\n df.merged <- df.merged.concat\n SA_calc <- df.merged$surfacearea\n }\n else if(CalcType=='simplified'){\n df.merged$surfacearea <- rep(NA,length=nrow(df.merged))\n df.merged.concat <- df.merged[is.na(df.merged$formulaforsurfacesimplified),]\n sa.formulas <- unique(df.merged[!is.na(df.merged$formulaforsurfacesimplified),]$formulaforsurfacesimplified)\n for(sa.form in sa.formulas){\n df.temp <- subset(df.merged,formulaforsurfacesimplified==sa.form)\n df.temp$surfacearea <- round(with(df.temp,eval(parse(text=sa.form))),2)\n df.merged.concat <- rbind(df.merged.concat,df.temp)\n }\n df.merged.concat <- df.merged.concat[order(df.merged.concat$index),]\n df.merged <- df.merged.concat\n SA_calc <- df.merged$surfacearea\n }\n}\n\nwrite.table(df.merged,paste('~/Unisalento/Lifewatch/Phyto_VRE/Script_R/Traits_Computation/dfmerged.csv',sep=''),row.names=FALSE,sep = \";\",dec = \".\",quote=FALSE)\n", + "metadata": {}, + "execution_count": 20, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": "Warning message in sqrt((h^2) - (d^2)):\n\"Si \u00e8 prodotto un NaN\"\nWarning message in sqrt((h^2) - (d^2)):\n\"Si \u00e8 prodotto un NaN\"\nWarning message in sqrt(4 * h^2 - ((b + c)^2)):\n\"Si \u00e8 prodotto un NaN\"\nWarning message in sqrt(4 * h^2 - ((b + c)^2)):\n\"Si \u00e8 prodotto un NaN\"\n" + } + ], + "id": "shared-explorer" + }, + { + "cell_type": "code", + "source": "df.merged=read.csv('~/Unisalento/Lifewatch/Phyto_VRE/Script_R/Traits_Computation/dfmerged.csv',stringsAsFactors=FALSE,sep = \";\", dec = \".\")\n\nTraitSurfacevolumeratio=param_surfacevolumeratio\nTraitSurfacearea=param_surfacearea\nTraitBiovolume=param_biovolume\n\nif(TraitSurfacevolumeratio==1){\n if((TraitSurfacearea==0) & (!'surfacearea'%in%names(df.merged))) df.merged$surfacearea<-NA\n if((TraitBiovolume==0) & (!'biovolume'%in%names(df.merged))) df.merged$biovolume<-NA\n SVR_calc<-round(df.merged$surfacearea/df.merged$biovolume,2)\n}\n\nwrite.table(df.merged,paste('~/Unisalento/Lifewatch/Phyto_VRE/Script_R/Traits_Computation/dfmerged.csv',sep=''),row.names=FALSE,sep = \";\",dec = \".\",quote=FALSE)\n", + "metadata": {}, + "execution_count": 21, + "outputs": [], + "id": "affected-movement" + }, + { + "cell_type": "code", + "source": "df.merged=read.csv('~/Unisalento/Lifewatch/Phyto_VRE/Script_R/Traits_Computation/dfmerged.csv',stringsAsFactors=FALSE,sep = \";\", dec = \".\")\n\nTraitTotalcarboncontent=param_totalcarboncontent\nTraitDensity=param_density\nTraitCellcarboncontent=param_cellcarboncontent\n\nif(TraitTotalcarboncontent==1){\n if((TraitDensity==0) & (!'density'%in%names(df.merged))) df.merged$density<-NA\n if((TraitCellcarboncontent==0) & (!'cellcarboncontent'%in%names(df.merged))) df.merged$cellcarboncontent<-NA\n TCC_calc<-round(df.merged$density*df.merged$cellcarboncontent,2)\n}\n\nwrite.table(df.merged,paste('~/Unisalento/Lifewatch/Phyto_VRE/Script_R/Traits_Computation/dfmerged.csv',sep=''),row.names=FALSE,sep = \";\",dec = \".\",quote=FALSE)\n", + "metadata": {}, + "execution_count": 22, + "outputs": [], + "id": "threaded-english" + }, + { + "cell_type": "code", + "source": "df.datain=read.csv('~/Unisalento/Lifewatch/Phyto_VRE/Script_R/Traits_Computation/dfdatain.csv',stringsAsFactors=FALSE,sep = \";\", dec = \".\")\n\nBV=TraitBiovolume \nTBV=TraitTotalbiovolume\nD=TraitDensity\nSA=TraitSurfacearea\nSVR=TraitSurfacevolumeratio\nCC=TraitCellcarboncontent\nTCC=TraitTotalcarboncontent\n\nif(BV==1) {\n BV_column=BV_calc\n if('biovolume'%in%names(df.datain)) df.datain<-subset(df.datain,select=-biovolume) # drop column if already present\n df.datain$biovolume <- BV_column # write column with the results at the end of the dataframe\n }\nif(CC==1) {\n CC_column=CC_calc\n if('cellcarboncontent'%in%names(df.datain)) df.datain<-subset(df.datain,select=-cellcarboncontent)\n df.datain$cellcarboncontent <- CC_column\n }\nif(D==1) {\n D_column=D_calc\n df.datain$density <- D_column\n }\nif(TBV==1) {\n TBV_column=TBV_calc\n df.datain$totalbiovolume <- TBV_column\n }\nif(SA==1) {\n SA_column=SA_calc\n if('surfacearea'%in%names(df.datain)) df.datain<-subset(df.datain,select=-surfacearea)\n df.datain$surfacearea <- SA_column\n }\nif(SVR==1) {\n SVR_column=SVR_calc\n df.datain$surfacevolumeratio <- SVR_column\n }\nif(TCC==1) {\n TCC_column=TCC_calc\n df.datain$totalcarboncontent <- TCC_column\n }\n\ndf.datain <- subset(df.datain,select = -index)\n\nwrite.table(df.datain,paste('~/Unisalento/Lifewatch/Phyto_VRE/Script_R/Traits_Computation/Output/TraitsOutput.csv',sep=''),row.names=FALSE,sep = \";\",dec = \".\",quote=FALSE) ", + "metadata": {}, + "execution_count": 23, + "outputs": [], + "id": "congressional-observation" + }, + { + "cell_type": "code", + "source": "", + "metadata": {}, + "execution_count": null, + "outputs": [], + "id": "refined-calibration" + }, + { + "cell_type": "code", + "source": "", + "metadata": {}, + "execution_count": null, + "outputs": [], + "id": "closing-seating" + } + ] + } +} diff --git a/vreapis/tests/resources/notebooks/dependency_with_submodule.notebook.json b/vreapis/tests/resources/notebooks/dependency_with_submodule.notebook.json new file mode 100644 index 00000000..252667a3 --- /dev/null +++ b/vreapis/tests/resources/notebooks/dependency_with_submodule.notebook.json @@ -0,0 +1,91 @@ +{ + "save": false, + "kernel": "ipython", + "cell_index": 1, + "notebook": { + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.7" + } + }, + "nbformat_minor": 5, + "nbformat": 4, + "cells": [ + { + "cell_type": "code", + "source": "!pip install numpy", + "metadata": { + "trusted": true + }, + "execution_count": 10, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": "Collecting torch\n Using cached torch-2.2.0-cp311-cp311-manylinux1_x86_64.whl.metadata (25 kB)\nCollecting filelock (from torch)\n Using cached filelock-3.13.1-py3-none-any.whl.metadata (2.8 kB)\nRequirement already satisfied: typing-extensions>=4.8.0 in /home/alogo/miniforge3/envs/jupyterlab/lib/python3.11/site-packages (from torch) (4.9.0)\nCollecting sympy (from torch)\n Using cached sympy-1.12-py3-none-any.whl.metadata (12 kB)\nRequirement already satisfied: networkx in /home/alogo/miniforge3/envs/jupyterlab/lib/python3.11/site-packages (from torch) (3.1)\nRequirement already satisfied: jinja2 in /home/alogo/miniforge3/envs/jupyterlab/lib/python3.11/site-packages (from torch) (3.1.3)\nCollecting fsspec (from torch)\n Using cached fsspec-2024.2.0-py3-none-any.whl.metadata (6.8 kB)\nCollecting nvidia-cuda-nvrtc-cu12==12.1.105 (from torch)\n Using cached nvidia_cuda_nvrtc_cu12-12.1.105-py3-none-manylinux1_x86_64.whl.metadata (1.5 kB)\nCollecting nvidia-cuda-runtime-cu12==12.1.105 (from torch)\n Using cached nvidia_cuda_runtime_cu12-12.1.105-py3-none-manylinux1_x86_64.whl.metadata (1.5 kB)\nCollecting nvidia-cuda-cupti-cu12==12.1.105 (from torch)\n Using cached nvidia_cuda_cupti_cu12-12.1.105-py3-none-manylinux1_x86_64.whl.metadata (1.6 kB)\nCollecting nvidia-cudnn-cu12==8.9.2.26 (from torch)\n Using cached nvidia_cudnn_cu12-8.9.2.26-py3-none-manylinux1_x86_64.whl.metadata (1.6 kB)\nCollecting nvidia-cublas-cu12==12.1.3.1 (from torch)\n Using cached nvidia_cublas_cu12-12.1.3.1-py3-none-manylinux1_x86_64.whl.metadata (1.5 kB)\nCollecting nvidia-cufft-cu12==11.0.2.54 (from torch)\n Using cached nvidia_cufft_cu12-11.0.2.54-py3-none-manylinux1_x86_64.whl.metadata (1.5 kB)\nCollecting nvidia-curand-cu12==10.3.2.106 (from torch)\n Using cached nvidia_curand_cu12-10.3.2.106-py3-none-manylinux1_x86_64.whl.metadata (1.5 kB)\nCollecting nvidia-cusolver-cu12==11.4.5.107 (from torch)\n Using cached nvidia_cusolver_cu12-11.4.5.107-py3-none-manylinux1_x86_64.whl.metadata (1.6 kB)\nCollecting nvidia-cusparse-cu12==12.1.0.106 (from torch)\n Using cached nvidia_cusparse_cu12-12.1.0.106-py3-none-manylinux1_x86_64.whl.metadata (1.6 kB)\nCollecting nvidia-nccl-cu12==2.19.3 (from torch)\n Using cached nvidia_nccl_cu12-2.19.3-py3-none-manylinux1_x86_64.whl.metadata (1.8 kB)\nCollecting nvidia-nvtx-cu12==12.1.105 (from torch)\n Using cached nvidia_nvtx_cu12-12.1.105-py3-none-manylinux1_x86_64.whl.metadata (1.7 kB)\nCollecting triton==2.2.0 (from torch)\n Using cached triton-2.2.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (1.4 kB)\nCollecting nvidia-nvjitlink-cu12 (from nvidia-cusolver-cu12==11.4.5.107->torch)\n Using cached nvidia_nvjitlink_cu12-12.3.101-py3-none-manylinux1_x86_64.whl.metadata (1.5 kB)\nRequirement already satisfied: MarkupSafe>=2.0 in /home/alogo/miniforge3/envs/jupyterlab/lib/python3.11/site-packages (from jinja2->torch) (2.1.4)\nCollecting mpmath>=0.19 (from sympy->torch)\n Using cached mpmath-1.3.0-py3-none-any.whl.metadata (8.6 kB)\nUsing cached torch-2.2.0-cp311-cp311-manylinux1_x86_64.whl (755.5 MB)\nUsing cached nvidia_cublas_cu12-12.1.3.1-py3-none-manylinux1_x86_64.whl (410.6 MB)\nUsing cached nvidia_cuda_cupti_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (14.1 MB)\nUsing cached nvidia_cuda_nvrtc_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (23.7 MB)\nUsing cached nvidia_cuda_runtime_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (823 kB)\nUsing cached nvidia_cudnn_cu12-8.9.2.26-py3-none-manylinux1_x86_64.whl (731.7 MB)\nUsing cached nvidia_cufft_cu12-11.0.2.54-py3-none-manylinux1_x86_64.whl (121.6 MB)\nUsing cached nvidia_curand_cu12-10.3.2.106-py3-none-manylinux1_x86_64.whl (56.5 MB)\nDownloading nvidia_cusolver_cu12-11.4.5.107-py3-none-manylinux1_x86_64.whl (124.2 MB)\n\u001b[2K \u001b[38;2;114;156;31m\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u001b[0m \u001b[32m124.2/124.2 MB\u001b[0m \u001b[31m4.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0mm eta \u001b[36m0:00:01\u001b[0m[36m0:00:01\u001b[0m\n\u001b[?25hDownloading nvidia_cusparse_cu12-12.1.0.106-py3-none-manylinux1_x86_64.whl (196.0 MB)\n\u001b[2K \u001b[38;2;114;156;31m\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u001b[0m \u001b[32m196.0/196.0 MB\u001b[0m \u001b[31m4.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0mm eta \u001b[36m0:00:01\u001b[0m[36m0:00:01\u001b[0mm\n\u001b[?25hDownloading nvidia_nccl_cu12-2.19.3-py3-none-manylinux1_x86_64.whl (166.0 MB)\n\u001b[2K \u001b[38;2;114;156;31m\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u001b[0m \u001b[32m166.0/166.0 MB\u001b[0m \u001b[31m3.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0mm eta \u001b[36m0:00:01\u001b[0m[36m0:00:02\u001b[0m\n\u001b[?25hDownloading nvidia_nvtx_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (99 kB)\n\u001b[2K \u001b[38;2;114;156;31m\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u001b[0m \u001b[32m99.1/99.1 kB\u001b[0m \u001b[31m2.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n\u001b[?25hDownloading triton-2.2.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (167.9 MB)\n\u001b[2K \u001b[38;2;114;156;31m\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u001b[0m \u001b[32m167.9/167.9 MB\u001b[0m \u001b[31m2.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0mm eta \u001b[36m0:00:01\u001b[0m[36m0:00:02\u001b[0m\n\u001b[?25hDownloading filelock-3.13.1-py3-none-any.whl (11 kB)\nDownloading fsspec-2024.2.0-py3-none-any.whl (170 kB)\n\u001b[2K \u001b[38;2;114;156;31m\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u001b[0m \u001b[32m170.9/170.9 kB\u001b[0m \u001b[31m1.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m MB/s\u001b[0m eta \u001b[36m0:00:01\u001b[0m:01\u001b[0m\n\u001b[?25hDownloading sympy-1.12-py3-none-any.whl (5.7 MB)\n\u001b[2K \u001b[38;2;114;156;31m\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u001b[0m \u001b[32m5.7/5.7 MB\u001b[0m \u001b[31m2.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0mm eta \u001b[36m0:00:01\u001b[0m[36m0:00:01\u001b[0m\n\u001b[?25hDownloading mpmath-1.3.0-py3-none-any.whl (536 kB)\n\u001b[2K \u001b[38;2;114;156;31m\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u001b[0m \u001b[32m536.2/536.2 kB\u001b[0m \u001b[31m2.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0mm eta \u001b[36m0:00:01\u001b[0m0:01\u001b[0m:01\u001b[0m\n\u001b[?25hDownloading nvidia_nvjitlink_cu12-12.3.101-py3-none-manylinux1_x86_64.whl (20.5 MB)\n\u001b[2K \u001b[38;2;114;156;31m\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u001b[0m \u001b[32m20.5/20.5 MB\u001b[0m \u001b[31m3.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0mm eta \u001b[36m0:00:01\u001b[0m[36m0:00:01\u001b[0m\n\u001b[?25hInstalling collected packages: mpmath, sympy, nvidia-nvtx-cu12, nvidia-nvjitlink-cu12, nvidia-nccl-cu12, nvidia-curand-cu12, nvidia-cufft-cu12, nvidia-cuda-runtime-cu12, nvidia-cuda-nvrtc-cu12, nvidia-cuda-cupti-cu12, nvidia-cublas-cu12, fsspec, filelock, triton, nvidia-cusparse-cu12, nvidia-cudnn-cu12, nvidia-cusolver-cu12, torch\nSuccessfully installed filelock-3.13.1 fsspec-2024.2.0 mpmath-1.3.0 nvidia-cublas-cu12-12.1.3.1 nvidia-cuda-cupti-cu12-12.1.105 nvidia-cuda-nvrtc-cu12-12.1.105 nvidia-cuda-runtime-cu12-12.1.105 nvidia-cudnn-cu12-8.9.2.26 nvidia-cufft-cu12-11.0.2.54 nvidia-curand-cu12-10.3.2.106 nvidia-cusolver-cu12-11.4.5.107 nvidia-cusparse-cu12-12.1.0.106 nvidia-nccl-cu12-2.19.3 nvidia-nvjitlink-cu12-12.3.101 nvidia-nvtx-cu12-12.1.105 sympy-1.12 torch-2.2.0 triton-2.2.0\n" + } + ], + "id": "4f10f166-b8cf-4f9f-b90f-270bf4522d2d" + }, + { + "cell_type": "code", + "source": "#Dependency resolution submodule\nimport matplotlib.pyplot as plt\n\n# Create some sample data\nx_values = [1, 2, 3, 4, 5]\ny_values = [2, 4, 6, 8, 10]\n\n# Plot the data\nplt.plot(x_values, y_values, label='Linear Function')\n\n# Add labels and a title\nplt.xlabel('X-axis')\nplt.ylabel('Y-axis')\nplt.title('Simple Linear Plot')\n\n# Add a legend\nplt.legend()\n\n# Display the plot\nplt.show()", + "metadata": { + "trusted": true + }, + "execution_count": 30, + "outputs": [ + { + "output_type": "display_data", + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjIAAAHHCAYAAACle7JuAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/H5lhTAAAACXBIWXMAAA9hAAAPYQGoP6dpAABX20lEQVR4nO3dd3gU5eL+//em90AglEASIBBKaKGINFHpIopKxyPWDypVQBQLRVQ4giIQxHIUPEe6AjYQRQSlSUkCobfQQ4ckJKTtzu8Pv+RngECCSWY3uV/Xtdflzs7s3k8msnfmmZ21GIZhICIiIuKAnMwOICIiInKnVGRERETEYanIiIiIiMNSkRERERGHpSIjIiIiDktFRkRERByWioyIiIg4LBUZERERcVgqMiIiIuKwVGREioEqVarw5JNPmvLa48aNw2KxFOlrHjlyBIvFwpw5c4r0dYvanDlzsFgsHDlyxOwoInZLRUbEjsXFxdG9e3dCQ0Px8PCgUqVKtG/fnhkzZpgdrdBce/PeunWr2VEKzbXyd+3m5eVFnTp1eOONN0hKSiqQ15g3bx4ffvhhgTyXiD1zMTuAiNzchg0buO+++wgJCeG5556jQoUKHD9+nE2bNjFt2jQGDx6cve6+fftwcio5f5eEhoZy9epVXF1dzY7yj8yaNQsfHx+uXLnCzz//zDvvvMPq1atZv379Pz7KNW/ePHbu3MmwYcMKJqyInVKREbFT77zzDv7+/mzZsoVSpUrleOzs2bM57ru7uxdhMvNZLBY8PDzMjnFLqampeHl53XKd7t27U7ZsWQCef/55HnvsMZYsWcKmTZto3rx5UcQUcXgl5084EQdz6NAhIiIibigxAOXKlctx//pzZK5Nz6xbt44hQ4YQGBhIqVKlGDBgABkZGVy+fJknnniC0qVLU7p0aUaNGoVhGNnbXzsHZcqUKUydOpXQ0FA8PT1p06YNO3fuzFP+r776isaNG+Pp6UlAQAC9e/fm+PHjd/SzuN7NzpF58skn8fHx4eTJk3Tr1g0fHx8CAwMZOXIkVqs1x/Y2m40PP/yQiIgIPDw8KF++PAMGDODSpUs51vv222/p0qULQUFBuLu7ExYWxoQJE254vnvvvZe6deuybds27rnnHry8vHjttdfyPa77778fgPj4+Fuu99FHHxEREYG7uztBQUEMHDiQy5cv58jz448/cvTo0ezpqypVquQ7j4gj0BEZETsVGhrKxo0b2blzJ3Xr1r2j5xg8eDAVKlRg/PjxbNq0iU8//ZRSpUqxYcMGQkJCePfdd1m+fDmTJ0+mbt26PPHEEzm2/+9//0tycjIDBw4kLS2NadOmcf/99xMXF0f58uVzfd133nmHN998k549e/Lss89y7tw5ZsyYwT333ENMTMxNy1lBsFqtdOzYkWbNmjFlyhRWrVrF+++/T1hYGC+88EL2egMGDGDOnDk89dRTDBkyhPj4eKKiooiJiWH9+vXZU1Zz5szBx8eH4cOH4+Pjw+rVqxkzZgxJSUlMnjw5x2tfuHCBzp0707t3bx5//PFb/nxyc+jQIQDKlCmT6zrjxo1j/PjxtGvXjhdeeIF9+/Yxa9YstmzZkp399ddfJzExkRMnTjB16lQAfHx88p1HxCEYImKXfv75Z8PZ2dlwdnY2mjdvbowaNcpYuXKlkZGRccO6oaGhRv/+/bPvz5492wCMjh07GjabLXt58+bNDYvFYjz//PPZy7KysozKlSsbbdq0yV4WHx9vAIanp6dx4sSJ7OV//vmnARgvvfRS9rKxY8caf/+n5MiRI4azs7Pxzjvv5MgYFxdnuLi43LD8eteyb9myJdd1ruWbPXt29rL+/fsbgPHWW2/lWDcyMtJo3Lhx9v0//vjDAIy5c+fmWO+nn366YXlqauoNrz1gwADDy8vLSEtLy17Wpk0bAzA+/vjjW47tmms/s3379hnnzp0z4uPjjU8++cRwd3c3ypcvb6SkpOT4WcTHxxuGYRhnz5413NzcjA4dOhhWqzX7+aKiogzA+OKLL7KXdenSxQgNDc1THhFHpqklETvVvn17Nm7cyEMPPcT27dt577336NixI5UqVeK7777L03M888wzOU4abdasGYZh8Mwzz2Qvc3Z2pkmTJhw+fPiG7bt160alSpWy79911100a9aM5cuX5/qaS5YswWaz0bNnT86fP599q1ChAjVq1OC3337LU/Y79fzzz+e437p16xxjW7x4Mf7+/rRv3z5HvsaNG+Pj45Mjn6enZ/Z/Jycnc/78eVq3bk1qaip79+7N8Tru7u489dRT+cpas2ZNAgMDqVq1KgMGDKB69er8+OOPuZ5bs2rVKjIyMhg2bFiOk7ufe+45/Pz8+PHHH/P1+iLFgaaWROxY06ZNWbJkCRkZGWzfvp2lS5cydepUunfvTmxsLHXq1Lnl9iEhITnu+/v7AxAcHHzD8uvPDwGoUaPGDcvCw8NZtGhRrq954MABDMO46bZAoX7SyMPDg8DAwBzLSpcunWNsBw4cIDEx8YbzjK75+4nUu3bt4o033mD16tU3fCw6MTExx/1KlSrh5uaWr7zffPMNfn5+uLq6UrlyZcLCwm65/tGjR4G/CtDfubm5Ua1atezHRUoSFRkRB+Dm5kbTpk1p2rQp4eHhPPXUUyxevJixY8fecjtnZ+c8Lzf+drLvP2Gz2bBYLKxYseKmr1OY52rkNt6/s9lslCtXjrlz59708WtF6PLly7Rp0wY/Pz/eeustwsLC8PDwIDo6mldeeQWbzZZju78fvcmre+65J/tTSyJyZ1RkRBxMkyZNAEhISCj01zpw4MANy/bv33/LT8CEhYVhGAZVq1YlPDy8ENPdmbCwMFatWkXLli1vWT7WrFnDhQsXWLJkCffcc0/28tt9oqgwhYaGAn9dN6hatWrZyzMyMoiPj6ddu3bZy4r6assiZtE5MiJ26rfffrvpUZJr56dcP71QGJYtW8bJkyez72/evJk///yTzp0757rNo48+irOzM+PHj78hv2EYXLhwodDy5kXPnj2xWq1MmDDhhseysrKyP8Z87ejO38eQkZHBRx99VCQ5b6Zdu3a4ubkxffr0HLk+//xzEhMT6dKlS/Yyb2/vG6a/RIojHZERsVODBw8mNTWVRx55hFq1apGRkcGGDRtYuHAhVapUyfeJpXeievXqtGrVihdeeIH09HQ+/PBDypQpw6hRo3LdJiwsjLfffpvRo0dz5MgRunXrhq+vL/Hx8SxdupT/+7//Y+TIkbd97S+++IKffvrphuVDhw79R2Nq06YNAwYMYOLEicTGxtKhQwdcXV05cOAAixcvZtq0aXTv3p0WLVpQunRp+vfvz5AhQ7BYLPzvf/8rsCm4OxEYGMjo0aMZP348nTp14qGHHmLfvn189NFHNG3alMcffzx73caNG7Nw4UKGDx9O06ZN8fHxoWvXrqZlFyksKjIidmrKlCksXryY5cuX8+mnn5KRkUFISAgvvvgib7zxRqFdi+XvnnjiCZycnPjwww85e/Ysd911F1FRUVSsWPGW27366quEh4czdepUxo8fD/x1gnGHDh146KGH8vTas2bNuunygvhyzI8//pjGjRvzySef8Nprr+Hi4kKVKlV4/PHHadmyJfDXtVx++OEHRowYwRtvvEHp0qV5/PHHadu2LR07dvzHGe7UuHHjCAwMJCoqipdeeomAgAD+7//+j3fffTfHidQvvvgisbGxzJ49O/uihioyUhxZDDP/vBARu3TkyBGqVq3K5MmT83T0RETELDpHRkRERByWioyIiIg4LBUZERERcVg6R0ZEREQclo7IiIiIiMNSkRERERGHVeyvI2Oz2Th16hS+vr66ZLeIiIiDMAyD5ORkgoKCcnzb+/WKfZE5derUDd/0KyIiIo7h+PHjVK5cOdfHi32R8fX1Bf76Qfj5+ZmcRkRERPIiKSmJ4ODg7Pfx3BT7InNtOsnPz09FRkRExMHc7rQQnewrIiIiDktFRkRERByWioyIiIg4rGJ/jkxeWa1WMjMzzY4hxZibm9stP0IoIiL5V+KLjGEYnD59msuXL5sdRYo5Jycnqlatipubm9lRRESKjRJfZK6VmHLlyuHl5aWL5kmhuHZhxoSEBEJCQvR7JiJSQEp0kbFardklpkyZMmbHkWIuMDCQU6dOkZWVhaurq9lxRESKhRI9YX/tnBgvLy+Tk0hJcG1KyWq1mpxERKT4KNFF5hod5peioN8zEZGCpyIjIiIiDsvUIvP777/TtWtXgoKCsFgsLFu2LMfjhmEwZswYKlasiKenJ+3atePAgQPmhHUgN/tZyq2tWbMGi8WiT6+JiDgYU4tMSkoKDRo0YObMmTd9/L333mP69Ol8/PHH/Pnnn3h7e9OxY0fS0tKKOKl9efLJJ+nWrVuujyckJNC5c+eiC5RPFovlhlurVq2K7PXvvfdehg0blmNZixYtSEhIwN/fv8hyiIjIP2fqp5Y6d+6c6xuuYRh8+OGHvPHGGzz88MMA/Pe//6V8+fIsW7aM3r17F2VUh1KhQgWzI2AYBlarFReXm/+KzZ49m06dOmXfN/vaKm5ubnbxcxMRcSRWm8Ha/We5v1Z50zLY7Tky8fHxnD59mnbt2mUv8/f3p1mzZmzcuDHX7dLT00lKSspxK2n+PrV05MgRLBYLS5Ys4b777sPLy4sGDRrc8DNct24drVu3xtPTk+DgYIYMGUJKSkr24//73/9o0qQJvr6+VKhQgb59+3L27Nnsx69NzaxYsYLGjRvj7u7OunXrcs1YqlQpKlSokH0LCAi4Ifvf150zZ06+xrN+/XruvfdevLy8KF26NB07duTSpUs8+eSTrF27lmnTpmUfDTpy5MhNp5a++eYbIiIicHd3p0qVKrz//vs5XqNKlSq8++67PP300/j6+hISEsKnn356y30jIlJcnE1O44kv/uTpOVv5bvsp03LYbZE5ffo0AOXL52x55cuXz37sZiZOnIi/v3/2LTg4OM+vaRgGqRlZptwMw7izH1Qevf7664wcOZLY2FjCw8Pp06cPWVlZABw6dIhOnTrx2GOPsWPHDhYuXMi6desYNGhQ9vaZmZlMmDCB7du3s2zZMo4cOcKTTz55w+u8+uqrTJo0iT179lC/fn1TxhMbG0vbtm2pU6cOGzduZN26dXTt2hWr1cq0adNo3rw5zz33HAkJCSQkJNz0d2Tbtm307NmT3r17ExcXx7hx43jzzTezC9U177//Pk2aNCEmJoYXX3yRF154gX379hXauEVE7MH6g+d5YNo61h+8gKerc6G/h91Ksbsg3ujRoxk+fHj2/aSkpDyXmauZVuqMWVlY0W5p91sd8XIrvN0xcuRIunTpAsD48eOJiIjg4MGD1KpVi4kTJ9KvX7/s80Zq1KjB9OnTadOmDbNmzcLDw4Onn346+7mqVavG9OnTadq0KVeuXMHHxyf7sbfeeov27dvfNk+fPn1wdnbOvv/VV1/d8ryf/Iznvffeo0mTJnz00UfZ60dERGT/t5ubG15eXrecSvrggw9o27Ytb775JgDh4eHs3r2byZMn5yhwDzzwAC+++CIAr7zyClOnTuW3336jZs2aeR6LiIijsNoMpv16gBmrD2AYULO8L1F9I6lR3te0THZ7RObam8yZM2dyLD9z5swt34Dc3d3x8/PLcRNyHB2pWLEiQPbU0Pbt25kzZw4+Pj7Zt44dO2Kz2YiPjwf+OkLRtWtXQkJC8PX1pU2bNgAcO3Ysx+s0adIkT3mmTp1KbGxs9i0v5Sev47l2ROaf2LNnDy1btsyxrGXLlhw4cCDHBe3+nsNisVChQoUcU24iIsXFmaQ0+n62iem//lViejcNZtnAlqaWGLDjIzJVq1alQoUK/PrrrzRs2BD46+jKn3/+yQsvvFAor+np6szutzoWynPn5bUL098viX/twmw2mw2AK1euMGDAAIYMGXLDdiEhIaSkpNCxY0c6duzI3LlzCQwM5NixY3Ts2JGMjIwc63t7e+cpT4UKFahevfoNyy0Wyw2HKG/2reS3Go+np2eeMhSE679qwGKxZOcQESku1u4/x0sLY7mYkoG3mzPvPlqPhxtWMjsWYHKRuXLlCgcPHsy+Hx8fT2xsLAEBAYSEhDBs2DDefvttatSoQdWqVXnzzTcJCgrK1xREflgslkKd3rFXjRo1Yvfu3TctFgBxcXFcuHCBSZMmZU/Tbd26tVCyBAYGkpCQkH3/wIEDpKam5us56tevz6+//sr48eNv+ribm9ttvyagdu3arF+/Psey9evXEx4enmNKTESkOMuy2nj/l/3MWnMIgNoV/ZjZN5JqgT632bLomPquvXXrVu67777s+9fObenfvz9z5sxh1KhRpKSk8H//939cvnyZVq1a8dNPP+Hh4WFWZLuRmJhIbGxsjmVlypTJ18nN17zyyivcfffdDBo0iGeffRZvb292797NL7/8QlRUFCEhIbi5uTFjxgyef/55du7cyYQJEwpoJDndf//9REVF0bx5c6xWK6+88kq+v2Bx9OjR1KtXjxdffJHnn38eNzc3fvvtN3r06EHZsmWpUqUKf/75J0eOHMHHxyf7E1N/N2LECJo2bcqECRPo1asXGzduJCoqKsd5NyIixdmpy1cZMj+GrUcvAfD43SG80aUOHoU8g5Bfpp4jc++992IYxg23a58MsVgsvPXWW5w+fZq0tDRWrVpFeHi4mZHtxpo1a4iMjMxxy+0IxO3Ur1+ftWvXsn//flq3bk1kZCRjxowhKCgI+OsoyZw5c1i8eDF16tRh0qRJTJkypSCHk+39998nODiY1q1b07dvX0aOHJnvL/UMDw/n559/Zvv27dx11100b96cb7/9NvuaNiNHjsTZ2Zk6depkT5Ndr1GjRixatIgFCxZQt25dxowZw1tvvXXTT2qJiBQ3q/ee4YHpf7D16CV83F2I6hvJ293q2V2JAbAYZn5mqggkJSXh7+9PYmLiDSf+pqWlER8fT9WqVXWURwqdft9ExN5lWm2899NePvvjrw961KvkT1TfSELL5O38x4J0q/fvvyt5J4SIiIjIDY5fTGXw/Bhij18G4MkWVRj9QC3cXezvKMzfqciIiIiUcCt3neblxdtJSsvCz8OF97o3oFNdx/jaFhUZERGREio9y8qkFXuZvf4IAA2CSxHVJ5LggPydm2gmFRkREZES6OiFFAbNiyHuZCIAz7Wuyssda+HmYrfXyr0pFRkw9TsipOTQ75mI2IsfdyTw6jc7SE7PopSXK1O6N6BdHfO+wfqfKNFF5tr1SVJTU4v0arBSMl27CrIuqCciZknLtPL2j7v5atNfl51oHFqaGX0iCSrluO+BJbrIODs7U6pUqezvxvHy8sq+3L1IQbLZbJw7dw4vL6/s69mIiBSl+PMpDJwbze6EJABeuDeM4e3DcXV2rKmk65X4f1GvfQGlvuhPCpuTkxMhISEqyyJS5L6NPclrS+JIybAS4O3GBz0bcG/NcmbHKhAlvshYLBYqVqxIuXLlbvrlhCIFxc3NDScnx/7LR0QcS1qmlXHf7WLBluMA3FU1gOm9I6ngX3wuylnii8w1zs7OOndBRESKjYNnkxk4N4Z9Z5KxWGDwfdUZ0rYGLg4+lXQ9FRkREZFi5pttJ3hj2U6uZlop6+POh70a0qpGWbNjFQoVGRERkWIiNSOLMd/u4uttJwBoEVaGD3s3pJxv8ZlKup6KjIiISDGw73QyA+dFc/DsFZwsMKxdOAPvq46zU/H+gIGKjIiIiAMzDINFW48z5ttdpGfZKOfrzrTekTQPK2N2tCKhIiMiIuKgrqRn8cbSOJbFngKgdY2yTO3VkLI+7iYnKzoqMiIiIg5o96kkBs2L5vD5FJydLIzoEM7z94ThVMynkq6nIiMiIuJADMNg7p/HeOuH3WRk2ajo78H0PpE0rRJgdjRTqMiIiIg4iKS0TEYviePHHQkA3F+rHFN6NCDA283kZOZRkREREXEAcScSGTQ/mqMXUnFxsvBKp1o806pqiZtKup6KjIiIiB0zDIMvNxzh3eV7ybDaqFTKkxl9I2kUUtrsaHZBRUZERMROJaZmMuqb7azcdQaADnXKM7l7A/y9XE1OZj9UZEREROxQzLFLDJ4fw4lLV3F1tvDaA7V5skUVLJaSPZV0PRUZERERO2IYBp+vi2fSir1k2QxCAryI6htJ/cqlzI5ml1RkRERE7MSllAxGLt7Or3vPAvBAvQpMeqw+fh6aSsqNioyIiIgd2Hb0IoPnxXAqMQ03FyfefLAOjzcL0VTSbajIiIiImMhmM/jk98NM+XkfVptB1bLeRPWNJCLI3+xoDkFFRkRExCQXrqQzfNF21u4/B8BDDYJ499F6+Ljr7Tmv9JMSERExwZ+HLzBkQQxnktJxd3Fi/EMR9GoarKmkfFKRERERKUJWm8FHvx1k6qr92AwIC/RmZr9G1KrgZ3Y0h6QiIyIiUkTOJafz0sJY1h08D8CjjSox4eG6eGsq6Y45mR3gdpKTkxk2bBihoaF4enrSokULtmzZYnYsERGRfFl/8Dydp/3BuoPn8XR1ZnL3+nzQs6FKzD9k9z+9Z599lp07d/K///2PoKAgvvrqK9q1a8fu3bupVKmS2fFERERuyWozmPbrAWasPoBhQHh5H2b2bUSN8r5mRysWLIZhGGaHyM3Vq1fx9fXl22+/pUuXLtnLGzduTOfOnXn77bdv+xxJSUn4+/uTmJiIn5/mH0VEpOicSUpj6IIYNh2+CECvJsGMeygCTzdnk5PZv7y+f9v1EZmsrCysViseHh45lnt6erJu3bqbbpOenk56enr2/aSkpELNKCIicjNr959j+MJYLqRk4OXmzLuP1KNbpGYSCppdnyPj6+tL8+bNmTBhAqdOncJqtfLVV1+xceNGEhISbrrNxIkT8ff3z74FBwcXcWoRESnJsqw23vtpL/2/2MyFlAxqV/Tjh8GtVGIKiV1PLQEcOnSIp59+mt9//x1nZ2caNWpEeHg427ZtY8+ePTesf7MjMsHBwZpaEhGRQpeQeJUh82PYcuQSAP2ahfDmg3XwcNVUUn4Vi6klgLCwMNauXUtKSgpJSUlUrFiRXr16Ua1atZuu7+7ujru7exGnFBGRkm713jOMWLSdS6mZ+Li7MOmxejxYP8jsWMWe3ReZa7y9vfH29ubSpUusXLmS9957z+xIIiIiZFptTF65j09/PwxA3Up+zOzbiNAy3iYnKxnsvsisXLkSwzCoWbMmBw8e5OWXX6ZWrVo89dRTZkcTEZES7sSlVAbPjyHm2GUAnmxRhdEP1MLdRVNJRcXui0xiYiKjR4/mxIkTBAQE8Nhjj/HOO+/g6upqdjQRESnBVu46zcuLt5OUloWvhwuTu9enU92KZscqcez+ZN9/SteRERGRgpSRZWPiij3MXn8EgAbBpYjqE0lwgJe5wYqZYnOyr4iIiL04diGVQfOj2XEiEYBnW1VlVKdauLnY9dVMijUVGRERkTxYHpfAK1/vIDk9C39PV97v0YB2dcqbHavEU5ERERG5hbRMK+/8uIf/bToKQOPQ0kzvE0mlUp4mJxNQkREREclV/PkUBs6NZnfCX19383ybMEZ0CMfVWVNJ9kJFRkRE5Ca+jT3Ja0viSMmwEuDtxvs9G3BfzXJmx5LrqMiIiIj8TVqmlfHf72L+5uMA3FUlgOl9Iqng73GbLcUMKjIiIiL/z8GzVxg0L5q9p5OxWGDQfdUZ2rYGLppKslsqMiIiIsA3207wxrKdXM20UtbHjQ97RdKqRlmzY8ltqMiIiEiJlpqRxZhvd/H1thMAtAgrw4e9GlLOT1NJjkBFRkRESqz9Z5IZODeaA2ev4GSBoW3DGXR/dZydLGZHkzxSkRERkRLHMAwWbT3O2O92kZZpo5yvO9N6R9I8rIzZ0SSfVGRERKREuZKexRtL41gWewqA1jXKMrVXQ8r6uJucTO6EioyIiJQYu08lMWheNIfPp+DsZGF4+3BeaBOGk6aSHJaKjIiIFHuGYTBv8zHGf7+bjCwbFfw8mNE3kqZVAsyOJv+QioyIiBRryWmZvLokjh93JABwX81A3u/ZkABvN5OTSUFQkRERkWJr58lEBs6L5uiFVFycLIzqVJNnW1XTVFIxoiIjIiLFjmEYfLnhCO8u30uG1UalUp7M6BtJo5DSZkeTAqYiIyIixUri1Uxe+XoHP+06DUD7OuWZ3L0+pbw0lVQcqciIiEixEXv8MoPmRXPi0lVcnS2M7lybp1pWwWLRVFJxpSIjIiIOzzAMPl8Xz6QVe8myGQQHeBLVpxENgkuZHU0KmYqMiIg4tMupGYxcvJ1Ve84C0LluBSY9Vh9/T1eTk0lRUJERERGHte3oRQbPi+FUYhpuzk68+WBtHr87VFNJJYiKjIiIOBybzeDTPw4zeeU+rDaDKmW8iOrbiLqV/M2OJkVMRUZERBzKhSvpjFi8nTX7zgHQtUEQ7z5SF18PTSWVRCoyIiLiMP48fIEhC2I4k5SOu4sT4x6KoHfTYE0llWAqMiIiYvesNoOPfjvI1FX7sRlQLdCbmX0bUbuin9nRxGQqMiIiYtfOJafz0sJY1h08D8CjkZWY0K0u3u56CxMVGRERsWMbDp5n6MJYziWn4+HqxISH69KjSbDZscSOqMiIiIjdsdoMpv16gBmrD2AYEF7eh5l9G1GjvK/Z0cTOqMiIiIhdOZOUxtAFMWw6fBGAnk0qM/6huni6OZucTOyRioyIiNiN3/ef46WFsVxIycDLzZl3HqnLI5GVzY4ldszJ7AC3YrVaefPNN6latSqenp6EhYUxYcIEDMMwO5qIiBSgLKuNySv30n/2Zi6kZFCrgi/fD26lEiO3ZddHZP79738za9YsvvzySyIiIti6dStPPfUU/v7+DBkyxOx4IiJSABISrzJkfgxbjlwCoG+zEMY8WAcPV00lye3ZdZHZsGEDDz/8MF26dAGgSpUqzJ8/n82bN5ucTERECsJve88yfFEsl1Iz8XF3YeKj9ejaIMjsWOJA7HpqqUWLFvz666/s378fgO3bt7Nu3To6d+6c6zbp6ekkJSXluImIiH3JtNqYuHwPT83ZwqXUTOpW8uOHwa1UYiTf7PqIzKuvvkpSUhK1atXC2dkZq9XKO++8Q79+/XLdZuLEiYwfP74IU4qISH6cuJTK4PkxxBy7DED/5qG81qU27i6aSpL8s+sis2jRIubOncu8efOIiIggNjaWYcOGERQURP/+/W+6zejRoxk+fHj2/aSkJIKDdfEkERF78POu07z89Q4Sr2bi6+HC5O716VS3otmxxIFZDDv+CFBwcDCvvvoqAwcOzF729ttv89VXX7F37948PUdSUhL+/v4kJibi56fv5BARMUNGlo2JK/Ywe/0RABpU9ieqbyOCA7zMDSZ2K6/v33Z9RCY1NRUnp5yn8Tg7O2Oz2UxKJCIi+XXsQiqD5kez40QiAM+0qsornWrh5mLXp2mKg7DrItO1a1feeecdQkJCiIiIICYmhg8++ICnn37a7GgiIpIHK+ISGPX1DpLTs/D3dGVKjwa0r1Pe7FhSjNj11FJycjJvvvkmS5cu5ezZswQFBdGnTx/GjBmDm5tbnp5DU0siIkUvLdPKu8v38N+NRwFoFFKKGX0bUamUp8nJxFHk9f3brotMQVCREREpWvHnUxg0L5pdp/66/MWANtUY2aEmrs6aSpK8KxbnyIiIiGP5bvspXlsSx5X0LAK83Xi/ZwPuq1nO7FhSjKnIiIjIP5aWaWX897uZv/kYAHdVCWB6n0gq+HuYnEyKOxUZERH5Rw6evcKgedHsPZ2MxQKD7qvO0LY1cNFUkhQBFRkREbljS6JP8MaynaRmWCnr48bUXg1pXSPQ7FhSgqjIiIhIvqVmZDH2210s3nYCgObVyjCtd0PK+WkqSYqWioyIiOTL/jPJDJwbzYGzV7BYYGjbGgy+vwbOThazo0kJpCIjIiJ5YhgGi7eeYMx3O0nLtBHo68603g1pEVbW7GhSgqnIiIjIbaWkZ/HGsp0sjTkJQOsaZZnaqyFlfdxNTiYlnYqMiIjc0p6EJAbOjebw+RScLDCiQ01eaBOGk6aSxA6oyIiIyE0ZhsG8zccY//1uMrJsVPDzYHqfSO6qGmB2NJFsKjIiInKD5LRMRi+J44cdCQDcVzOQ93s2JMA7b99zJ1JUVGRERCSHnScTGTQvmiMXUnFxsvByx5o817qappLELqnIiIgI8NdU0n83HuWdH/eQYbVRqZQn0/tE0ji0tNnRRHKlIiMiIiRezeTVb3awYudpANrVLs+UHvUp5aWpJLFvKjIiIiVc7PHLDJoXzYlLV3F1tvBq59o83bIKFoumksT+qciIiJRQhmHw+bp4/v3TXjKtBsEBnkT1aUSD4FJmRxPJMxUZEZES6HJqBiMX72DVnjMAdK5bgUmP1cff09XkZCL5oyIjIlLCbDt6icHzojmVmIabsxNvPFibf90dqqkkcUgqMiIiJYTNZvDpH4eZvHIfVptBlTJeRPVtRN1K/mZHE7ljKjIiIiXAxZQMhi+KZc2+cwB0bRDEu4/UxddDU0ni2FRkRESKuc3xFxkyP4bTSWm4uzgxtmsEfe4K1lSSFAsqMiIixZTNZvDRmoN88Mt+bAZUC/RmZt9G1K7oZ3Y0kQKjIiMiUgydS05n+KJY/jhwHoBHIysxoVtdvN31z74UL/qNFhEpZjYcPM/QhbGcS07Hw9WJtx6uS4/GlTWVJMWSioyISDFhtRlM//UA01cfwDCgRjkfZvZrRHh5X7OjiRQaFRkRkWLgbFIaQxfEsvHwBQB6NqnM+Ifq4unmbHIykcKlIiMi4uD+OHCOlxbGcv5KBl5uzrzzSF0eiaxsdiyRIqEiIyLioLKsNj5cdYCZaw5iGFCrgi9RfRtRvZyP2dFEioyKjIiIA0pIvMrQ+bFsPnIRgL7NQhjzYB08XDWVJCWLioyIiIP5be9Zhi+K5VJqJj7uLrz7aD0eahBkdiwRU6jIiIg4iEyrjSkr9/HJ74cBiAjyY2bfRlQp621yMhHzOJkd4HaqVKmCxWK54TZw4ECzo4mIFJmTl6/S65ON2SWmf/NQvnmhhUqMlHh2f0Rmy5YtWK3W7Ps7d+6kffv29OjRw8RUIiJF55fdZxi5eDuJVzPx9XDhvcfq07leRbNjidgFuy8ygYGBOe5PmjSJsLAw2rRpY1IiEZGikZFl498/7eXzdfEANKjsT1TfRgQHeJmcTMR+2H2R+buMjAy++uorhg8fnuulttPT00lPT8++n5SUVFTxREQKzPGLqQyaF832E4kAPNOqKq90qoWbi92fESBSpByqyCxbtozLly/z5JNP5rrOxIkTGT9+fNGFEhEpYD/tTODlr3eQnJaFv6crU3o0oH2d8mbHErFLFsMwDLND5FXHjh1xc3Pj+++/z3Wdmx2RCQ4OJjExET8/fXW9iNivtEwrE5fv4cuNRwFoFFKK6X0iqVxaU0lS8iQlJeHv73/b92+HOSJz9OhRVq1axZIlS265nru7O+7u7kWUSkSkYBw5n8LAedHsOvXXdPiANtUY2aEmrs6aShK5FYcpMrNnz6ZcuXJ06dLF7CgiIgXq++2nGL0kjivpWZT2cuWDng25r1Y5s2OJOASHKDI2m43Zs2fTv39/XFwcIrKIyG2lZVp564fdzPvzGABNq5Rmep9IKvp7mpxMxHE4RCtYtWoVx44d4+mnnzY7iohIgTh07goD50az93QyFgsMvLc6w9rVwEVTSSL54hBFpkOHDjjQOckiIre0NOYEry/dSWqGlbI+bkzt1ZDWNQJvv6GI3MAhioyISHFwNcPK2O92smjrCQCaVyvDtN4NKefnYXIyEcelIiMiUgQOnEnmxbnRHDh7BYsFhratweD7a+DsdPOLe4pI3qjIiIgUIsMwWLztBGO+3Ulapo1AX3em9W5Ii7CyZkcTKRZUZERECklKehZvLtvJkpiTALSuUZYPejYk0FfXuhIpKCoyIiKFYE9CEgPnRXP4XApOFhjRoSYvtAnDSVNJIgVKRUZEpAAZhsH8zccZ//0u0rNsVPDzYHqfSO6qGmB2NJFiSUVGRKSAJKdl8trSnXy//RQA99YM5IOeDQnwdjM5mUjxpSIjIlIAdp5MZNC8aI5cSMXZycKojjV5rnU1TSWJFDIVGRGRf8AwDP636Shv/7CHDKuNSqU8md4nksahpc2OJlIiqMiIiNyhxKuZjF6yg+VxpwFoV7s8U3rUp5SXppJEioqKjIjIHdh+/DKD5kdz/OJVXJ0tvNq5Nk+3rILFoqkkkaKkIiMikg+GYfDF+iNMWrGHTKtB5dKezOzbiAbBpcyOJlIiqciIiOTR5dQMXv56B7/sPgNAp4gK/Lt7ffw9XU1OJlJyqciIiORB9LFLDJ4Xw8nLV3FzduKNB2vzr7tDNZUkYjIVGRGRW7DZDD774zCTV+4jy2YQWsaLmX0bUbeSv9nRRAQVGRGRXF1MyWDEolh+23cOgAfrV2Tio/Xw9dBUkoi9UJEREbmJzfEXGTI/htNJabi5ODGuawR97grWVJKInXHK7wY//fQT69aty74/c+ZMGjZsSN++fbl06VKBhhMRKWo2m8HM3w7S57NNnE5Ko1qgN98ObEnfZiEqMSJ2KN9F5uWXXyYpKQmAuLg4RowYwQMPPEB8fDzDhw8v8IAiIkXl/JV0+s/ezOSV+7DaDB6JrMT3g1pRu6Kf2dFEJBf5nlqKj4+nTp06AHzzzTc8+OCDvPvuu0RHR/PAAw8UeEARkaKw8dAFhi6I4WxyOh6uTrz1cF16NK6sozAidi7fRcbNzY3U1FQAVq1axRNPPAFAQEBA9pEaERFHYbUZzFh9gOm/HsBmQI1yPszs14jw8r5mRxORPMh3kWnVqhXDhw+nZcuWbN68mYULFwKwf/9+KleuXOABRUQKy9nkNIYtiGXDoQsA9GhcmfEPR+Dlps9BiDiKfJ8jExUVhYuLC19//TWzZs2iUqVKAKxYsYJOnToVeEARkcKw7sB5Hpj2BxsOXcDLzZkPejZgco8GKjEiDsZiGIZhdojClJSUhL+/P4mJifj56YQ9kZIuy2rjw1UHmLnmIIYBtSr4EtW3EdXL+ZgdTUT+Jq/v33n60yMpKSn7SW53HozKgojYq9OJaQyZH8PmIxcB6NsshDEP1sHD1dnkZCJyp/JUZEqXLk1CQgLlypWjVKlSNz2L3zAMLBYLVqu1wEOKiPxTv+07y4hF27mYkoGPuwvvPlqPhxoEmR1LRP6hPBWZ1atXExAQkP3f+jiiiDiKTKuNKT/v45O1hwGICPIjqm8jqpb1NjmZiBQEnSMjIsXWyctXGTI/hm1H/7rq+BPNQ3ntgdqaShJxAHl9/873p5bGjRuHzWa7YXliYiJ9+vTJ79OJiBSKVbvP8MC0P9h29BK+Hi7M6teItx6uqxIjUszku8h8/vnntGrVisOHD2cvW7NmDfXq1ePQoUMFGk5EJL8ysmy8/cNunv3vVhKvZtKgsj8/Dm5N53oVzY4mIoUg30Vmx44dVK5cmYYNG/LZZ5/x8ssv06FDB/71r3+xYcOGwsgoIpInxy+m0uOTjfxnXTwAT7esyuLnWxBSxsvkZCJSWPJ95afSpUuzaNEiXnvtNQYMGICLiwsrVqygbdu2hZGPkydP8sorr7BixQpSU1OpXr06s2fPpkmTJoXyeiLimH7amcDLX+8gOS0LPw8XpvRoQIeICmbHEpFClu8jMgAzZsxg2rRp9OnTh2rVqjFkyBC2b99e0Nm4dOkSLVu2xNXVlRUrVrB7927ef/99SpcuXeCvJSKOKT3Lythvd/L8V9Ekp2URGVKK5UNbq8SIlBD5PiLTqVMntm7dypdffkn37t25evUqw4cP5+6772b8+PGMGjWqwML9+9//Jjg4mNmzZ2cvq1q1aoE9v4g4tiPnUxg0P5qdJ/+6UOeANtUY2aEmrs539DeaiDigfP/fbrVa2bFjB927dwfA09OTWbNm8fXXXzN16tQCDffdd9/RpEkTevToQbly5YiMjOSzzz675Tbp6ekkJSXluIlI8fPDjlM8OGMdO08mUdrLldlPNmV059oqMSIlTIFeR+b8+fOULVu2oJ4ODw8PAIYPH06PHj3YsmULQ4cO5eOPP6Z///433WbcuHGMHz/+huW6joxI8ZCWaeWtH3Yz789jADStUprpfSKp6O9pcjIRKUh5vY6MXV8Qz83NjSZNmuT4NNSQIUPYsmULGzduvOk26enppKenZ99PSkoiODhYRUakGDh07goD50az93QyFgu8eG8YL7ULx0VHYUSKnQL90si/s1qtTJ06lUWLFnHs2DEyMjJyPH7x4sX8p81FxYoVqVOnTo5ltWvX5ptvvsl1G3d3d9zd3Qssg4jYh2UxJ3ltaRypGVbKeLsxtVdD7gkPNDuWiJgs33/GjB8/ng8++IBevXqRmJjI8OHDefTRR3FycmLcuHEFGq5ly5bs27cvx7L9+/cTGhpaoK8jIvbraoaVV77ewbCFsaRmWLm7WgArhrZWiRER4A6mlsLCwpg+fTpdunTB19eX2NjY7GWbNm1i3rx5BRZuy5YttGjRgvHjx9OzZ082b97Mc889x6effkq/fv3y9Bz6riURx3XgTDID50Wz/8wVLBYYcn8NhrStgbOTvrhWpLgrtO9aOn36NPXq1QPAx8eHxMREAB588EF+/PHHO4x7c02bNmXp0qXMnz+funXrMmHCBD788MM8lxgRcVyLtx7noaj17D9zhUBfd+Y+04yX2oerxIhIDvk+R6Zy5cokJCQQEhJCWFgYP//8M40aNWLLli2Fcm7Kgw8+yIMPPljgzysi9iklPYs3v93JkuiTALSqXpapvRoS6Ktz30TkRvkuMo888gi//vorzZo1Y/DgwTz++ON8/vnnHDt2jJdeeqkwMopICbH3dBID50Zz6FwKThYY3j6cF++tjpOOwohILv7xx683btzIxo0bqVGjBl27di2oXAVG58iI2D/DMFiw5TjjvttFepaN8n7uTO8dSbNqZcyOJiImKbSPX1+vefPmNG/e/J8+jYiUUFfSs3htSRzfbT8FwL01A3m/RwPK+GgqSURu7x8VGT8/P2JjY6lWrVpB5RGREmTnyUQGzYvmyIVUnJ0svNyxJv/XupqmkkQkz/JcZE6dOkVQUFCOZXZ8UWARsWOGYfDVpqNM+GEPGVYbQf4ezOgbSePQALOjiYiDyfPHryMiIgr0GjEiUjIlpWUycF40b367iwyrjXa1y7F8aGuVGBG5I3kuMu+88w4DBgygR48e2V9D8Pjjj+sEWhHJsx0nLtNl+h8sjzuNq7OFN7rU5rMnmlDKy83saCLioPL1qaX4+HieeeYZdu/ezWeffWaXn1K6nj61JGI+wzCYvf4IE1fsIdNqULm0J1F9G9EwuJTZ0UTEThXKp5aqVq3K6tWriYqK4tFHH6V27dq4uOR8iujo6DtLLCLFUmJqJi9/vZ2fd58BoFNEBf7dvT7+nq4mJxOR4iDfn1o6evQoS5YsoXTp0jz88MM3FBkRkWuij11i8LwYTl6+ipuzE693qc0TzUOxWPSpJBEpGPlqIZ999hkjRoygXbt27Nq1i8BAffusiNzIZjP4z7rDvPfTPrJsBqFlvIjq04h6lf3NjiYixUyei0ynTp3YvHkzUVFRPPHEE4WZSUQc2KWUDEYs3s7qvWcBeLB+RSY+Wg9fD00liUjBy3ORsVqt7Nixg8qVKxdmHhFxYFuOXGTI/BgSEtNwc3FibNc69L0rRFNJIlJo8lxkfvnll8LMISIOzGYzmLX2EB/8sh+rzaBaWW+i+jaiTpA+KSgihUtn6orIP3L+SjovLYzljwPnAXgkshJvd6uLt7v+eRGRwqd/aUTkjm08dIGhC2I4m5yOh6sTbz1Ulx5NKmsqSUSKjIqMiOSb1WYQtfog037dj82A6uV8+KhfI8LL+5odTURKGBUZEcmXs8lpDFsQy4ZDFwDo0bgy4x+OwMtN/5yISNHTvzwikmfrDpxn2MJYzl9Jx9PVmXceqcujjfRJRhExj4qMiNxWltXGtF8PEPXbQQwDalXwJapvI6qX8zE7moiUcCoyInJLpxPTGLIghs3xf33rfZ+7ghnbNQIPV2eTk4mIqMiIyC2s2XeW4Yu2czElA283Z959tB4PN6xkdiwRkWwqMiJyg0yrjQ9+2c+sNYcAqFPRj5n9GlG1rLfJyUREclKREZEcTl2+yuD5MWw7egmAJ5qH8toDtTWVJCJ2SUVGRLKt2n2GkV9v53JqJr7uLvy7e30eqFfR7FgiIrlSkRERMrJsvPfTXv6zLh6A+pX9ierTiJAyXiYnExG5NRUZkRLu+MVUBs2PYfvxywA83bIqr3SuibuLppJExP6pyIiUYD/tPM2or7eTlJaFn4cLU3o0oENEBbNjiYjkmYqMSAmUnmVl4vK9zNlwBIDIkFLM6BNJ5dKaShIRx6IiI1LCHL2QwqB5McSdTATg/+6pxssda+Lq7GRyMhGR/FORESlBftyRwKvf7CA5PYvSXq6837MB99cqb3YsEZE7Ztd/go0bNw6LxZLjVqtWLbNjiTictEwrbyyLY+C8aJLTs2hapTTLh7ZWiRERh2f3R2QiIiJYtWpV9n0XF7uPLGJXDp+7wsB5MexJSALgxXvDGN4+HBdNJYlIMWD3rcDFxYUKFfQpCpE78W3sSV5bEkdKhpUy3m580KshbcIDzY4lIlJg7L7IHDhwgKCgIDw8PGjevDkTJ04kJCQk1/XT09NJT0/Pvp+UlFQUMUXsytUMK+O+28XCrccBuLtaANN6R1Lez8PkZCIiBcuujy03a9aMOXPm8NNPPzFr1izi4+Np3bo1ycnJuW4zceJE/P39s2/BwcFFmFjEfAfPJtNt5noWbj2OxQJD2tZg7rN3q8SISLFkMQzDMDtEXl2+fJnQ0FA++OADnnnmmZuuc7MjMsHBwSQmJuLn51dUUUVM8fW2E7y5bCdXM62U9XFneu+GtKhe1uxYIiL5lpSUhL+//23fv+1+aunvSpUqRXh4OAcPHsx1HXd3d9zd3YswlYj5UjOyeGPZTpZEnwSgVfWyTO3VkEBf/b8gIsWbXU8tXe/KlSscOnSIihX1bbwi1+w9nUTXGetYEn0SJwuMaB/Ol0/fpRIjIiWCXR+RGTlyJF27diU0NJRTp04xduxYnJ2d6dOnj9nRRExnGAYLtxxn7He7SM+yUd7PnWm9I7m7Whmzo4mIFBm7LjInTpygT58+XLhwgcDAQFq1asWmTZsIDNTHR6Vku5KexetL4/g29hQAbcID+aBnA8r46CiMiJQsdl1kFixYYHYEEbuz61Qig+bFEH8+BWcnCyM71GTAPdVwcrKYHU1EpMjZdZERkf+fYRh89ecxJvywm4wsG0H+HszoG0nj0ACzo4mImEZFRsQBJKVlMvqbOH6MSwCgXe1yTO7egNLebiYnExExl4qMiJ3bceIyg+bFcOxiKi5OFl7tXItnWlXFYtFUkoiIioyInTIMgzkbjvDu8j1kWg0qlfIkqm8kkSGlzY4mImI3VGRE7FBiaiYvf72dn3efAaBjRHnee6wB/l6uJicTEbEvKjIidibm2CUGzYvh5OWruDk78doDtejfooqmkkREbkJFRsROGIbBf/6I598/7SXLZhAS4MXMvo2oV9nf7GgiInZLRUbEDlxKyWDk4u38uvcsAF3qV2Tio/Xw89BUkojIrajIiJhs65GLDJ4fQ0JiGm4uTox5sA79moVoKklEJA9UZERMYrMZfPz7Id7/eT9Wm0HVst5E9Y0kIkhTSSIieaUiI2KCC1fSGb5oO2v3nwPg4YZBvPNIPXzc9b+kiEh+6F9NkSK26fAFhi6I4UxSOh6uTox/KIKeTYI1lSQicgdUZESKiNVmMPO3g3y4aj82A6qX82Fm30bUrOBrdjQREYelIiNSBM4mp/HSwljWH7wAQPfGlXnr4Qi83PS/oIjIP6F/RUUK2fqD5xm6IJbzV9LxdHXm7W51eaxxZbNjiYgUCyoyIoXEajOYtmo/M347iGFAzfK+zOwXSfVymkoSESkoKjIiheBMUhpD5sfwZ/xFAPrcFczYrhF4uDqbnExEpHhRkREpYGv3n+OlhbFcTMnA282Zdx+tx8MNK5kdS0SkWFKRESkgWVYb7/+yn1lrDgFQu6IfM/tGUi3Qx+RkIiLFl4qMSAE4dfkqQ+bHsPXoJQD+dXcor3eprakkEZFCpiIj8g+t3nuG4Yu2czk1E193FyY9Vp8u9SuaHUtEpERQkRG5Q5lWG+/9tJfP/ogHoF4lf6L6RhJaxtvkZCIiJYeKjMgdOH4xlcHzY4g9fhmAp1pW4dXOtXB30VSSiEhRUpERyaeVu07z8uLtJKVl4efhwuQeDegYUcHsWCIiJZKKjEgepWdZmbh8L3M2HAGgYXApZvSJJDjAy9xgIiIlmIqMSB4cvZDCoHkxxJ1MBOC51lV5uWMt3FycTE4mIlKyqciI3MaPOxJ49ZsdJKdnUcrLlfd7NKBt7fJmxxIREVRkRHKVlmnl7R9389WmYwA0CS3N9D6RBJXyNDmZiIhcoyIjchPx51MYODea3QlJALx4bxgvtQ/H1VlTSSIi9kRFRuQ638ae5LUlcaRkWAnwdmNqr4a0CQ80O5aIiNyEiozI/5OWaWXcd7tYsOU4AM2qBjC9TyTl/TxMTiYiIrlxqOPkkyZNwmKxMGzYMLOjSDFz8GwyD0etZ8GW41gsMKRtDeY+20wlRkTEzjnMEZktW7bwySefUL9+fbOjSDHzzbYTvLFsJ1czrZT1cWda74a0rF7W7FgiIpIHDnFE5sqVK/Tr14/PPvuM0qVLmx1HionUjCxGLt7OiMXbuZpppWX1Miwf2kolRkTEgThEkRk4cCBdunShXbt2t103PT2dpKSkHDeR6+07ncxDUev5etsJnCwwvH04/326GeV8NZUkIuJI7H5qacGCBURHR7Nly5Y8rT9x4kTGjx9fyKnEURmGwaKtxxnz7S7Ss2yU83Vnep9I7q5WxuxoIiJyB+z6iMzx48cZOnQoc+fOxcMjb38pjx49msTExOzb8ePHCzmlOIor6Vm8tDCWV76JIz3Lxj3hgawY2lolRkTEgVkMwzDMDpGbZcuW8cgjj+Ds7Jy9zGq1YrFYcHJyIj09PcdjN5OUlIS/vz+JiYn4+fkVdmSxU7tPJTFoXjSHz6fg7GRhRIdwnr8nDCcni9nRRETkJvL6/m3XU0tt27YlLi4ux7KnnnqKWrVq8corr9y2xIgYhsHcP4/x1g+7yciyUdHfgxl9ImlSJcDsaCIiUgDsusj4+vpSt27dHMu8vb0pU6bMDctFrpeUlsnoJXH8uCMBgLa1yjGlRwNKe7uZnExERAqKXRcZkTsVdyKRQfOjOXohFRcnC690qsWzratisWgqSUSkOHG4IrNmzRqzI4gdMwyDLzcc4d3le8mw2qhUypMZfSNpFKLrD4mIFEcOV2REcpOYmsmob7azctcZADrUKc/k7g3w93I1OZmIiBQWFRkpFmKPX2bQvGhOXLqKq7OF1x6ozZMtqmgqSUSkmFOREYdmGAafr4tn0oq9ZNkMQgK8iOobSf3KpcyOJiIiRUBFRhzW5dQMRi7ezqo9ZwHoUq8iEx+rh5+HppJEREoKFRlxSNuOXmTwvBhOJabh5uLEmw/W4fFmIZpKEhEpYVRkxKHYbAaf/H6YKT/vw2ozqFrWm6i+kUQE+ZsdTURETKAiIw7jwpV0hi/aztr95wB4uGEQ7zxSDx93/RqLiJRUegcQh/Dn4QsMWRDDmaR03F2cGP9QBL2aBmsqSUSkhFOREbtmtRl89NtBpq7aj82AsEBvZvZrRK0K+gJQERFRkRE7di45nZcWxrLu4HkAHmtUmQndIvBy06+tiIj8Re8IYpc2HDzPkAWxnL+SjqerMxO61aV748pmxxIRETujIiN2xWozmPbrAWasPoBhQM3yvkT1jaRGeV+zo4mIiB1SkRG7cSYpjaELYth0+CIAvZsGM7ZrBJ5uziYnExERe6UiI3bh9/3neGlhLBdSMvB2c+bdR+vxcMNKZscSERE7pyIjpsqy2vjgl/18tOYQALUr+jGzbyTVAn1MTiYiIo5ARUZMk5B4lSHzY9hy5BIAj98dwhtd6uDhqqkkERHJGxUZMcVve88yfFEsl1Iz8XF3YdJj9XiwfpDZsURExMGoyEiRyrTamLJyH5/8fhiAepX8ieobSWgZb5OTiYiII1KRkSJz4lIqg+fHEHPsMgBPtqjC6Adq4e6iqSQREbkzKjJSJH7edZqRi7eTlJaFn4cL73VvQKe6FcyOJSIiDk5FRgpVRpaNiSv2MHv9EQAaBJciqk8kwQFe5gYTEZFiQUVGCs2xC6kMmh/NjhOJADzXuiovd6yFm4uTyclERKS4UJGRQrE8LoFXvt5BcnoWpbxcmdK9Ae3qlDc7loiIFDMqMlKg0jKtvPPjHv636SgAjUNLM6NPJEGlPE1OJiIixZGKjBSY+PMpDJoXza5TSQC8cG8Yw9uH4+qsqSQRESkcKjJSIL7bforR3+wgJcNKgLcbH/RswL01y5kdS0REijkVGflH0jKtjP9+N/M3HwPgrqoBTO8dSQV/D5OTiYhISaAiI3fs4NkrDJoXzd7TyVgsMPi+6gxpWwMXTSWJiEgRUZGRO7Ik+gRvLNtJaoaVsj7ufNirIa1qlDU7loiIlDAqMpIvqRlZjP12F4u3nQCgRVgZPuzdkHK+mkoSEZGipyIjebb/TDID50Zz4OwVnCwwrF04A++rjrOTxexoIiJSQtn1yQyzZs2ifv36+Pn54efnR/PmzVmxYoXZsUocwzBYtOU4D0Wt48DZK5TzdWfus3czpG0NlRgRETGVXR+RqVy5MpMmTaJGjRoYhsGXX37Jww8/TExMDBEREWbHKxFS0rN4fWkcy2JPAdC6Rlmm9mpIWR93k5OJiIiAxTAMw+wQ+REQEMDkyZN55pln8rR+UlIS/v7+JCYm4ufnV8jpipfdp5IYNC+aw+dTcHayMKJDOM/fE4aTjsKIiEghy+v7t10fkfk7q9XK4sWLSUlJoXnz5rmul56eTnp6evb9pKSkoohXrBiGwbzNxxj//W4ysmxU9Pdgep9ImlYJMDuaiIhIDnZfZOLi4mjevDlpaWn4+PiwdOlS6tSpk+v6EydOZPz48UWYsHhJTstk9JI4ftiRAMD9tcoxpUcDArzdTE4mIiJyI7ufWsrIyODYsWMkJiby9ddf85///Ie1a9fmWmZudkQmODhYU0t5sPNkIgPnRXP0QiouThZe6VSLZ1pV1VSSiIgUubxOLdl9kbleu3btCAsL45NPPsnT+jpH5vYMw+C/G4/yzo97yLDaqFTKkxl9I2kUUtrsaCIiUkIVu3NkrrHZbDmOuMg/k3g1k1e+3sFPu04D0KFOeSZ3b4C/l6vJyURERG7ProvM6NGj6dy5MyEhISQnJzNv3jzWrFnDypUrzY5WLMQev8ygedGcuHQVV2cLrz1QmydbVMFi0VSSiIg4BrsuMmfPnuWJJ54gISEBf39/6tevz8qVK2nfvr3Z0RyaYRh8vi6ef/+0l0yrQUiAF1F9I6lfuZTZ0URERPLFrovM559/bnaEYudyagYjF+9g1Z4zADxQrwKTHquPn4emkkRExPHYdZGRgrXt6CUGz4vmVGIabi5OvPlgHR5vFqKpJBERcVgqMiWAzWbw6R+HmbxyH1abQdWy3kT1jSQiyN/saCIiIv+Iikwxd+FKOiMWb2fNvnMAPNQgiHcfrYePu3a9iIg4Pr2bFWOb4y8yeH40Z5LScXdxYvxDEfRqGqypJBERKTZUZIohm83gozUH+eCX/dgMCAv0Zma/RtSqoAsCiohI8aIiU8ycS05n+KJY/jhwHoBHG1ViwsN18dZUkoiIFEN6dytGNhw8z9CFsZxLTsfT1Zm3Ho6gR5Ngs2OJiIgUGhWZYsBqM5j+6wGmrz6AYUB4eR9m9m1EjfK+ZkcTEREpVCoyDu5sUhpDF8Sy8fAFAHo1CWbcQxF4ujmbnExERKTwqcg4sD8OnOOlhbGcv5KBl5sz7z5Sj26RlcyOJSIiUmRUZBxQltXGh6sOMHPNQQwDalf0Y2bfSKoF+pgdTUREpEipyDiYhMSrDJ0fy+YjFwHo1yyENx+sg4erppJERKTkUZFxIL/tPcvwRbFcSs3Ex92FSY/V48H6QWbHEhERMY2KjAPItNqYsnIfn/x+GIC6lfyY2bcRoWW8TU4mIiJiLhUZO3fy8lUGz4sm+thlAJ5sUYXRD9TC3UVTSSIiIioyduyX3WcYuXg7iVcz8fVwYXL3+nSqW9HsWCIiInZDRcYOZWTZmLRiL1+sjwegQXApovpEEhzgZXIyERER+6IiY2eOX0xl0Lxotp9IBODZVlUZ1akWbi5OJicTERGxPyoyduSnnQm8/PUOktOy8Pd05f0eDWhXp7zZsUREROyWiowdSMu0MnH5Hr7ceBSAxqGlmd4nkkqlPE1OJiIiYt9UZEx25HwKA+dFs+tUEgDPtwljRIdwXJ01lSQiInI7KjIm+n77KUYvieNKehYB3m580LMB99YsZ3YsERERh6EiY4K0TCtv/bCbeX8eA+CuqgFM7x1JBX8Pk5OJiIg4FhWZInbo3BUGzo1m7+lkLBYYdF91hratgYumkkRERPJNRaYILY05wetLd5KaYaWsjxsf9oqkVY2yZscSERFxWCoyReBqhpWx3+1k0dYTALQIK8OHvRpSzk9TSSIiIv+EikwhO3AmmRfnRnPg7BWcLDC0bTiD7q+Os5PF7GgiIiIOT0WmkBiGweJtJxjz7U7SMm2U83VnWu9ImoeVMTuaiIhIsaEiUwhS0rN4c9lOlsScBKB1jbJM7dWQsj7uJicTEREpXlRkCtiehCQGzovm8LkUnJ0sDG8fzgttwnDSVJKIiEiBU5EpIIZhMH/zccZ/v4v0LBsV/DyY0TeSplUCzI4mIiJSbNn1xUsmTpxI06ZN8fX1pVy5cnTr1o19+/aZHesGyWmZDFkQy2tL40jPsnF/rXIsH9paJUZERKSQ2XWRWbt2LQMHDmTTpk388ssvZGZm0qFDB1JSUsyOlm3nyUS6zljH99tP4eJk4bUHavGfJ5oQ4O1mdjQREZFiz2IYhmF2iLw6d+4c5cqVY+3atdxzzz152iYpKQl/f38SExPx8/MrsCyGYfC/TUd5+4c9ZFhtVCrlyYy+kTQKKV1gryEiIlJS5fX926HOkUlMTAQgICD3KZv09HTS09Oz7yclJRV4DsMweGlhLMtiTwHQvk55JnevTykvHYUREREpSnY9tfR3NpuNYcOG0bJlS+rWrZvrehMnTsTf3z/7FhwcXOBZLBYLkSGlcXW2MObBOnz6r8YqMSIiIiZwmKmlF154gRUrVrBu3ToqV66c63o3OyITHBxcKFNL8edTqBboU2DPKSIiIn8pVlNLgwYN4ocffuD333+/ZYkBcHd3x9298C88Z7FYVGJERERMZtdFxjAMBg8ezNKlS1mzZg1Vq1Y1O5KIiIjYEbsuMgMHDmTevHl8++23+Pr6cvr0aQD8/f3x9PQ0OZ2IiIiYza7PkbFYbn5Z/9mzZ/Pkk0/m6TkK6+PXIiIiUniKxTkydtyxRERExA44zMevRURERK6nIiMiIiIOS0VGREREHJaKjIiIiDgsFRkRERFxWCoyIiIi4rBUZERERMRhqciIiIiIw1KREREREYdl11f2LQjXrg6clJRkchIRERHJq2vv27e7yn+xLzLJyckABAcHm5xERERE8is5ORl/f/9cH7frL40sCDabjVOnTuHr65vrl1DeiaSkJIKDgzl+/Hix/TLK4j7G4j4+KP5j1PgcX3Efo8Z35wzDIDk5maCgIJyccj8TptgfkXFycqJy5cqF9vx+fn7F8pfz74r7GIv7+KD4j1Hjc3zFfYwa35251ZGYa3Syr4iIiDgsFRkRERFxWCoyd8jd3Z2xY8fi7u5udpRCU9zHWNzHB8V/jBqf4yvuY9T4Cl+xP9lXREREii8dkRERERGHpSIjIiIiDktFRkRERByWioyIiIg4LBWZXPz+++907dqVoKAgLBYLy5Ytu+02a9asoVGjRri7u1O9enXmzJlT6DnvVH7Ht2bNGiwWyw2306dPF03gfJo4cSJNmzbF19eXcuXK0a1bN/bt23fb7RYvXkytWrXw8PCgXr16LF++vAjS3pk7GeOcOXNu2IceHh5FlDh/Zs2aRf369bMvtNW8eXNWrFhxy20caf/ld3yOtO9uZtKkSVgsFoYNG3bL9RxpH14vL2N0pP04bty4G7LWqlXrltuYsf9UZHKRkpJCgwYNmDlzZp7Wj4+Pp0uXLtx3333ExsYybNgwnn32WVauXFnISe9Mfsd3zb59+0hISMi+lStXrpAS/jNr165l4MCBbNq0iV9++YXMzEw6dOhASkpKrtts2LCBPn368MwzzxATE0O3bt3o1q0bO3fuLMLkeXcnY4S/rsD593149OjRIkqcP5UrV2bSpEls27aNrVu3cv/99/Pwww+za9eum67vaPsvv+MDx9l319uyZQuffPIJ9evXv+V6jrYP/y6vYwTH2o8RERE5sq5bty7XdU3bf4bcFmAsXbr0luuMGjXKiIiIyLGsV69eRseOHQsxWcHIy/h+++03AzAuXbpUJJkK2tmzZw3AWLt2ba7r9OzZ0+jSpUuOZc2aNTMGDBhQ2PEKRF7GOHv2bMPf37/oQhWw0qVLG//5z39u+pij7z/DuPX4HHXfJScnGzVq1DB++eUXo02bNsbQoUNzXddR92F+xuhI+3Hs2LFGgwYN8ry+WftPR2QKyMaNG2nXrl2OZR07dmTjxo0mJSocDRs2pGLFirRv357169ebHSfPEhMTAQgICMh1HUffh3kZI8CVK1cIDQ0lODj4tkcA7IXVamXBggWkpKTQvHnzm67jyPsvL+MDx9x3AwcOpEuXLjfsm5tx1H2YnzGCY+3HAwcOEBQURLVq1ejXrx/Hjh3LdV2z9l+x/9LIonL69GnKly+fY1n58uVJSkri6tWreHp6mpSsYFSsWJGPP/6YJk2akJ6ezn/+8x/uvfde/vzzTxo1amR2vFuy2WwMGzaMli1bUrdu3VzXy20f2ut5QH+X1zHWrFmTL774gvr165OYmMiUKVNo0aIFu3btKtQvV71TcXFxNG/enLS0NHx8fFi6dCl16tS56bqOuP/yMz5H23cACxYsIDo6mi1btuRpfUfch/kdoyPtx2bNmjFnzhxq1qxJQkIC48ePp3Xr1uzcuRNfX98b1jdr/6nISJ7UrFmTmjVrZt9v0aIFhw4dYurUqfzvf/8zMdntDRw4kJ07d95ybtfR5XWMzZs3z/EXf4sWLahduzaffPIJEyZMKOyY+VazZk1iY2NJTEzk66+/pn///qxduzbXN3tHk5/xOdq+O378OEOHDuWXX36x25NZ/6k7GaMj7cfOnTtn/3f9+vVp1qwZoaGhLFq0iGeeecbEZDmpyBSQChUqcObMmRzLzpw5g5+fn8MfjcnNXXfdZfflYNCgQfzwww/8/vvvt/1rJ7d9WKFChcKM+I/lZ4zXc3V1JTIykoMHDxZSun/Gzc2N6tWrA9C4cWO2bNnCtGnT+OSTT25Y1xH3X37Gdz1733fbtm3j7NmzOY7YWq1Wfv/9d6KiokhPT8fZ2TnHNo62D+9kjNez9/34d6VKlSI8PDzXrGbtP50jU0CaN2/Or7/+mmPZL7/8csv5bkcXGxtLxYoVzY5xU4ZhMGjQIJYuXcrq1aupWrXqbbdxtH14J2O8ntVqJS4uzm734/VsNhvp6ek3fczR9t/N3Gp817P3fde2bVvi4uKIjY3NvjVp0oR+/foRGxt70zd4R9uHdzLG69n7fvy7K1eucOjQoVyzmrb/CvVUYgeWnJxsxMTEGDExMQZgfPDBB0ZMTIxx9OhRwzAM49VXXzX+9a9/Za9/+PBhw8vLy3j55ZeNPXv2GDNnzjScnZ2Nn376yawh3FJ+xzd16lRj2bJlxoEDB4y4uDhj6NChhpOTk7Fq1SqzhnBLL7zwguHv72+sWbPGSEhIyL6lpqZmr/Ovf/3LePXVV7Pvr1+/3nBxcTGmTJli7Nmzxxg7dqzh6upqxMXFmTGE27qTMY4fP95YuXKlcejQIWPbtm1G7969DQ8PD2PXrl1mDOGWXn31VWPt2rVGfHy8sWPHDuPVV181LBaL8fPPPxuG4fj7L7/jc6R9l5vrP9Hj6PvwZm43RkfajyNGjDDWrFljxMfHG+vXrzfatWtnlC1b1jh79qxhGPaz/1RkcnHt48bX3/r3728YhmH079/faNOmzQ3bNGzY0HBzczOqVatmzJ49u8hz51V+x/fvf//bCAsLMzw8PIyAgADj3nvvNVavXm1O+Dy42diAHPukTZs22eO9ZtGiRUZ4eLjh5uZmREREGD/++GPRBs+HOxnjsGHDjJCQEMPNzc0oX7688cADDxjR0dFFHz4Pnn76aSM0NNRwc3MzAgMDjbZt22a/yRuG4++//I7PkfZdbq5/k3f0fXgztxujI+3HXr16GRUrVjTc3NyMSpUqGb169TIOHjyY/bi97D+LYRhG4R7zERERESkcOkdGREREHJaKjIiIiDgsFRkRERFxWCoyIiIi4rBUZERERMRhqciIiIiIw1KREREREYelIiMiJcKaNWuwWCxcvnzZ7CgiUoBUZESkSFmtVlq0aMGjjz6aY3liYiLBwcG8/vrrhfK6LVq0ICEhAX9//0J5fhExh67sKyJFbv/+/TRs2JDPPvuMfv36AfDEE0+wfft2tmzZgpubm8kJRcRR6IiMiBS58PBwJk2axODBg0lISODbb79lwYIF/Pe//821xLzyyiuEh4fj5eVFtWrVePPNN8nMzAT++ibwdu3a0bFjR679bXbx4kUqV67MmDFjgBunlo4ePUrXrl0pXbo03t7eREREsHz58sIfvIgUKBezA4hIyTR48GCWLl3Kv/71L+Li4hgzZgwNGjTIdX1fX1/mzJlDUFAQcXFxPPfcc/j6+jJq1CgsFgtffvkl9erVY/r06QwdOpTnn3+eSpUqZReZ6w0cOJCMjAx+//13vL292b17Nz4+PoU1XBEpJJpaEhHT7N27l9q1a1OvXj2io6Nxccn731ZTpkxhwYIFbN26NXvZ4sWLeeKJJxg2bBgzZswgJiaGGjVqAH8dkbnvvvu4dOkSpUqVon79+jz22GOMHTu2wMclIkVHU0siYpovvvgCLy8v4uPjOXHiBADPP/88Pj4+2bdrFi5cSMuWLalQoQI+Pj688cYbHDt2LMfz9ejRg0ceeYRJkyYxZcqU7BJzM0OGDOHtt9+mZcuWjB07lh07dhTOIEWkUKnIiIgpNmzYwNSpU/nhhx+46667eOaZZzAMg7feeovY2NjsG8DGjRvp168fDzzwAD/88AMxMTG8/vrrZGRk5HjO1NRUtm3bhrOzMwcOHLjl6z/77LMcPnw4e2qrSZMmzJgxo7CGKyKFREVGRIpcamoqTz75JC+88AL33Xcfn3/+OZs3b+bjjz+mXLlyVK9ePfsGf5We0NBQXn/9dZo0aUKNGjU4evToDc87YsQInJycWLFiBdOnT2f16tW3zBEcHMzzzz/PkiVLGDFiBJ999lmhjFdECo+KjIgUudGjR2MYBpMmTQKgSpUqTJkyhVGjRnHkyJEb1q9RowbHjh1jwYIFHDp0iOnTp7N06dIc6/z444988cUXzJ07l/bt2/Pyyy/Tv39/Ll26dNMMw4YNY+XKlcTHxxMdHc1vv/1G7dq1C3ysIlK4dLKviBSptWvX0rZtW9asWUOrVq1yPNaxY0eysrJYtWoVFoslx2OjRo3iiy++ID09nS5dunD33Xczbtw4Ll++zLlz56hXrx5Dhw5l9OjRAGRmZtK8eXPCwsJYuHDhDSf7Dh48mBUrVnDixAn8/Pzo1KkTU6dOpUyZMkX2sxCRf05FRkRERByWppZERETEYanIiIiIiMNSkRERERGHpSIjIiIiDktFRkRERByWioyIiIg4LBUZERERcVgqMiIiIuKwVGRERETEYanIiIiIiMNSkRERERGHpSIjIiIiDuv/A9e35220Xd/CAAAAAElFTkSuQmCC", + "text/plain": "
" + }, + "metadata": {} + } + ], + "id": "591678dd-b904-4a8d-855a-37b5b1829df9" + }, + { + "cell_type": "code", + "source": "# Dependency resolution submodule consumer\nprint(x_values)", + "metadata": { + "trusted": true + }, + "execution_count": null, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": "Output:\ntensor([[-2.1823]], grad_fn=)\n" + } + ], + "id": "d8b17425-8ff3-4997-bc4f-dee428909dda" + }, + { + "cell_type": "code", + "source": "", + "metadata": { + "trusted": true + }, + "execution_count": null, + "outputs": [], + "id": "35a36bb3-eef9-44a7-8276-2f4d0b4f8c6c" + } + ] + } +} \ No newline at end of file diff --git a/vreapis/tests/resources/notebooks/laserfarm.json b/vreapis/tests/resources/notebooks/laserfarm.json new file mode 100644 index 00000000..9eff9c87 --- /dev/null +++ b/vreapis/tests/resources/notebooks/laserfarm.json @@ -0,0 +1,178 @@ +{ + "save": false, + "kernel": "ipython", + "cell_index": 4, + "notebook": { + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.6" + } + }, + "nbformat_minor": 5, + "nbformat": 4, + "cells": [ + { + "cell_type": "code", + "source": "import fnmatch\nimport json\nimport getpass\nimport os\nimport pathlib\nimport datetime\nimport laspy\n\n\nimport time\nimport requests\n \nfrom dask.distributed import LocalCluster, SSHCluster \nfrom laserfarm import Retiler, DataProcessing, GeotiffWriter, MacroPipeline\nfrom laserfarm.remote_utils import get_wdclient, get_info_remote, list_remote", + "metadata": { + "trusted": true + }, + "execution_count": 1, + "outputs": [ + { + "ename": "ModuleNotFoundError", + "evalue": "No module named 'laspy'", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mModuleNotFoundError\u001b[0m Traceback (most recent call last)", + "Cell \u001b[0;32mIn[1], line 7\u001b[0m\n\u001b[1;32m 5\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mpathlib\u001b[39;00m\n\u001b[1;32m 6\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mdatetime\u001b[39;00m\n\u001b[0;32m----> 7\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mlaspy\u001b[39;00m\n\u001b[1;32m 10\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mtime\u001b[39;00m\n\u001b[1;32m 11\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mrequests\u001b[39;00m\n", + "\u001b[0;31mModuleNotFoundError\u001b[0m: No module named 'laspy'" + ] + } + ], + "id": "8f0e44d1" + }, + { + "cell_type": "markdown", + "source": "## Global Configuration", + "metadata": {}, + "id": "7404fb96" + }, + { + "cell_type": "code", + "source": "import fnmatch\nimport json\nimport getpass\nimport os\nimport pathlib\nimport datetime\n \nfrom dask.distributed import LocalCluster, SSHCluster \nfrom laserfarm import Retiler, DataProcessing, GeotiffWriter, MacroPipeline\nfrom laserfarm.remote_utils import get_wdclient, get_info_remote, list_remote\n\nparam_username = ''\nif 'JUPYTERHUB_USER' in os.environ:\n param_username = os.environ['JUPYTERHUB_USER']\n \nconf_remote_path_root = '/webdav/LAZ'\nconf_remote_path_split = pathlib.Path(conf_remote_path_root + '/split_'+param_username)\nconf_remote_path_retiled = pathlib.Path(conf_remote_path_root + '/retiled_'+param_username)\nconf_remote_path_norm = pathlib.Path(conf_remote_path_root + '/norm_'+param_username)\nconf_remote_path_targets = pathlib.Path(conf_remote_path_root + '/targets_'+param_username)\nconf_local_tmp = pathlib.Path('/tmp')\nconf_remote_path_ahn = conf_remote_path_root\n\n\nparam_hostname = 'https://lfw-ds001-i022.lifewatch.dev:32443/'\nparam_login = '20BNXDdL8mg24OaD'\nparam_password = 'zDoy0hNKkcnsdsQ@OYAVd'\n\nconf_feature_name = 'perc_95_normalized_height'\nconf_validate_precision = '0.001'\nconf_tile_mesh_size = '10.'\nconf_filter_type= 'select_equal'\nconf_attribute = 'raw_classification'\nconf_min_x = '-113107.81'\nconf_max_x = '398892.19'\nconf_min_y = '214783.87'\nconf_max_y = '726783.87'\nconf_n_tiles_side = '512'\nconf_apply_filter_value = '1'\nconf_laz_compression_factor = '7'\nconf_max_filesize = '262144000' # desired max file size (in bytes)\n\nconf_wd_opts = { 'webdav_hostname': param_hostname, 'webdav_login': param_login, 'webdav_password': param_password}", + "metadata": { + "trusted": true + }, + "execution_count": null, + "outputs": [], + "id": "18041536" + }, + { + "cell_type": "markdown", + "source": "## Fetching Laz Files from remote WebDAV", + "metadata": {}, + "id": "d03d1b76" + }, + { + "cell_type": "code", + "source": "# Fetch Laz Files\nprint(conf_remote_path_ahn)\nlaz_files = [f for f in list_remote(get_wdclient(conf_wd_opts), pathlib.Path(conf_remote_path_ahn).as_posix())\n if f.lower().endswith('.laz')]\nprint(laz_files)", + "metadata": { + "trusted": true + }, + "execution_count": null, + "outputs": [ + { + "ename": "NameError", + "evalue": "name 'conf_remote_path_ahn' is not defined", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)", + "Cell \u001b[0;32mIn [2], line 2\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[38;5;66;03m# Fetch Laz Files\u001b[39;00m\n\u001b[0;32m----> 2\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[43mconf_remote_path_ahn\u001b[49m)\n\u001b[1;32m 3\u001b[0m laz_files \u001b[38;5;241m=\u001b[39m [f \u001b[38;5;28;01mfor\u001b[39;00m f \u001b[38;5;129;01min\u001b[39;00m list_remote(get_wdclient(conf_wd_opts), pathlib\u001b[38;5;241m.\u001b[39mPath(conf_remote_path_ahn)\u001b[38;5;241m.\u001b[39mas_posix())\n\u001b[1;32m 4\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m f\u001b[38;5;241m.\u001b[39mlower()\u001b[38;5;241m.\u001b[39mendswith(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m.laz\u001b[39m\u001b[38;5;124m'\u001b[39m)]\n\u001b[1;32m 5\u001b[0m \u001b[38;5;28mprint\u001b[39m(laz_files)\n", + "\u001b[0;31mNameError\u001b[0m: name 'conf_remote_path_ahn' is not defined" + ] + } + ], + "id": "573679af" + }, + { + "cell_type": "markdown", + "source": "## Splitting big files into smaller files before retiling\nThis step can be added if the original files are too large for normal VMs to process", + "metadata": {}, + "id": "bc0773b1" + }, + { + "cell_type": "code", + "source": "# split big files\n\nimport numpy as np\n\ndef save_chunk_to_laz_file(in_filename, \n out_filename, \n offset, \n n_points):\n \"\"\"Read points from a LAS/LAZ file and write them to a new file.\"\"\"\n \n points = np.array([])\n \n with laspy.open(in_filename) as in_file:\n with laspy.open(out_filename, \n mode=\"w\", \n header=in_file.header) as out_file:\n in_file.seek(offset)\n points = in_file.read_points(n_points)\n out_file.write_points(points)\n return len(points)\n\ndef split_strategy(filename, max_filesize):\n \"\"\"Set up splitting strategy for a LAS/LAZ file.\"\"\"\n with laspy.open(filename) as f:\n bytes_per_point = (\n f.header.point_format.num_standard_bytes +\n f.header.point_format.num_extra_bytes\n )\n n_points = f.header.point_count\n n_points_target = int(\n max_filesize * int(conf_laz_compression_factor) / bytes_per_point\n )\n stem, ext = os.path.splitext(filename)\n return [\n (filename, f\"{stem}-{n}{ext}\", offset, n_points_target)\n for n, offset in enumerate(range(0, n_points, n_points_target))\n ]\n\nfrom webdav3.client import Client\n\nclient = Client(conf_wd_opts)\nclient.mkdir(conf_remote_path_split.as_posix())\n\n\nremote_path_split = conf_remote_path_split\n\n\nfor file in laz_files:\n print('Splitting: '+file )\n client.download_sync(remote_path=os.path.join(conf_remote_path_ahn,file), local_path=file)\n inps = split_strategy(file, int(conf_max_filesize))\n for inp in inps:\n save_chunk_to_laz_file(*inp)\n client.upload_sync(remote_path=os.path.join(conf_remote_path_split,file), local_path=file)\n\n for f in os.listdir('.'):\n if not f.endswith('.LAZ'):\n continue\n os.remove(os.path.join('.', f))\n \nsplit_laz_files = laz_files", + "metadata": { + "trusted": true + }, + "execution_count": null, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": "Splitting: C_18HZ2.LAZ\n" + }, + { + "ename": "NameError", + "evalue": "name 'laspy' is not defined", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)", + "Input \u001b[0;32mIn [9]\u001b[0m, in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[1;32m 49\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mSplitting: \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;241m+\u001b[39mfile )\n\u001b[1;32m 50\u001b[0m client\u001b[38;5;241m.\u001b[39mdownload_sync(remote_path\u001b[38;5;241m=\u001b[39mos\u001b[38;5;241m.\u001b[39mpath\u001b[38;5;241m.\u001b[39mjoin(conf_remote_path_ahn,file), local_path\u001b[38;5;241m=\u001b[39mfile)\n\u001b[0;32m---> 51\u001b[0m inps \u001b[38;5;241m=\u001b[39m \u001b[43msplit_strategy\u001b[49m\u001b[43m(\u001b[49m\u001b[43mfile\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mint\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43mconf_max_filesize\u001b[49m\u001b[43m)\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 52\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m inp \u001b[38;5;129;01min\u001b[39;00m inps:\n\u001b[1;32m 53\u001b[0m save_chunk_to_laz_file(\u001b[38;5;241m*\u001b[39minp)\n", + "Input \u001b[0;32mIn [9]\u001b[0m, in \u001b[0;36msplit_strategy\u001b[0;34m(filename, max_filesize)\u001b[0m\n\u001b[1;32m 22\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21msplit_strategy\u001b[39m(filename, max_filesize):\n\u001b[1;32m 23\u001b[0m \u001b[38;5;124;03m\"\"\"Set up splitting strategy for a LAS/LAZ file.\"\"\"\u001b[39;00m\n\u001b[0;32m---> 24\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m \u001b[43mlaspy\u001b[49m\u001b[38;5;241m.\u001b[39mopen(filename) \u001b[38;5;28;01mas\u001b[39;00m f:\n\u001b[1;32m 25\u001b[0m bytes_per_point \u001b[38;5;241m=\u001b[39m (\n\u001b[1;32m 26\u001b[0m f\u001b[38;5;241m.\u001b[39mheader\u001b[38;5;241m.\u001b[39mpoint_format\u001b[38;5;241m.\u001b[39mnum_standard_bytes \u001b[38;5;241m+\u001b[39m\n\u001b[1;32m 27\u001b[0m f\u001b[38;5;241m.\u001b[39mheader\u001b[38;5;241m.\u001b[39mpoint_format\u001b[38;5;241m.\u001b[39mnum_extra_bytes\n\u001b[1;32m 28\u001b[0m )\n\u001b[1;32m 29\u001b[0m n_points \u001b[38;5;241m=\u001b[39m f\u001b[38;5;241m.\u001b[39mheader\u001b[38;5;241m.\u001b[39mpoint_count\n", + "\u001b[0;31mNameError\u001b[0m: name 'laspy' is not defined" + ] + } + ], + "id": "c074eba8" + }, + { + "cell_type": "markdown", + "source": "## Retiling of big files into smaller tiles", + "metadata": {}, + "id": "f416e842" + }, + { + "cell_type": "code", + "source": "# Retiling\nsplit_laz_files\nremote_path_retiled = str(conf_remote_path_retiled)\n\ngrid_retile = {\n 'min_x': float(conf_min_x),\n 'max_x': float(conf_max_x),\n 'min_y': float(conf_min_y),\n 'max_y': float(conf_max_y),\n 'n_tiles_side': int(conf_n_tiles_side)\n}\n\nretiling_input = {\n 'setup_local_fs': {'tmp_folder': conf_local_tmp.as_posix()},\n 'pullremote': conf_remote_path_split.as_posix(),\n 'set_grid': grid_retile,\n 'split_and_redistribute': {},\n 'validate': {},\n 'pushremote': conf_remote_path_retiled.as_posix(),\n 'cleanlocalfs': {}\n}\n\nfor file in split_laz_files:\n retiler = Retiler(file.replace('\"',''),label=file).config(retiling_input).setup_webdav_client(conf_wd_opts)\n retiler_output = retiler.run()", + "metadata": { + "trusted": true + }, + "execution_count": null, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": "2022-07-10 21:10:04,119 - laserfarm.pipeline_remote_data - INFO - Input dir set to /tmp/C_18HZ2.LAZ_input\n2022-07-10 21:10:04,121 - laserfarm.pipeline_remote_data - INFO - Output dir set to /tmp/C_18HZ2.LAZ_output\n2022-07-10 21:10:04,125 - laserfarm.pipeline_remote_data - INFO - Pulling from WebDAV /webdav/pointcloud/split_skoulouzis/C_18HZ2.LAZ ...\n2022-07-10 21:10:05,596 - laserfarm.pipeline_remote_data - INFO - ... pulling completed.\n2022-07-10 21:10:05,597 - laserfarm.retiler - INFO - Setting up the target grid\n2022-07-10 21:10:05,600 - laserfarm.retiler - INFO - Splitting file /tmp/C_18HZ2.LAZ_input/C_18HZ2.LAZ with PDAL ...\n2022-07-10 21:10:06,079 - laserfarm.retiler - INFO - ... splitting completed.\n2022-07-10 21:10:06,083 - laserfarm.retiler - INFO - Redistributing files to tiles ...\n2022-07-10 21:10:06,084 - laserfarm.retiler - INFO - ... file C_18HZ2_1.LAZ to tile_213_285\n2022-07-10 21:10:06,086 - laserfarm.retiler - INFO - ... file C_18HZ2_2.LAZ to tile_212_285\n2022-07-10 21:10:06,089 - laserfarm.retiler - INFO - ... redistributing completed.\n2022-07-10 21:10:06,095 - laserfarm.retiler - INFO - Validating split ...\n2022-07-10 21:10:06,099 - laserfarm.retiler - INFO - ... 257128 points in parent file\n2022-07-10 21:10:06,104 - laserfarm.retiler - INFO - ... 25414 points in C_18HZ2_2.LAZ\n2022-07-10 21:10:06,105 - laserfarm.retiler - INFO - ... 231714 points in C_18HZ2_1.LAZ\n2022-07-10 21:10:06,107 - laserfarm.retiler - INFO - ... split validation completed.\n2022-07-10 21:10:06,109 - laserfarm.pipeline_remote_data - INFO - Pushing to WebDAV /webdav/pointcloud/retiled_skoulouzis ...\n2022-07-10 21:10:12,031 - laserfarm.pipeline_remote_data - INFO - ... pushing completed.\n2022-07-10 21:10:12,032 - laserfarm.pipeline_remote_data - INFO - Removing input and output folders\n2022-07-10 21:10:12,036 - laserfarm.pipeline_remote_data - INFO - Input dir set to /tmp/C_19HZ2.LAZ_input\n2022-07-10 21:10:12,037 - laserfarm.pipeline_remote_data - INFO - Output dir set to /tmp/C_19HZ2.LAZ_output\n2022-07-10 21:10:12,038 - laserfarm.pipeline_remote_data - INFO - Pulling from WebDAV /webdav/pointcloud/split_skoulouzis/C_19HZ2.LAZ ...\n2022-07-10 21:10:13,474 - laserfarm.pipeline_remote_data - INFO - ... pulling completed.\n2022-07-10 21:10:13,475 - laserfarm.retiler - INFO - Setting up the target grid\n2022-07-10 21:10:13,476 - laserfarm.retiler - INFO - Splitting file /tmp/C_19HZ2.LAZ_input/C_19HZ2.LAZ with PDAL ...\n2022-07-10 21:10:13,615 - laserfarm.retiler - INFO - ... splitting completed.\n2022-07-10 21:10:13,616 - laserfarm.retiler - INFO - Redistributing files to tiles ...\n2022-07-10 21:10:13,618 - laserfarm.retiler - INFO - ... file C_19HZ2_1.LAZ to tile_248_285\n2022-07-10 21:10:13,618 - laserfarm.retiler - INFO - ... redistributing completed.\n2022-07-10 21:10:13,619 - laserfarm.retiler - INFO - Validating split ...\n2022-07-10 21:10:13,620 - laserfarm.retiler - INFO - ... 66663 points in parent file\n2022-07-10 21:10:13,621 - laserfarm.retiler - INFO - ... 66663 points in C_19HZ2_1.LAZ\n2022-07-10 21:10:13,622 - laserfarm.retiler - INFO - ... split validation completed.\n2022-07-10 21:10:13,623 - laserfarm.pipeline_remote_data - INFO - Pushing to WebDAV /webdav/pointcloud/retiled_skoulouzis ...\n2022-07-10 21:10:17,817 - laserfarm.pipeline_remote_data - INFO - ... pushing completed.\n2022-07-10 21:10:17,818 - laserfarm.pipeline_remote_data - INFO - Removing input and output folders\n2022-07-10 21:10:17,821 - laserfarm.pipeline_remote_data - INFO - Input dir set to /tmp/C_01GN2.LAZ_input\n2022-07-10 21:10:17,822 - laserfarm.pipeline_remote_data - INFO - Output dir set to /tmp/C_01GN2.LAZ_output\n2022-07-10 21:10:17,824 - laserfarm.pipeline_remote_data - INFO - Pulling from WebDAV /webdav/pointcloud/split_skoulouzis/C_01GN2.LAZ ...\n2022-07-10 21:10:20,467 - laserfarm.pipeline_remote_data - INFO - ... pulling completed.\n2022-07-10 21:10:20,468 - laserfarm.retiler - INFO - Setting up the target grid\n2022-07-10 21:10:20,470 - laserfarm.retiler - INFO - Splitting file /tmp/C_01GN2.LAZ_input/C_01GN2.LAZ with PDAL ...\n2022-07-10 21:10:26,047 - laserfarm.retiler - INFO - ... splitting completed.\n2022-07-10 21:10:26,049 - laserfarm.retiler - INFO - Redistributing files to tiles ...\n2022-07-10 21:10:26,050 - laserfarm.retiler - INFO - ... file C_01GN2_6.LAZ to tile_282_392\n2022-07-10 21:10:26,052 - laserfarm.retiler - INFO - ... file C_01GN2_2.LAZ to tile_278_392\n2022-07-10 21:10:26,053 - laserfarm.retiler - INFO - ... file C_01GN2_4.LAZ to tile_282_391\n2022-07-10 21:10:26,054 - laserfarm.retiler - INFO - ... file C_01GN2_1.LAZ to tile_278_391\n2022-07-10 21:10:26,056 - laserfarm.retiler - INFO - ... file C_01GN2_3.LAZ to tile_279_391\n2022-07-10 21:10:26,057 - laserfarm.retiler - INFO - ... file C_01GN2_7.LAZ to tile_283_392\n2022-07-10 21:10:26,058 - laserfarm.retiler - INFO - ... file C_01GN2_5.LAZ to tile_283_391\n2022-07-10 21:10:26,067 - laserfarm.retiler - INFO - ... redistributing completed.\n2022-07-10 21:10:26,068 - laserfarm.retiler - INFO - Validating split ...\n2022-07-10 21:10:26,069 - laserfarm.retiler - INFO - ... 3516323 points in parent file\n2022-07-10 21:10:26,070 - laserfarm.retiler - INFO - ... 2286839 points in C_01GN2_1.LAZ\n2022-07-10 21:10:26,071 - laserfarm.retiler - INFO - ... 105908 points in C_01GN2_4.LAZ\n2022-07-10 21:10:26,072 - laserfarm.retiler - INFO - ... 193772 points in C_01GN2_6.LAZ\n2022-07-10 21:10:26,073 - laserfarm.retiler - INFO - ... 219926 points in C_01GN2_3.LAZ\n2022-07-10 21:10:26,074 - laserfarm.retiler - INFO - ... 266823 points in C_01GN2_5.LAZ\n2022-07-10 21:10:26,079 - laserfarm.retiler - INFO - ... 407361 points in C_01GN2_7.LAZ\n2022-07-10 21:10:26,080 - laserfarm.retiler - INFO - ... 35694 points in C_01GN2_2.LAZ\n2022-07-10 21:10:26,080 - laserfarm.retiler - INFO - ... split validation completed.\n2022-07-10 21:10:26,081 - laserfarm.pipeline_remote_data - INFO - Pushing to WebDAV /webdav/pointcloud/retiled_skoulouzis ...\n2022-07-10 21:10:41,777 - laserfarm.pipeline_remote_data - INFO - ... pushing completed.\n2022-07-10 21:10:41,778 - laserfarm.pipeline_remote_data - INFO - Removing input and output folders\n2022-07-10 21:10:41,792 - laserfarm.pipeline_remote_data - INFO - Input dir set to /tmp/C_05FN1.LAZ_input\n2022-07-10 21:10:41,793 - laserfarm.pipeline_remote_data - INFO - Output dir set to /tmp/C_05FN1.LAZ_output\n2022-07-10 21:10:41,795 - laserfarm.pipeline_remote_data - INFO - Pulling from WebDAV /webdav/pointcloud/split_skoulouzis/C_05FN1.LAZ ...\n2022-07-10 21:10:43,375 - laserfarm.pipeline_remote_data - INFO - ... pulling completed.\n2022-07-10 21:10:43,377 - laserfarm.retiler - INFO - Setting up the target grid\n2022-07-10 21:10:43,378 - laserfarm.retiler - INFO - Splitting file /tmp/C_05FN1.LAZ_input/C_05FN1.LAZ with PDAL ...\n2022-07-10 21:10:44,596 - laserfarm.retiler - INFO - ... splitting completed.\n2022-07-10 21:10:44,597 - laserfarm.retiler - INFO - Redistributing files to tiles ...\n2022-07-10 21:10:44,599 - laserfarm.retiler - INFO - ... file C_05FN1_4.LAZ to tile_288_379\n2022-07-10 21:10:44,600 - laserfarm.retiler - INFO - ... file C_05FN1_2.LAZ to tile_287_379\n2022-07-10 21:10:44,601 - laserfarm.retiler - INFO - ... file C_05FN1_3.LAZ to tile_288_378\n2022-07-10 21:10:44,603 - laserfarm.retiler - INFO - ... file C_05FN1_1.LAZ to tile_287_378\n2022-07-10 21:10:44,603 - laserfarm.retiler - INFO - ... redistributing completed.\n2022-07-10 21:10:44,606 - laserfarm.retiler - INFO - Validating split ...\n2022-07-10 21:10:44,606 - laserfarm.retiler - INFO - ... 771576 points in parent file\n2022-07-10 21:10:44,607 - laserfarm.retiler - INFO - ... 232882 points in C_05FN1_4.LAZ\n2022-07-10 21:10:44,609 - laserfarm.retiler - INFO - ... 294136 points in C_05FN1_2.LAZ\n2022-07-10 21:10:44,610 - laserfarm.retiler - INFO - ... 55494 points in C_05FN1_3.LAZ\n2022-07-10 21:10:44,614 - laserfarm.retiler - INFO - ... 189064 points in C_05FN1_1.LAZ\n2022-07-10 21:10:44,615 - laserfarm.retiler - INFO - ... split validation completed.\n2022-07-10 21:10:44,616 - laserfarm.pipeline_remote_data - INFO - Pushing to WebDAV /webdav/pointcloud/retiled_skoulouzis ...\n2022-07-10 21:10:54,348 - laserfarm.pipeline_remote_data - INFO - ... pushing completed.\n2022-07-10 21:10:54,350 - laserfarm.pipeline_remote_data - INFO - Removing input and output folders\n2022-07-10 21:10:54,354 - laserfarm.pipeline_remote_data - INFO - Input dir set to /tmp/C_50GZ2.LAZ_input\n2022-07-10 21:10:54,355 - laserfarm.pipeline_remote_data - INFO - Output dir set to /tmp/C_50GZ2.LAZ_output\n2022-07-10 21:10:54,357 - laserfarm.pipeline_remote_data - INFO - Pulling from WebDAV /webdav/pointcloud/split_skoulouzis/C_50GZ2.LAZ ...\n2022-07-10 21:10:55,925 - laserfarm.pipeline_remote_data - INFO - ... pulling completed.\n2022-07-10 21:10:55,927 - laserfarm.retiler - INFO - Setting up the target grid\n2022-07-10 21:10:55,934 - laserfarm.retiler - INFO - Splitting file /tmp/C_50GZ2.LAZ_input/C_50GZ2.LAZ with PDAL ...\n2022-07-10 21:10:56,748 - laserfarm.retiler - INFO - ... splitting completed.\n2022-07-10 21:10:56,749 - laserfarm.retiler - INFO - Redistributing files to tiles ...\n2022-07-10 21:10:56,751 - laserfarm.retiler - INFO - ... file C_50GZ2_1.LAZ to tile_238_166\n2022-07-10 21:10:56,753 - laserfarm.retiler - INFO - ... redistributing completed.\n2022-07-10 21:10:56,755 - laserfarm.retiler - INFO - Validating split ...\n2022-07-10 21:10:56,758 - laserfarm.retiler - INFO - ... 499572 points in parent file\n2022-07-10 21:10:56,760 - laserfarm.retiler - INFO - ... 499572 points in C_50GZ2_1.LAZ\n2022-07-10 21:10:56,763 - laserfarm.retiler - INFO - ... split validation completed.\n2022-07-10 21:10:56,766 - laserfarm.pipeline_remote_data - INFO - Pushing to WebDAV /webdav/pointcloud/retiled_skoulouzis ...\n2022-07-10 21:11:01,220 - laserfarm.pipeline_remote_data - INFO - ... pushing completed.\n2022-07-10 21:11:01,221 - laserfarm.pipeline_remote_data - INFO - Removing input and output folders\n" + } + ], + "id": "8b489f30" + }, + { + "cell_type": "code", + "source": "print(retiler_output)", + "metadata": { + "trusted": true + }, + "execution_count": null, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": "None\n" + } + ], + "id": "986c70f3" + }, + { + "cell_type": "code", + "source": "", + "metadata": { + "trusted": true + }, + "execution_count": null, + "outputs": [], + "id": "c8814681" + } + ] + } +} diff --git a/vreapis/tests/resources/notebooks/r-notebook.json b/vreapis/tests/resources/notebooks/r-notebook.json new file mode 100644 index 00000000..3ff1dea3 --- /dev/null +++ b/vreapis/tests/resources/notebooks/r-notebook.json @@ -0,0 +1,63 @@ +{ + "save": false, + "kernel": "IRkernel", + "cell_index": 2, + "notebook": { + "metadata": { + "kernelspec": { + "display_name": "R", + "language": "R", + "name": "ir" + }, + "language_info": { + "codemirror_mode": "r", + "file_extension": ".r", + "mimetype": "text/x-r-source", + "name": "R", + "pygments_lexer": "r", + "version": "4.3.2" + } + }, + "nbformat_minor": 5, + "nbformat": 4, + "cells": [ + { + "cell_type": "code", + "source": "# inputR\na = 2 \nnumbers <- c(a, 4, 6, 8, 10)\n", + "metadata": { + "tags": [], + "trusted": true + }, + "execution_count": 1, + "outputs": [], + "id": "59b2ee0b-958d-431f-a396-b4b34c77d540" + }, + { + "cell_type": "code", + "source": "#averageR\n\n\naverage <- mean(numbers)", + "metadata": { + "trusted": true + }, + "execution_count": 2, + "outputs": [], + "id": "0a83d716-23c8-4eb2-b246-9a8f4cdf9ac2" + }, + { + "cell_type": "code", + "source": "#print \n\nprint(average)", + "metadata": { + "trusted": true + }, + "execution_count": 3, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": "[1] 6\n" + } + ], + "id": "bc5ace1d-809a-4dc1-b5f7-075a93e8474a" + } + ] + } +} \ No newline at end of file diff --git a/vreapis/tests/resources/notebooks/test_!.json b/vreapis/tests/resources/notebooks/test_!.json new file mode 100644 index 00000000..c7735f45 --- /dev/null +++ b/vreapis/tests/resources/notebooks/test_!.json @@ -0,0 +1,158 @@ +{ + "save": false, + "kernel": "ipython", + "cell_index": 1, + "notebook": { + "metadata": { + "kernelspec": { + "display_name": "Python [conda env:jupyterlab] *", + "language": "python", + "name": "conda-env-jupyterlab-py" + }, + "language_info": { + "name": "python", + "version": "3.10.12", + "mimetype": "text/x-python", + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "pygments_lexer": "ipython3", + "nbconvert_exporter": "python", + "file_extension": ".py" + } + }, + "nbformat_minor": 5, + "nbformat": 4, + "cells": [ + { + "cell_type": "code", + "source": "!conda install --yes -c conda-forge pdal python-pdal gdal dask\n!conda activate --yes pdalpy\n!pip install dask distributed laserfarm\n!conda upgrade --yes numpy", + "metadata": { + "trusted": true + }, + "execution_count": null, + "outputs": [], + "id": "af2a436e-6682-40cb-bd4c-fc7aa0a719ba" + }, + { + "cell_type": "code", + "source": "# conf \nimport os\n\nconf_data_folder = os.path.join('/tmp','data')\n\nconf_feature_name = 'perc_95_normalized_height'\nconf_validate_precision = '0.001'\nconf_tile_mesh_size = '10.'\nconf_filter_type= 'select_equal'\nconf_attribute = 'raw_classification'\nconf_min_x = '-113107.81'\nconf_max_x = '398892.19'\nconf_min_y = '214783.87'\nconf_max_y = '726783.87'\nconf_n_tiles_side = '512'\nconf_apply_filter_value = '1'\nconf_laz_compression_factor = '7'\nconf_max_filesize = '262144000' # desired max file size (in bytes)\n\n", + "metadata": { + "trusted": true + }, + "execution_count": 1, + "outputs": [], + "id": "bca08f46-0546-47a6-b0eb-d9fa3a5c9265" + }, + { + "cell_type": "code", + "source": "# Create file \n\nL = [\"a\\n\", \"b\\n\", \"c\\n\"]\nfile_path = os.path.join(conf_data_folder,'hello.txt')\nfp = open(file_path, 'w')\nfp.writelines(L)\nfp.close()", + "metadata": { + "trusted": true + }, + "execution_count": 2, + "outputs": [], + "id": "00dd23e8-531e-4dd8-90aa-43573c240a48" + }, + { + "cell_type": "code", + "source": "#read file lines\n\nf = open(file_path, 'r')\nlines = f.readlines()\nf.close()", + "metadata": { + "trusted": true + }, + "execution_count": 3, + "outputs": [], + "id": "452c7037-2686-46bc-90d2-1d681b2b4a72" + }, + { + "cell_type": "code", + "source": "#loop file lines\n\ncount = 0\n# Strips the newline character\nfor l in lines:\n count += 1\n print(\"Line{}: {}\".format(count, l.strip()))", + "metadata": { + "trusted": true + }, + "execution_count": 4, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": "Line1: a\nLine2: b\nLine3: c\n" + } + ], + "id": "918da96c-0e5d-4c49-8604-80dbea97f034" + }, + { + "cell_type": "code", + "source": "# Add1\n\na = count + 1", + "metadata": { + "trusted": true + }, + "execution_count": null, + "outputs": [], + "id": "25bdee30-184c-4fb9-9f60-6cb2bf3830a4" + }, + { + "cell_type": "code", + "source": "#Anti-pattern \nsome_list = range(count, a+1)\n\nmsg = '1'\n\n", + "metadata": { + "trusted": true + }, + "execution_count": 11, + "outputs": [], + "id": "c1ee0030-e097-49e9-b67c-69f21a8a6da3" + }, + { + "cell_type": "code", + "source": "# input lists\n\nlist_of_paths = [\"/webdav/LAZ/targets_myname\",\"/webdav/LAZ/targets_myname\",\"/webdav/LAZ/targets_myname\",\"/webdav/LAZ/targets_myname\",\"/webdav/LAZ/targets_myname\",\"/webdav/LAZ/targets_myname\",\"/webdav/LAZ/targets_myname\",\"/webdav/LAZ/targets_myname\",\"/webdav/LAZ/targets_myname\",\"/webdav/LAZ/targets_myname\"]\nlist_of_ints = [1,2,35,6,65]\nprint(msg)", + "metadata": { + "trusted": true + }, + "execution_count": 6, + "outputs": [], + "id": "7b8027f6-7be6-43a9-958f-2337a0ed0733" + }, + { + "cell_type": "code", + "source": "#loop list \n\nfor l in list_of_paths:\n print(l)", + "metadata": { + "trusted": true + }, + "execution_count": 6, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": "/webdav/LAZ/targets_myname\n/webdav/LAZ/targets_myname\n/webdav/LAZ/targets_myname\n/webdav/LAZ/targets_myname\n/webdav/LAZ/targets_myname\n/webdav/LAZ/targets_myname\n/webdav/LAZ/targets_myname\n/webdav/LAZ/targets_myname\n/webdav/LAZ/targets_myname\n/webdav/LAZ/targets_myname\n" + } + ], + "id": "ffdc9192-db4e-44f4-8a13-62a450ac3aa1" + }, + { + "cell_type": "code", + "source": "#loop int list \n\nfor i in list_of_ints:\n a = i -1\n print(a)", + "metadata": { + "trusted": true + }, + "execution_count": 8, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": "2\n3\n36\n7\n66\n" + } + ], + "id": "2899e58e-2c8b-46f4-a7a5-ed02156371a4" + }, + { + "cell_type": "code", + "source": "", + "metadata": { + "trusted": true + }, + "execution_count": null, + "outputs": [], + "id": "3ea24b6d-f13c-42d9-a340-42cd0aa7ae4c" + } + ] + } +} \ No newline at end of file diff --git a/vreapis/tests/resources/notebooks/test_R_param_in_cell.json b/vreapis/tests/resources/notebooks/test_R_param_in_cell.json new file mode 100644 index 00000000..b4b3112c --- /dev/null +++ b/vreapis/tests/resources/notebooks/test_R_param_in_cell.json @@ -0,0 +1,36 @@ +{ + "save": false, + "kernel": "IRkernel", + "cell_index": 0, + "notebook": { + "metadata": { + "kernelspec": { + "display_name": "R", + "language": "R", + "name": "ir" + }, + "language_info": { + "codemirror_mode": "r", + "file_extension": ".r", + "mimetype": "text/x-r-source", + "name": "R", + "pygments_lexer": "r", + "version": "4.3.1" + } + }, + "nbformat_minor": 5, + "nbformat": 4, + "cells": [ + { + "cell_type": "code", + "source": "# print param R in cell\nparam_b = 'value b'\nprint(param_b)", + "metadata": { + "trusted": true + }, + "execution_count": null, + "outputs": [], + "id": "f1a6b009-966b-4c7d-bb2a-e2381ead781f" + } + ] + } +} \ No newline at end of file diff --git a/vreapis/tests/resources/notebooks/test_R_param_outside_cell.json b/vreapis/tests/resources/notebooks/test_R_param_outside_cell.json new file mode 100644 index 00000000..b0cf19b9 --- /dev/null +++ b/vreapis/tests/resources/notebooks/test_R_param_outside_cell.json @@ -0,0 +1,46 @@ +{ + "save": false, + "kernel": "IRkernel", + "cell_index": 1, + "notebook": { + "metadata": { + "kernelspec": { + "display_name": "R", + "language": "R", + "name": "ir" + }, + "language_info": { + "codemirror_mode": "r", + "file_extension": ".r", + "mimetype": "text/x-r-source", + "name": "R", + "pygments_lexer": "r", + "version": "4.3.1" + } + }, + "nbformat_minor": 5, + "nbformat": 4, + "cells": [ + { + "cell_type": "code", + "source": "param_a = 'value a'", + "metadata": { + "trusted": true + }, + "execution_count": 1, + "outputs": [], + "id": "1b784005-98ec-48d0-8d6d-339def7e42cb" + }, + { + "cell_type": "code", + "source": "# print param R outside cell\nprint(param_a)", + "metadata": { + "trusted": true + }, + "execution_count": null, + "outputs": [], + "id": "e7657fee-d6ef-4748-83d9-997c713afb52" + } + ] + } +} \ No newline at end of file diff --git a/vreapis/tests/resources/notebooks/test_cell_header_R.json b/vreapis/tests/resources/notebooks/test_cell_header_R.json new file mode 100644 index 00000000..fa4bd032 --- /dev/null +++ b/vreapis/tests/resources/notebooks/test_cell_header_R.json @@ -0,0 +1,87 @@ +{ + "save": false, + "kernel": "IRkernel", + "cell_index": 1, + "notebook": { + "metadata": { + "kernelspec": { + "display_name": "R", + "language": "R", + "name": "ir" + }, + "language_info": { + "codemirror_mode": "r", + "file_extension": ".r", + "mimetype": "text/x-r-source", + "name": "R", + "pygments_lexer": "r", + "version": "4.3.1" + } + }, + "nbformat_minor": 5, + "nbformat": 4, + "cells": [ + { + "cell_type": "code", + "source": [ + "# params\n", + "param_string = 'param_string value'\n", + "param_string_with_comment = 'param_string value' # comment\n", + "param_int = 1\n", + "param_float = 1.1" + ], + "metadata": { + "trusted": true + }, + "execution_count": null, + "outputs": [], + "id": "de7cc7cf-56c9-4538-92be-1a258e5ee812" + }, + { + "cell_type": "code", + "source": [ + "# Param check\n", + "# ---\n", + "# NaaVRE:\n", + "# cell:\n", + "# inputs:\n", + "# - a\n", + "# - b: String\n", + "# - c:\n", + "# type: Integer\n", + "# outputs:\n", + "# - e\n", + "# - f: Float\n", + "# - g:\n", + "# type: List\n", + "# params:\n", + "# - param_h\n", + "# - param_i: Float\n", + "# - param_j:\n", + "# type: List\n", + "# - param_k:\n", + "# type: List\n", + "# default_value: 'param_k value'\n", + "# confs:\n", + "# - conf_l:\n", + "# assignation: 'conf_l = 1'\n", + "# dependencies:\n", + "# - name: numpy\n", + "# asname: np\n", + "# ...\n", + "print(param_string)\n", + "print(param_string_with_comment)\n", + "print(param_int)\n", + "print(param_float)" + ], + "metadata": { + "trusted": true, + "tags": [] + }, + "execution_count": null, + "outputs": [], + "id": "9824a80c-b00b-441c-8152-7b2e671e081b" + } + ] + } +} diff --git a/vreapis/tests/resources/notebooks/test_conf_nesting.json b/vreapis/tests/resources/notebooks/test_conf_nesting.json new file mode 100644 index 00000000..ec314a3e --- /dev/null +++ b/vreapis/tests/resources/notebooks/test_conf_nesting.json @@ -0,0 +1,53 @@ +{ + "cell_index": 1, + "kernel": "ipython", + "notebook": { + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "8684f3d5-5269-4caf-a594-1b6549e75732", + "metadata": { + "tags": [], + "trusted": true + }, + "outputs": [], + "source": "conf_a = \"a\"\nconf_b = conf_a + \"b\"\nconf_c = conf_b + \"c\"" + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6f9efd5a-a9ee-45d1-8f24-29701890e98d", + "metadata": { + "tags": [], + "trusted": true + }, + "outputs": [], + "source": "conf_c" + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.8" + }, + "toc-showcode": true + }, + "nbformat": 4, + "nbformat_minor": 5 + }, + "save": false +} \ No newline at end of file diff --git a/vreapis/tests/resources/notebooks/test_new_line.json b/vreapis/tests/resources/notebooks/test_new_line.json new file mode 100644 index 00000000..f9ddc618 --- /dev/null +++ b/vreapis/tests/resources/notebooks/test_new_line.json @@ -0,0 +1,48 @@ +{ + "save": false, + "kernel": "ipython", + "cell_index": 1, + "notebook": { + "metadata": { + "language_info": { + "name": "python", + "version": "3.11.6", + "mimetype": "text/x-python", + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "pygments_lexer": "ipython3", + "nbconvert_exporter": "python", + "file_extension": ".py" + }, + "kernelspec": { + "name": "python3", + "display_name": "Python 3 (ipykernel)", + "language": "python" + } + }, + "nbformat_minor": 5, + "nbformat": 4, + "cells": [ + { + "cell_type": "code", + "source": "#Conf_bug\n\nconf_herwijnen = ['radar_volume_full_herwijnen',\n 1.0,\n 'https://api.dataplatform.knmi.nl/open-data/v1/datasets/radar_volume_full_herwijnen/versions/1.0/files',\n 'NL/HRW'] \nconf_denhelder = ['radar_volume_full_denhelder',\n 2.0,\n 'https://api.dataplatform.knmi.nl/open-data/v1/datasets/radar_volume_denhelder/versions/2.0/files',\n 'NL/DHL']\nconf_radars = {'herwijnen' : conf_herwijnen,\n 'denhelder' : conf_denhelder}", + "metadata": { + "trusted": true + }, + "execution_count": null, + "outputs": [], + "id": "1781e699-c90a-43f2-8890-11ac185422bd" + }, + { + "cell_type": "code", + "source": "#Print_cong\n\nprint(conf_herwijnen)\nprint(conf_denhelder)\nprint(conf_radars)", + "metadata": {}, + "execution_count": null, + "outputs": [], + "id": "1b887d36-e009-46c0-9bc8-cbf633fb597d" + } + ] + } +} diff --git a/vreapis/tests/resources/notebooks/test_notebook.ipynb b/vreapis/tests/resources/notebooks/test_notebook.ipynb new file mode 100644 index 00000000..c4b4e90b --- /dev/null +++ b/vreapis/tests/resources/notebooks/test_notebook.ipynb @@ -0,0 +1,229 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "e4f3e05b-90d9-4f19-85ef-413125f854ca", + "metadata": {}, + "outputs": [], + "source": [ + "# Configurations\n", + "\n", + "import fnmatch\n", + "import json\n", + "import getpass\n", + "import os\n", + "import pathlib\n", + "import datetime\n", + " \n", + "from dask.distributed import LocalCluster, SSHCluster \n", + "from laserfarm import Retiler, DataProcessing, GeotiffWriter, MacroPipeline\n", + "from laserfarm.remote_utils import get_wdclient, get_info_remote, list_remote\n", + "\n", + "conf_remote_path_root = pathlib.Path('/webdav')\n", + "conf_remote_path_ahn = pathlib.Path('/webdav/ahn')\n", + "conf_remote_path_retiled = pathlib.Path('/webdav/retiled/')\n", + "conf_remote_path_targets = pathlib.Path('/webdav/targets')\n", + "conf_local_tmp = pathlib.Path('/tmp')\n", + "\n", + "param_hostname = 'https://param_hostname'\n", + "param_login = 'param_login'\n", + "param_password = 'param_password'\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "aceacd5f-e791-444f-8526-62b355b5a967", + "metadata": {}, + "outputs": [], + "source": [ + "# Fetch 1\n", + "laz_files = [f for f in list_remote(get_wdclient(conf_wd_opts), conf_remote_path_ahn.as_posix())\n", + " if f.lower().endswith('.laz')]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d28f21bb-21c6-4838-9cb8-2c174d0b70f3", + "metadata": {}, + "outputs": [], + "source": [ + "# fetch\n", + "import time\n", + "import requests\n", + "from laserfarm.remote_utils import get_wdclient, get_info_remote, list_remote\n", + " \n", + "laz_files = [f for f in list_remote(get_wdclient(conf_wd_opts), conf_remote_path_ahn.as_posix())\n", + " if f.lower().endswith('.laz')]\n", + "\n", + "def send_annotation(start=None,end=None,message=None,tags=None):\n", + " if not tags:\n", + " tags = []\n", + " \n", + " tags.append(theNotebook)\n", + " \n", + " headers = {\n", + " 'Accept':'application/json',\n", + " 'Content-Type': 'application/json',\n", + " }\n", + " \n", + " data ={\n", + " \"dashboardId\":1,\n", + " # \"panelId\":8,\n", + " \"time\":start,\n", + " \"timeEnd\":end,\n", + " \"created\": end,\n", + " \"tags\":tags,\n", + " \"text\": message\n", + " }\n", + " resp = requests.post(grafana_base_url+'/api/annotations',verify=False,auth=('admin', grafana_pwd),headers=headers,json=data)\n", + " \n", + " data ={\n", + " \"dashboardId\":2,\n", + " # \"panelId\":8,\n", + " \"time\":start,\n", + " \"timeEnd\":end,\n", + " \"created\": end,\n", + " \"tags\":tags,\n", + " \"text\": message\n", + " }\n", + " resp = requests.post(grafana_base_url+'/api/annotations',verify=False,auth=('admin', grafana_pwd),headers=headers,json=data)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "75bcb98f-22d0-47d1-a63f-cef4be7a4733", + "metadata": {}, + "outputs": [], + "source": [ + "# Feature Extraction spi\n", + "from laserfarm import Retiler, DataProcessing, GeotiffWriter, MacroPipeline\n", + "import time\n", + "import requests \n", + " \n", + "def send_annotation(start=None,end=None,message=None,tags=None):\n", + " if not tags:\n", + " tags = []\n", + " \n", + " tags.append(theNotebook)\n", + " \n", + " headers = {\n", + " 'Accept':'application/json',\n", + " 'Content-Type': 'application/json',\n", + " }\n", + " \n", + " data ={\n", + " \"dashboardId\":1,\n", + " # \"panelId\":8,\n", + " \"time\":start,\n", + " \"timeEnd\":end,\n", + " \"created\": end,\n", + " \"tags\":tags,\n", + " \"text\": message\n", + " }\n", + " resp = requests.post(grafana_base_url+'/api/annotations',verify=False,auth=('admin', grafana_pwd),headers=headers,json=data)\n", + " \n", + " data ={\n", + " \"dashboardId\":2,\n", + " # \"panelId\":8,\n", + " \"time\":start,\n", + " \"timeEnd\":end,\n", + " \"created\": end,\n", + " \"tags\":tags,\n", + " \"text\": message\n", + " }\n", + " resp = requests.post(grafana_base_url+'/api/annotations',verify=False,auth=('admin', grafana_pwd),headers=headers,json=data)\n", + " \n", + "\n", + "\n", + "t = tiles[2]\n", + "\n", + "tile_mesh_size = 10.\n", + "features = ['perc_95_normalized_height']\n", + "\n", + "grid_feature = {\n", + " 'min_x': -113107.81,\n", + " 'max_x': 398892.19,\n", + " 'min_y': 214783.87,\n", + " 'max_y': 726783.87,\n", + " 'n_tiles_side': 512\n", + "}\n", + "\n", + "feature_extraction_input = {\n", + " 'setup_local_fs': {'tmp_folder': conf_local_tmp.as_posix()},\n", + " 'pullremote': conf_remote_path_retiled.as_posix(),\n", + " 'load': {'attributes': ['raw_classification']},\n", + " 'normalize': 1,\n", + " 'apply_filter': {\n", + " 'filter_type': 'select_equal', \n", + " 'attribute': 'raw_classification',\n", + " 'value': [1, 6]#ground surface (2), water (9), buildings (6), artificial objects (26), vegetation (?), and unclassified (1)\n", + " },\n", + " 'generate_targets': {\n", + " 'tile_mesh_size' : tile_mesh_size,\n", + " 'validate' : True,\n", + " **grid_feature\n", + " },\n", + " 'extract_features': {\n", + " 'feature_names': features,\n", + " 'volume_type': 'cell',\n", + " 'volume_size': tile_mesh_size\n", + " },\n", + " 'export_targets': {\n", + " 'attributes': features,\n", + " 'multi_band_files': False\n", + " },\n", + " 'pushremote': conf_remote_path_targets.as_posix(),\n", + "# 'cleanlocalfs': {}\n", + "}\n", + "idx = (t.split('_')[1:])\n", + "processing = DataProcessing(t, tile_index=idx).config(feature_extraction_input).setup_webdav_client(conf_wd_opts)\n", + "processing.run()\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "34b843b6-5949-41ba-a412-5af3804d2540", + "metadata": {}, + "outputs": [], + "source": [ + "# title_with_underscore\n", + "\n", + "a = 1\n", + "\n", + "b = 2\n", + "\n", + "c = a + b\n", + "\n", + "c\n" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.10" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} + diff --git a/vreapis/tests/resources/notebooks/test_param_in_cell_notebook.json b/vreapis/tests/resources/notebooks/test_param_in_cell_notebook.json new file mode 100644 index 00000000..0ffa5e6b --- /dev/null +++ b/vreapis/tests/resources/notebooks/test_param_in_cell_notebook.json @@ -0,0 +1,159 @@ +{ + "save": false, + "kernel": "ipython", + "cell_index": 6, + "notebook": { + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.4" + } + }, + "nbformat_minor": 5, + "nbformat": 4, + "cells": [ + { + "cell_type": "code", + "source": "# a list\n\na_list = ['a','b','c','d']", + "metadata": { + "trusted": true + }, + "execution_count": 2, + "outputs": [], + "id": "5f68e3d0-8fa2-4237-8fe3-29d1ec2cfbc1" + }, + { + "cell_type": "code", + "source": "# process a list\n\nb_list = []\nfor elem in a_list:\n b_list.append(elem+'42')\n", + "metadata": { + "trusted": true + }, + "execution_count": 3, + "outputs": [], + "id": "e0621bdf-05ee-44b7-9519-c9b136b6ba62" + }, + { + "cell_type": "code", + "source": "# concat\n\nres = ''\nfor elem in b_list:\n res+=elem\n \nprint(res)\n ", + "metadata": { + "trusted": true + }, + "execution_count": 4, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": "a42b42c42d42\n" + } + ], + "id": "97969827-90fa-41fc-b32e-a3f8d583e08e" + }, + { + "cell_type": "code", + "source": "#verage\n\na = 2 \nnumbers = [a, 4, 6, 8, 10]\n", + "metadata": { + "trusted": true, + "tags": [] + }, + "execution_count": 1, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": "6\n" + } + ], + "id": "39790b85-3619-409d-af6c-a1096e36b751" + }, + { + "cell_type": "code", + "source": "# bb\nimport statistics\n\nave = statistics.mean(numbers)\n\n", + "metadata": { + "trusted": true + }, + "execution_count": null, + "outputs": [], + "id": "578545cc-8459-4092-a503-329928fc6ad1" + }, + { + "cell_type": "code", + "source": "# params\n\n", + "metadata": { + "trusted": true + }, + "execution_count": null, + "outputs": [], + "id": "de7cc7cf-56c9-4538-92be-1a258e5ee812" + }, + { + "cell_type": "code", + "source": "# Param check\n\n\nparam_density = 1\n\nif param_density == 1: \n param_CountingStrategy = 'density0'\n \n# check if mandatory fields are present\ndiameterofsedimentationchamber = 'diameterofsedimentationchamber'", + "metadata": { + "trusted": true, + "tags": [] + }, + "execution_count": 2, + "outputs": [], + "id": "9824a80c-b00b-441c-8152-7b2e671e081b" + }, + { + "cell_type": "code", + "source": "# print \n\nprint(param_biovolume)\nparam_totalbiovolume\nparam_density\nparam_surfacearea\nparam_surfacevolumeratio\nparam_cellcarboncontent\nparam_totalcarboncontent\nparam_CountingStrategy\n\nprint(diameterofsedimentationchamber)\ndiameterofsedimentationchamber\ndiameteroffieldofview\ntransectcounting\nnumberofcountedfields\nnumberoftransects\nsettlingvolume\ndilutionfactor", + "metadata": { + "trusted": true, + "tags": [] + }, + "execution_count": 3, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": "1\ndiameterofsedimentationchamber\n" + }, + { + "execution_count": 3, + "output_type": "execute_result", + "data": { + "text/plain": "'dilutionfactor'" + }, + "metadata": {} + } + ], + "id": "287f6cad-2029-4693-97da-5834db20c59e" + }, + { + "cell_type": "code", + "source": "", + "metadata": { + "trusted": true + }, + "execution_count": null, + "outputs": [], + "id": "c2b4ba69-640f-4708-8d4e-7f033721f20f" + }, + { + "cell_type": "code", + "source": "", + "metadata": { + "trusted": true + }, + "execution_count": null, + "outputs": [], + "id": "00dd23e8-531e-4dd8-90aa-43573c240a48" + } + ] + } +} diff --git a/vreapis/tests/resources/notebooks/test_param_outside_cell_notebook.json b/vreapis/tests/resources/notebooks/test_param_outside_cell_notebook.json new file mode 100644 index 00000000..afdd2046 --- /dev/null +++ b/vreapis/tests/resources/notebooks/test_param_outside_cell_notebook.json @@ -0,0 +1,159 @@ +{ + "save": false, + "kernel": "ipython", + "cell_index": 6, + "notebook": { + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.4" + } + }, + "nbformat_minor": 5, + "nbformat": 4, + "cells": [ + { + "cell_type": "code", + "source": "# a list\n\na_list = ['a','b','c','d']", + "metadata": { + "trusted": true + }, + "execution_count": 2, + "outputs": [], + "id": "5f68e3d0-8fa2-4237-8fe3-29d1ec2cfbc1" + }, + { + "cell_type": "code", + "source": "# process a list\n\nb_list = []\nfor elem in a_list:\n b_list.append(elem+'42')\n", + "metadata": { + "trusted": true + }, + "execution_count": 3, + "outputs": [], + "id": "e0621bdf-05ee-44b7-9519-c9b136b6ba62" + }, + { + "cell_type": "code", + "source": "# concat\n\nres = ''\nfor elem in b_list:\n res+=elem\n \nprint(res)\n ", + "metadata": { + "trusted": true + }, + "execution_count": 4, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": "a42b42c42d42\n" + } + ], + "id": "97969827-90fa-41fc-b32e-a3f8d583e08e" + }, + { + "cell_type": "code", + "source": "#verage\n\na = 2 \nnumbers = [a, 4, 6, 8, 10]\n", + "metadata": { + "trusted": true, + "tags": [] + }, + "execution_count": 1, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": "6\n" + } + ], + "id": "39790b85-3619-409d-af6c-a1096e36b751" + }, + { + "cell_type": "code", + "source": "# bb\nimport statistics\n\nave = statistics.mean(numbers)\n\n", + "metadata": { + "trusted": true + }, + "execution_count": null, + "outputs": [], + "id": "578545cc-8459-4092-a503-329928fc6ad1" + }, + { + "cell_type": "code", + "source": "# params\n\nparam_density = 1", + "metadata": { + "trusted": true + }, + "execution_count": null, + "outputs": [], + "id": "de7cc7cf-56c9-4538-92be-1a258e5ee812" + }, + { + "cell_type": "code", + "source": "# Param check\n\n\n\n\nif param_density == 1: \n param_CountingStrategy = 'density0'\n \n# check if mandatory fields are present\ndiameterofsedimentationchamber = 'diameterofsedimentationchamber'", + "metadata": { + "trusted": true, + "tags": [] + }, + "execution_count": 2, + "outputs": [], + "id": "9824a80c-b00b-441c-8152-7b2e671e081b" + }, + { + "cell_type": "code", + "source": "# print \n\nprint(param_biovolume)\nparam_totalbiovolume\nparam_density\nparam_surfacearea\nparam_surfacevolumeratio\nparam_cellcarboncontent\nparam_totalcarboncontent\nparam_CountingStrategy\n\nprint(diameterofsedimentationchamber)\ndiameterofsedimentationchamber\ndiameteroffieldofview\ntransectcounting\nnumberofcountedfields\nnumberoftransects\nsettlingvolume\ndilutionfactor", + "metadata": { + "trusted": true, + "tags": [] + }, + "execution_count": 3, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": "1\ndiameterofsedimentationchamber\n" + }, + { + "execution_count": 3, + "output_type": "execute_result", + "data": { + "text/plain": "'dilutionfactor'" + }, + "metadata": {} + } + ], + "id": "287f6cad-2029-4693-97da-5834db20c59e" + }, + { + "cell_type": "code", + "source": "", + "metadata": { + "trusted": true + }, + "execution_count": null, + "outputs": [], + "id": "c2b4ba69-640f-4708-8d4e-7f033721f20f" + }, + { + "cell_type": "code", + "source": "", + "metadata": { + "trusted": true + }, + "execution_count": null, + "outputs": [], + "id": "00dd23e8-531e-4dd8-90aa-43573c240a48" + } + ] + } +} diff --git a/vreapis/tests/resources/notebooks/test_param_values_Python.json b/vreapis/tests/resources/notebooks/test_param_values_Python.json new file mode 100644 index 00000000..044c1a84 --- /dev/null +++ b/vreapis/tests/resources/notebooks/test_param_values_Python.json @@ -0,0 +1,56 @@ +{ + "save": false, + "kernel": "ipython", + "cell_index": 1, + "notebook": { + "metadata": { + "kernelspec": { + "name": "conda-env-anaconda3-jupyterlab2-py", + "display_name": "Python [conda env:anaconda3-jupyterlab2]", + "language": "python" + }, + "language_info": { + "name": "python", + "version": "3.12.1", + "mimetype": "text/x-python", + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "pygments_lexer": "ipython3", + "nbconvert_exporter": "python", + "file_extension": ".py" + } + }, + "nbformat_minor": 5, + "nbformat": 4, + "cells": [ + { + "cell_type": "code", + "source": "# params\nparam_string = 'param_string value'\nparam_string_with_comment = 'param_string value' # comment\nparam_int = 1\nparam_float = 1.1\nparam_list = [1, 2, 3]", + "metadata": { + "trusted": true + }, + "execution_count": 1, + "outputs": [], + "id": "8519a0de-0023-40ab-8b53-b7342950fed3" + }, + { + "cell_type": "code", + "source": "# Param check\nprint(param_string)\nprint(param_string_with_comment)\nprint(param_int)\nprint(param_float)\nprint(param_list)", + "metadata": { + "trusted": true + }, + "execution_count": 2, + "outputs": [ + { + "name": "stdout", + "text": "param_string value\nparam_string value\n1\n1.1\n[1, 2, 3]\n", + "output_type": "stream" + } + ], + "id": "3c6969b8-ffd7-4f09-bb9a-01d1e01a834a" + } + ] + } +} \ No newline at end of file diff --git a/vreapis/tests/resources/notebooks/test_param_values_R.json b/vreapis/tests/resources/notebooks/test_param_values_R.json new file mode 100644 index 00000000..83bf34b1 --- /dev/null +++ b/vreapis/tests/resources/notebooks/test_param_values_R.json @@ -0,0 +1,59 @@ +{ + "save": false, + "kernel": "IRkernel", + "cell_index": 1, + "notebook": { + "metadata": { + "kernelspec": { + "display_name": "R", + "language": "R", + "name": "ir" + }, + "language_info": { + "codemirror_mode": "r", + "file_extension": ".r", + "mimetype": "text/x-r-source", + "name": "R", + "pygments_lexer": "r", + "version": "4.3.1" + } + }, + "nbformat_minor": 5, + "nbformat": 4, + "cells": [ + { + "cell_type": "code", + "source": [ + "# params\n", + "param_string = 'param_string value'\n", + "param_string_with_comment = 'param_string value' # comment\n", + "param_int = 1\n", + "param_float = 1.1" + ], + "metadata": { + "trusted": true + }, + "execution_count": null, + "outputs": [], + "id": "de7cc7cf-56c9-4538-92be-1a258e5ee812" + }, + { + "cell_type": "code", + "source": [ + "# Param check\n", + "print(param_string)\n", + "print(param_string_with_comment)\n", + "print(param_int)\n", + "print(param_float)" + ], + "metadata": { + "trusted": true, + "tags": [] + }, + "execution_count": null, + "outputs": [], + "id": "9824a80c-b00b-441c-8152-7b2e671e081b" + } + ] + } +} diff --git a/vreapis/tests/resources/search/keyword.json b/vreapis/tests/resources/search/keyword.json new file mode 100644 index 00000000..bd11dff1 --- /dev/null +++ b/vreapis/tests/resources/search/keyword.json @@ -0,0 +1,3 @@ +{ + "keyword": "data model" +} diff --git a/vreapis/tests/resources/workflows/NaaVRE/468f918f414f96df.json b/vreapis/tests/resources/workflows/NaaVRE/468f918f414f96df.json new file mode 100644 index 00000000..6306093e --- /dev/null +++ b/vreapis/tests/resources/workflows/NaaVRE/468f918f414f96df.json @@ -0,0 +1,160 @@ +{ + "chart": { + "offset": { + "x": 0, + "y": 0 + }, + "scale": 1, + "nodes": { + "442d10bc-d999-400b-8050-0289d8586031": { + "id": "442d10bc-d999-400b-8050-0289d8586031", + "position": { + "x": 414.66668701171875, + "y": 111 + }, + "orientation": 0, + "type": "input-output", + "ports": { + "rolling_mean_temp_str": { + "properties": { + "color": "#c5a987" + }, + "id": "rolling_mean_temp_str", + "type": "right", + "position": { + "x": 197.5, + "y": 60 + } + }, + "temperature_data_str": { + "properties": { + "color": "#aca953" + }, + "id": "temperature_data_str", + "type": "right", + "position": { + "x": 197.5, + "y": 88 + } + } + }, + "properties": { + "title": "Install and load the climwinb packages-dev-user-name-at-domain-com", + "vars": [ + { + "name": "rolling_mean_temp_str", + "direction": "output", + "type": "datatype", + "color": "#c5a987" + }, + { + "name": "temperature_data_str", + "direction": "output", + "type": "datatype", + "color": "#aca953" + } + ], + "params": [], + "inputs": [], + "outputs": [ + "rolling_mean_temp_str", + "temperature_data_str" + ], + "og_node_id": "7b4271d" + }, + "size": { + "width": 250, + "height": 150 + } + }, + "112085dc-408b-4b9d-b6e8-242257ffebea": { + "id": "112085dc-408b-4b9d-b6e8-242257ffebea", + "position": { + "x": 855.6666870117188, + "y": 76 + }, + "orientation": 0, + "type": "input-output", + "ports": { + "rolling_mean_temp_str": { + "properties": { + "color": "#c5a987" + }, + "id": "rolling_mean_temp_str", + "type": "left", + "position": { + "x": 50.5, + "y": 60 + } + }, + "temperature_data_str": { + "properties": { + "color": "#aca953" + }, + "id": "temperature_data_str", + "type": "left", + "position": { + "x": 50.5, + "y": 88 + } + } + }, + "properties": { + "title": "consume climwin-dev-user-name-at-domain-com", + "vars": [ + { + "name": "rolling_mean_temp_str", + "direction": "input", + "type": "datatype", + "color": "#c5a987" + }, + { + "name": "temperature_data_str", + "direction": "input", + "type": "datatype", + "color": "#aca953" + } + ], + "params": [], + "inputs": [ + "rolling_mean_temp_str", + "temperature_data_str" + ], + "outputs": [], + "og_node_id": "dd373f8" + }, + "size": { + "width": 250, + "height": 150 + } + } + }, + "links": { + "a88cacb6-cfe8-4c71-a116-c2f2ad919863": { + "id": "a88cacb6-cfe8-4c71-a116-c2f2ad919863", + "from": { + "nodeId": "442d10bc-d999-400b-8050-0289d8586031", + "portId": "rolling_mean_temp_str" + }, + "to": { + "nodeId": "112085dc-408b-4b9d-b6e8-242257ffebea", + "portId": "rolling_mean_temp_str" + } + }, + "4c2d5463-1da4-4a06-838f-c20a5cd433ca": { + "id": "4c2d5463-1da4-4a06-838f-c20a5cd433ca", + "from": { + "nodeId": "442d10bc-d999-400b-8050-0289d8586031", + "portId": "temperature_data_str" + }, + "to": { + "nodeId": "112085dc-408b-4b9d-b6e8-242257ffebea", + "portId": "temperature_data_str" + } + } + }, + "selected": {}, + "hovered": {} + }, + "params": {} +} diff --git a/vreapis/tests/resources/workflows/NaaVRE/test_R_workflow.json b/vreapis/tests/resources/workflows/NaaVRE/test_R_workflow.json new file mode 100644 index 00000000..ad6233d0 --- /dev/null +++ b/vreapis/tests/resources/workflows/NaaVRE/test_R_workflow.json @@ -0,0 +1,844 @@ +{ + "chart": { + "offset": { + "x": 0, + "y": 0 + }, + "scale": 1, + "nodes": { + "12883ad5-9f59-47c9-bd6d-8063db9c8ca8": { + "id": "12883ad5-9f59-47c9-bd6d-8063db9c8ca8", + "position": { + "x": 0, + "y": 0 + }, + "orientation": 0, + "type": "input-output", + "ports": { + "file_path": { + "properties": { + "color": "#663a78" + }, + "id": "file_path", + "type": "right", + "position": { + "x": 217.5, + "y": 74.5 + } + } + }, + "properties": { + "title": "Create file R-dev-user-name-at-domain-com", + "vars": [ + { + "name": "file_path", + "direction": "output", + "type": "datatype", + "color": "#663a78" + } + ], + "params": [], + "inputs": [], + "outputs": [ + "file_path" + ], + "og_node_id": "c45fb67" + }, + "size": { + "width": 250, + "height": 150 + } + }, + "f8326a6b-a14d-4762-a738-1e51e648665e": { + "id": "f8326a6b-a14d-4762-a738-1e51e648665e", + "position": { + "x": 485.1721079707927, + "y": 0 + }, + "orientation": 0, + "type": "input-output", + "ports": { + "file_path": { + "properties": { + "color": "#663a78" + }, + "id": "file_path", + "type": "left", + "position": { + "x": 30.5, + "y": 74.5 + } + }, + "lines": { + "properties": { + "color": "#ac535c" + }, + "id": "lines", + "type": "right", + "position": { + "x": 231, + "y": 74.5 + } + } + }, + "properties": { + "title": "read file lines R-dev-user-name-at-domain-com", + "vars": [ + { + "name": "file_path", + "direction": "input", + "type": "datatype", + "color": "#663a78" + }, + { + "name": "lines", + "direction": "output", + "type": "datatype", + "color": "#ac535c" + } + ], + "params": [], + "inputs": [ + "file_path" + ], + "outputs": [ + "lines" + ], + "og_node_id": "47b2d97" + }, + "size": { + "width": 250, + "height": 150 + } + }, + "a3363399-53a3-49f4-97a2-405db0ac69f7": { + "id": "a3363399-53a3-49f4-97a2-405db0ac69f7", + "position": { + "x": 494.3224347681782, + "y": 173.46405228758172 + }, + "orientation": 0, + "type": "input-output", + "ports": { + "file_path": { + "properties": { + "color": "#663a78" + }, + "id": "file_path", + "type": "left", + "position": { + "x": 30.5, + "y": 74.5 + } + }, + "lines": { + "properties": { + "color": "#ac535c" + }, + "id": "lines", + "type": "right", + "position": { + "x": 231, + "y": 74.5 + } + } + }, + "properties": { + "title": "read file lines R-dev-user-name-at-domain-com", + "vars": [ + { + "name": "file_path", + "direction": "input", + "type": "datatype", + "color": "#663a78" + }, + { + "name": "lines", + "direction": "output", + "type": "datatype", + "color": "#ac535c" + } + ], + "params": [], + "inputs": [ + "file_path" + ], + "outputs": [ + "lines" + ], + "og_node_id": "47b2d97" + }, + "size": { + "width": 250, + "height": 150 + } + }, + "9f57774e-1133-4b8e-ab8e-2882836d564c": { + "id": "9f57774e-1133-4b8e-ab8e-2882836d564c", + "position": { + "x": 873.4074020884393, + "y": 24.44444444444442 + }, + "orientation": 0, + "type": "splitter", + "ports": { + "splitter_source": { + "id": "splitter_source", + "type": "left", + "properties": { + "special_node": 1, + "color": "#000000" + }, + "position": { + "x": 0.5, + "y": 49.5 + } + }, + "splitter_target": { + "id": "splitter_target", + "type": "right", + "properties": { + "special_node": 1, + "color": "#000000" + }, + "position": { + "x": 197.5, + "y": 49.5 + } + } + }, + "properties": { + "title": "Splitter", + "scalingFactor": 1 + }, + "size": { + "width": 200, + "height": 100 + } + }, + "ecfb5854-8511-47db-8cc8-d26f0b63bc8f": { + "id": "ecfb5854-8511-47db-8cc8-d26f0b63bc8f", + "position": { + "x": 868.1786439185047, + "y": 186.53594771241845 + }, + "orientation": 0, + "type": "input-output", + "ports": { + "lines": { + "properties": { + "color": "#ac535c" + }, + "id": "lines", + "type": "left", + "position": { + "x": 17, + "y": 74.5 + } + }, + "count": { + "properties": { + "color": "#783d3a" + }, + "id": "count", + "type": "right", + "position": { + "x": 228, + "y": 74.5 + } + } + }, + "properties": { + "title": "loop file lines R-dev-user-name-at-domain-com", + "vars": [ + { + "name": "lines", + "direction": "input", + "type": "datatype", + "color": "#ac535c" + }, + { + "name": "count", + "direction": "output", + "type": "datatype", + "color": "#783d3a" + } + ], + "params": [], + "inputs": [ + "lines" + ], + "outputs": [ + "count" + ], + "og_node_id": "f893a78" + }, + "size": { + "width": 250, + "height": 150 + } + }, + "0d044bb6-3457-41df-8796-304f57a76a25": { + "id": "0d044bb6-3457-41df-8796-304f57a76a25", + "position": { + "x": 1223.7341994740607, + "y": 0 + }, + "orientation": 0, + "type": "input-output", + "ports": { + "lines": { + "properties": { + "color": "#ac535c" + }, + "id": "lines", + "type": "left", + "position": { + "x": 17, + "y": 74.5 + } + }, + "count": { + "properties": { + "color": "#783d3a" + }, + "id": "count", + "type": "right", + "position": { + "x": 228, + "y": 74.5 + } + } + }, + "properties": { + "title": "loop file lines R-dev-user-name-at-domain-com", + "vars": [ + { + "name": "lines", + "direction": "input", + "type": "datatype", + "color": "#ac535c" + }, + { + "name": "count", + "direction": "output", + "type": "datatype", + "color": "#783d3a" + } + ], + "params": [], + "inputs": [ + "lines" + ], + "outputs": [ + "count" + ], + "og_node_id": "f893a78" + }, + "size": { + "width": 250, + "height": 150 + } + }, + "6b3fdf4f-69f5-48b4-addc-1666407cdbe0": { + "id": "6b3fdf4f-69f5-48b4-addc-1666407cdbe0", + "position": { + "x": 1260.3355066636025, + "y": 187.84313725490202 + }, + "orientation": 0, + "type": "input-output", + "ports": { + "count": { + "properties": { + "color": "#783d3a" + }, + "id": "count", + "type": "left", + "position": { + "x": 20, + "y": 74.5 + } + } + }, + "properties": { + "title": "Add1 R-dev-user-name-at-domain-com", + "vars": [ + { + "name": "count", + "direction": "input", + "type": "datatype", + "color": "#783d3a" + } + ], + "params": [], + "inputs": [ + "count" + ], + "outputs": [], + "og_node_id": "d9acfa2" + }, + "size": { + "width": 250, + "height": 150 + } + }, + "0da9edb7-eb4b-4edb-a314-ebfd34ba56ea": { + "id": "0da9edb7-eb4b-4edb-a314-ebfd34ba56ea", + "position": { + "x": 832.8845262714459, + "y": 419.2156862745097 + }, + "orientation": 0, + "type": "input-output", + "ports": { + "list_of_ints": { + "properties": { + "color": "#5f1f93" + }, + "id": "list_of_ints", + "type": "right", + "position": { + "x": 204, + "y": 60.5 + } + }, + "list_of_paths": { + "properties": { + "color": "#79c5d2" + }, + "id": "list_of_paths", + "type": "right", + "position": { + "x": 204, + "y": 88.5 + } + } + }, + "properties": { + "title": "input lists R-dev-user-name-at-domain-com", + "vars": [ + { + "name": "list_of_ints", + "direction": "output", + "type": "datatype", + "color": "#5f1f93" + }, + { + "name": "list_of_paths", + "direction": "output", + "type": "datatype", + "color": "#79c5d2" + } + ], + "params": [], + "inputs": [], + "outputs": [ + "list_of_ints", + "list_of_paths" + ], + "og_node_id": "3cfdc6f" + }, + "size": { + "width": 250, + "height": 150 + } + }, + "7b288a3c-672e-4069-95f2-37a044600119": { + "id": "7b288a3c-672e-4069-95f2-37a044600119", + "position": { + "x": 1277.3289707158906, + "y": 476.73202614379073 + }, + "orientation": 0, + "type": "splitter", + "ports": { + "splitter_source": { + "id": "splitter_source", + "type": "left", + "properties": { + "special_node": 1, + "color": "#000000" + }, + "position": { + "x": 0.5, + "y": 49.5 + } + }, + "splitter_target": { + "id": "splitter_target", + "type": "right", + "properties": { + "special_node": 1, + "color": "#000000" + }, + "position": { + "x": 197.5, + "y": 49.5 + } + } + }, + "properties": { + "title": "Splitter", + "scalingFactor": 1 + }, + "size": { + "width": 200, + "height": 100 + } + }, + "d4c8fac5-606b-4d39-8fcd-d93e2603a6a7": { + "id": "d4c8fac5-606b-4d39-8fcd-d93e2603a6a7", + "position": { + "x": 1264.2570752910538, + "y": 353.85620915032683 + }, + "orientation": 0, + "type": "splitter", + "ports": { + "splitter_source": { + "id": "splitter_source", + "type": "left", + "properties": { + "special_node": 1, + "color": "#000000" + }, + "position": { + "x": 0.5, + "y": 49.5 + } + }, + "splitter_target": { + "id": "splitter_target", + "type": "right", + "properties": { + "special_node": 1, + "color": "#000000" + }, + "position": { + "x": 197.5, + "y": 49.5 + } + } + }, + "properties": { + "title": "Splitter", + "scalingFactor": 1 + }, + "size": { + "width": 200, + "height": 100 + } + }, + "27ff8437-75d0-4f60-b73a-e33e53d86dda": { + "id": "27ff8437-75d0-4f60-b73a-e33e53d86dda", + "position": { + "x": 1602.819166794322, + "y": 322.4836601307189 + }, + "orientation": 0, + "type": "input-output", + "ports": { + "list_of_ints": { + "properties": { + "color": "#5f1f93" + }, + "id": "list_of_ints", + "type": "left", + "position": { + "x": 37.5, + "y": 74.5 + } + } + }, + "properties": { + "title": "loop int list R-dev-user-name-at-domain-com", + "vars": [ + { + "name": "list_of_ints", + "direction": "input", + "type": "datatype", + "color": "#5f1f93" + } + ], + "params": [], + "inputs": [ + "list_of_ints" + ], + "outputs": [], + "og_node_id": "3285abd" + }, + "size": { + "width": 250, + "height": 150 + } + }, + "0a44c471-853b-4f86-b69b-17b41a079591": { + "id": "0a44c471-853b-4f86-b69b-17b41a079591", + "position": { + "x": 1649.8779903237335, + "y": 143.39869281045745 + }, + "orientation": 0, + "type": "input-output", + "ports": { + "list_of_ints": { + "properties": { + "color": "#5f1f93" + }, + "id": "list_of_ints", + "type": "left", + "position": { + "x": 37.5, + "y": 74.5 + } + } + }, + "properties": { + "title": "loop int list R-dev-user-name-at-domain-com", + "vars": [ + { + "name": "list_of_ints", + "direction": "input", + "type": "datatype", + "color": "#5f1f93" + } + ], + "params": [], + "inputs": [ + "list_of_ints" + ], + "outputs": [], + "og_node_id": "3285abd" + }, + "size": { + "width": 250, + "height": 150 + } + }, + "9be7f1a9-7fb8-4671-b9c5-0b9aa6263dc2": { + "id": "9be7f1a9-7fb8-4671-b9c5-0b9aa6263dc2", + "position": { + "x": 1609.3551145067397, + "y": 492.4183006535946 + }, + "orientation": 0, + "type": "input-output", + "ports": { + "list_of_paths": { + "properties": { + "color": "#79c5d2" + }, + "id": "list_of_paths", + "type": "left", + "position": { + "x": 44, + "y": 74.5 + } + } + }, + "properties": { + "title": "loop list R-dev-user-name-at-domain-com", + "vars": [ + { + "name": "list_of_paths", + "direction": "input", + "type": "datatype", + "color": "#79c5d2" + } + ], + "params": [], + "inputs": [ + "list_of_paths" + ], + "outputs": [], + "og_node_id": "943e143" + }, + "size": { + "width": 250, + "height": 150 + } + }, + "beb396d1-358b-4dcc-952d-e140acb5f13d": { + "id": "beb396d1-358b-4dcc-952d-e140acb5f13d", + "position": { + "x": 1524.3877942453023, + "y": 722.4836601307189 + }, + "orientation": 0, + "type": "input-output", + "ports": { + "list_of_paths": { + "properties": { + "color": "#79c5d2" + }, + "id": "list_of_paths", + "type": "left", + "position": { + "x": 44, + "y": 74.5 + } + } + }, + "properties": { + "title": "loop list R-dev-user-name-at-domain-com", + "vars": [ + { + "name": "list_of_paths", + "direction": "input", + "type": "datatype", + "color": "#79c5d2" + } + ], + "params": [], + "inputs": [ + "list_of_paths" + ], + "outputs": [], + "og_node_id": "943e143" + }, + "size": { + "width": 250, + "height": 150 + } + } + }, + "links": { + "f7cdf578-7c18-4486-9579-6398c4b171be": { + "id": "f7cdf578-7c18-4486-9579-6398c4b171be", + "from": { + "nodeId": "12883ad5-9f59-47c9-bd6d-8063db9c8ca8", + "portId": "file_path" + }, + "to": { + "nodeId": "f8326a6b-a14d-4762-a738-1e51e648665e", + "portId": "file_path" + } + }, + "2bd830fb-4305-4e56-9667-14ef6663dbc2": { + "id": "2bd830fb-4305-4e56-9667-14ef6663dbc2", + "from": { + "nodeId": "12883ad5-9f59-47c9-bd6d-8063db9c8ca8", + "portId": "file_path" + }, + "to": { + "nodeId": "a3363399-53a3-49f4-97a2-405db0ac69f7", + "portId": "file_path" + } + }, + "e4210144-6b90-427a-9a27-1408aa7f8062": { + "id": "e4210144-6b90-427a-9a27-1408aa7f8062", + "from": { + "nodeId": "f8326a6b-a14d-4762-a738-1e51e648665e", + "portId": "lines" + }, + "to": { + "nodeId": "9f57774e-1133-4b8e-ab8e-2882836d564c", + "portId": "splitter_source" + } + }, + "b25dc7dd-1015-4b35-bef9-54baadd95beb": { + "id": "b25dc7dd-1015-4b35-bef9-54baadd95beb", + "from": { + "nodeId": "a3363399-53a3-49f4-97a2-405db0ac69f7", + "portId": "lines" + }, + "to": { + "nodeId": "ecfb5854-8511-47db-8cc8-d26f0b63bc8f", + "portId": "lines" + } + }, + "020b28ad-59c0-46d9-b345-d27f729b29c1": { + "id": "020b28ad-59c0-46d9-b345-d27f729b29c1", + "from": { + "nodeId": "9f57774e-1133-4b8e-ab8e-2882836d564c", + "portId": "splitter_target" + }, + "to": { + "nodeId": "0d044bb6-3457-41df-8796-304f57a76a25", + "portId": "lines" + } + }, + "7916ab47-a8b1-4c38-9001-cf81298d4655": { + "id": "7916ab47-a8b1-4c38-9001-cf81298d4655", + "from": { + "nodeId": "ecfb5854-8511-47db-8cc8-d26f0b63bc8f", + "portId": "count" + }, + "to": { + "nodeId": "6b3fdf4f-69f5-48b4-addc-1666407cdbe0", + "portId": "count" + } + }, + "2d2b3c9c-02be-4a20-84c6-c8018f155c1f": { + "id": "2d2b3c9c-02be-4a20-84c6-c8018f155c1f", + "from": { + "nodeId": "0da9edb7-eb4b-4edb-a314-ebfd34ba56ea", + "portId": "list_of_ints" + }, + "to": { + "nodeId": "d4c8fac5-606b-4d39-8fcd-d93e2603a6a7", + "portId": "splitter_source" + } + }, + "c79aee7c-2516-40c4-a149-7a413df6db5e": { + "id": "c79aee7c-2516-40c4-a149-7a413df6db5e", + "from": { + "nodeId": "0da9edb7-eb4b-4edb-a314-ebfd34ba56ea", + "portId": "list_of_paths" + }, + "to": { + "nodeId": "7b288a3c-672e-4069-95f2-37a044600119", + "portId": "splitter_source" + } + }, + "e7e5aeaa-6222-406f-b8e3-ccc0b7a83bc7": { + "id": "e7e5aeaa-6222-406f-b8e3-ccc0b7a83bc7", + "from": { + "nodeId": "d4c8fac5-606b-4d39-8fcd-d93e2603a6a7", + "portId": "splitter_target" + }, + "to": { + "nodeId": "27ff8437-75d0-4f60-b73a-e33e53d86dda", + "portId": "list_of_ints" + } + }, + "fa489149-1708-4993-bf6f-099596614383": { + "id": "fa489149-1708-4993-bf6f-099596614383", + "from": { + "nodeId": "0da9edb7-eb4b-4edb-a314-ebfd34ba56ea", + "portId": "list_of_ints" + }, + "to": { + "nodeId": "0a44c471-853b-4f86-b69b-17b41a079591", + "portId": "list_of_ints" + } + }, + "172cf546-272f-4247-9488-5a7aa226fd0b": { + "id": "172cf546-272f-4247-9488-5a7aa226fd0b", + "from": { + "nodeId": "7b288a3c-672e-4069-95f2-37a044600119", + "portId": "splitter_target" + }, + "to": { + "nodeId": "9be7f1a9-7fb8-4671-b9c5-0b9aa6263dc2", + "portId": "list_of_paths" + } + }, + "8c1ed73b-b34b-48bc-bf2a-6af3aff5df27": { + "id": "8c1ed73b-b34b-48bc-bf2a-6af3aff5df27", + "from": { + "nodeId": "0da9edb7-eb4b-4edb-a314-ebfd34ba56ea", + "portId": "list_of_paths" + }, + "to": { + "nodeId": "beb396d1-358b-4dcc-952d-e140acb5f13d", + "portId": "list_of_paths" + } + } + }, + "selected": {}, + "hovered": {} + }, + "params": {} +} diff --git a/vreapis/tests/resources/workflows/NaaVRE/test_py_workflow.json b/vreapis/tests/resources/workflows/NaaVRE/test_py_workflow.json new file mode 100644 index 00000000..5a63be06 --- /dev/null +++ b/vreapis/tests/resources/workflows/NaaVRE/test_py_workflow.json @@ -0,0 +1,1378 @@ +{ + "chart": { + "offset": { + "x": 0, + "y": 0 + }, + "scale": 1, + "nodes": { + "e05a69bc-d0a4-4f79-93e2-7c79ba2b04e1": { + "id": "e05a69bc-d0a4-4f79-93e2-7c79ba2b04e1", + "position": { + "x": 0, + "y": 0 + }, + "orientation": 0, + "type": "input-output", + "ports": { + "file_path": { + "properties": { + "color": "#663a78" + }, + "id": "file_path", + "type": "right", + "position": { + "x": 217.5, + "y": 74.5 + } + } + }, + "properties": { + "title": "Create file-dev-user-name-at-domain-com", + "vars": [ + { + "name": "file_path", + "direction": "output", + "type": "datatype", + "color": "#663a78" + } + ], + "params": [], + "inputs": [], + "outputs": [ + "file_path" + ], + "og_node_id": "5a918c8" + }, + "size": { + "width": 250, + "height": 150 + } + }, + "7d1c8e18-880d-4bab-be5c-dd47d5eed5b4": { + "id": "7d1c8e18-880d-4bab-be5c-dd47d5eed5b4", + "position": { + "x": 365.51197725183823, + "y": 0 + }, + "orientation": 0, + "type": "input-output", + "ports": { + "file_path": { + "properties": { + "color": "#663a78" + }, + "id": "file_path", + "type": "left", + "position": { + "x": 30.5, + "y": 74.5 + } + }, + "lines": { + "properties": { + "color": "#ac535c" + }, + "id": "lines", + "type": "right", + "position": { + "x": 231, + "y": 74.5 + } + } + }, + "properties": { + "title": "read file lines-dev-user-name-at-domain-com", + "vars": [ + { + "name": "file_path", + "direction": "input", + "type": "datatype", + "color": "#663a78" + }, + { + "name": "lines", + "direction": "output", + "type": "datatype", + "color": "#ac535c" + } + ], + "params": [], + "inputs": [ + "file_path" + ], + "outputs": [ + "lines" + ], + "og_node_id": "4deac70" + }, + "size": { + "width": 250, + "height": 150 + } + }, + "f847ca0a-1a4d-4e2a-85bd-6ec7be87c3ed": { + "id": "f847ca0a-1a4d-4e2a-85bd-6ec7be87c3ed", + "position": { + "x": 364.20478770935455, + "y": 177.11111111111117 + }, + "orientation": 0, + "type": "input-output", + "ports": { + "file_path": { + "properties": { + "color": "#663a78" + }, + "id": "file_path", + "type": "left", + "position": { + "x": 30.5, + "y": 74.5 + } + }, + "lines": { + "properties": { + "color": "#ac535c" + }, + "id": "lines", + "type": "right", + "position": { + "x": 231, + "y": 74.5 + } + } + }, + "properties": { + "title": "read file lines-dev-user-name-at-domain-com", + "vars": [ + { + "name": "file_path", + "direction": "input", + "type": "datatype", + "color": "#663a78" + }, + { + "name": "lines", + "direction": "output", + "type": "datatype", + "color": "#ac535c" + } + ], + "params": [], + "inputs": [ + "file_path" + ], + "outputs": [ + "lines" + ], + "og_node_id": "4deac70" + }, + "size": { + "width": 250, + "height": 150 + } + }, + "8f44f21c-829c-4abc-881f-377159a2d2b1": { + "id": "8f44f21c-829c-4abc-881f-377159a2d2b1", + "position": { + "x": 697.5381210426875, + "y": 17.63398692810455 + }, + "orientation": 0, + "type": "splitter", + "ports": { + "splitter_source": { + "id": "splitter_source", + "type": "left", + "properties": { + "special_node": 1, + "color": "#000000" + }, + "position": { + "x": 0.5, + "y": 49.5 + } + }, + "splitter_target": { + "id": "splitter_target", + "type": "right", + "properties": { + "special_node": 1, + "color": "#000000" + }, + "position": { + "x": 197.5, + "y": 49.5 + } + } + }, + "properties": { + "title": "Splitter", + "scalingFactor": 1 + }, + "size": { + "width": 200, + "height": 100 + } + }, + "1d67688c-536d-48ef-a40f-4b0aaa3f3638": { + "id": "1d67688c-536d-48ef-a40f-4b0aaa3f3638", + "position": { + "x": 1000.8060948988971, + "y": 3.254901960784335 + }, + "orientation": 0, + "type": "input-output", + "ports": { + "lines": { + "properties": { + "color": "#ac535c" + }, + "id": "lines", + "type": "left", + "position": { + "x": 17, + "y": 74.5 + } + }, + "count": { + "properties": { + "color": "#783d3a" + }, + "id": "count", + "type": "right", + "position": { + "x": 228, + "y": 74.5 + } + } + }, + "properties": { + "title": "loop file lines-dev-user-name-at-domain-com", + "vars": [ + { + "name": "lines", + "direction": "input", + "type": "datatype", + "color": "#ac535c" + }, + { + "name": "count", + "direction": "output", + "type": "datatype", + "color": "#783d3a" + } + ], + "params": [], + "inputs": [ + "lines" + ], + "outputs": [ + "count" + ], + "og_node_id": "5d9de73" + }, + "size": { + "width": 250, + "height": 150 + } + }, + "473de030-05fc-4f09-9325-11b838bcc140": { + "id": "473de030-05fc-4f09-9325-11b838bcc140", + "position": { + "x": 982.5054413041258, + "y": 290.8366013071896 + }, + "orientation": 0, + "type": "input-output", + "ports": { + "lines": { + "properties": { + "color": "#ac535c" + }, + "id": "lines", + "type": "left", + "position": { + "x": 17, + "y": 74.5 + } + }, + "count": { + "properties": { + "color": "#783d3a" + }, + "id": "count", + "type": "right", + "position": { + "x": 228, + "y": 74.5 + } + } + }, + "properties": { + "title": "loop file lines-dev-user-name-at-domain-com", + "vars": [ + { + "name": "lines", + "direction": "input", + "type": "datatype", + "color": "#ac535c" + }, + { + "name": "count", + "direction": "output", + "type": "datatype", + "color": "#783d3a" + } + ], + "params": [], + "inputs": [ + "lines" + ], + "outputs": [ + "count" + ], + "og_node_id": "5d9de73" + }, + "size": { + "width": 250, + "height": 150 + } + }, + "62fe958c-92d6-4be1-bbcc-73bfe88b9e2a": { + "id": "62fe958c-92d6-4be1-bbcc-73bfe88b9e2a", + "position": { + "x": 1349.8257027420339, + "y": 177.77777777777786 + }, + "orientation": 0, + "type": "input-output", + "ports": { + "count": { + "properties": { + "color": "#783d3a" + }, + "id": "count", + "type": "left", + "position": { + "x": 20, + "y": 74.5 + } + }, + "a": { + "properties": { + "color": "#77862d" + }, + "id": "a", + "type": "right", + "position": { + "x": 243, + "y": 74.5 + } + } + }, + "properties": { + "title": "Add1-dev-user-name-at-domain-com", + "vars": [ + { + "name": "count", + "direction": "input", + "type": "datatype", + "color": "#783d3a" + }, + { + "name": "a", + "direction": "output", + "type": "datatype", + "color": "#77862d" + } + ], + "params": [], + "inputs": [ + "count" + ], + "outputs": [ + "a" + ], + "og_node_id": "49d5dda" + }, + "size": { + "width": 250, + "height": 150 + } + }, + "57c8b452-35f5-4b7b-b777-e2bee59ef99d": { + "id": "57c8b452-35f5-4b7b-b777-e2bee59ef99d", + "position": { + "x": 1694.9237419577207, + "y": 268.61437908496737 + }, + "orientation": 0, + "type": "input-output", + "ports": { + "a": { + "properties": { + "color": "#77862d" + }, + "id": "a", + "type": "left", + "position": { + "x": 20, + "y": 60.5 + } + }, + "count": { + "properties": { + "color": "#783d3a" + }, + "id": "count", + "type": "left", + "position": { + "x": 20, + "y": 88.5 + } + }, + "msg": { + "properties": { + "color": "#7bd279" + }, + "id": "msg", + "type": "right", + "position": { + "x": 232.5, + "y": 74.5 + } + } + }, + "properties": { + "title": "Anti-pattern-dev-user-name-at-domain-com", + "vars": [ + { + "name": "a", + "direction": "input", + "type": "datatype", + "color": "#77862d" + }, + { + "name": "count", + "direction": "input", + "type": "datatype", + "color": "#783d3a" + }, + { + "name": "msg", + "direction": "output", + "type": "datatype", + "color": "#7bd279" + } + ], + "params": [], + "inputs": [ + "a", + "count" + ], + "outputs": [ + "msg" + ], + "og_node_id": "44dec46" + }, + "size": { + "width": 250, + "height": 150 + } + }, + "12ae508b-b472-4924-af9a-a3051c7612fa": { + "id": "12ae508b-b472-4924-af9a-a3051c7612fa", + "position": { + "x": 2223.028317121121, + "y": 103.90849673202612 + }, + "orientation": 0, + "type": "input-output", + "ports": { + "msg": { + "properties": { + "color": "#7bd279" + }, + "id": "msg", + "type": "left", + "position": { + "x": 15.5, + "y": 74.5 + } + }, + "list_of_ints": { + "properties": { + "color": "#5f1f93" + }, + "id": "list_of_ints", + "type": "right", + "position": { + "x": 204, + "y": 60.5 + } + }, + "list_of_paths": { + "properties": { + "color": "#79c5d2" + }, + "id": "list_of_paths", + "type": "right", + "position": { + "x": 204, + "y": 88.5 + } + } + }, + "properties": { + "title": "input lists-dev-user-name-at-domain-com", + "vars": [ + { + "name": "msg", + "direction": "input", + "type": "datatype", + "color": "#7bd279" + }, + { + "name": "list_of_ints", + "direction": "output", + "type": "datatype", + "color": "#5f1f93" + }, + { + "name": "list_of_paths", + "direction": "output", + "type": "datatype", + "color": "#79c5d2" + } + ], + "params": [], + "inputs": [ + "msg" + ], + "outputs": [ + "list_of_ints", + "list_of_paths" + ], + "og_node_id": "3e4f548" + }, + "size": { + "width": 250, + "height": 150 + } + }, + "c8cf52ae-446f-402e-8ea5-d26dbf4ff903": { + "id": "c8cf52ae-446f-402e-8ea5-d26dbf4ff903", + "position": { + "x": 2203.420473983864, + "y": 302.6013071895425 + }, + "orientation": 0, + "type": "input-output", + "ports": { + "msg": { + "properties": { + "color": "#7bd279" + }, + "id": "msg", + "type": "left", + "position": { + "x": 15.5, + "y": 74.5 + } + }, + "list_of_ints": { + "properties": { + "color": "#5f1f93" + }, + "id": "list_of_ints", + "type": "right", + "position": { + "x": 204, + "y": 60.5 + } + }, + "list_of_paths": { + "properties": { + "color": "#79c5d2" + }, + "id": "list_of_paths", + "type": "right", + "position": { + "x": 204, + "y": 88.5 + } + } + }, + "properties": { + "title": "input lists-dev-user-name-at-domain-com", + "vars": [ + { + "name": "msg", + "direction": "input", + "type": "datatype", + "color": "#7bd279" + }, + { + "name": "list_of_ints", + "direction": "output", + "type": "datatype", + "color": "#5f1f93" + }, + { + "name": "list_of_paths", + "direction": "output", + "type": "datatype", + "color": "#79c5d2" + } + ], + "params": [], + "inputs": [ + "msg" + ], + "outputs": [ + "list_of_ints", + "list_of_paths" + ], + "og_node_id": "3e4f548" + }, + "size": { + "width": 250, + "height": 150 + } + }, + "c0f6bebc-1529-4901-85d7-42eb1342868e": { + "id": "c0f6bebc-1529-4901-85d7-42eb1342868e", + "position": { + "x": 2617.799558951182, + "y": 0 + }, + "orientation": 0, + "type": "splitter", + "ports": { + "splitter_source": { + "id": "splitter_source", + "type": "left", + "properties": { + "special_node": 1, + "color": "#000000" + }, + "position": { + "x": 0.5, + "y": 49.5 + } + }, + "splitter_target": { + "id": "splitter_target", + "type": "right", + "properties": { + "special_node": 1, + "color": "#000000" + }, + "position": { + "x": 197.5, + "y": 49.5 + } + } + }, + "properties": { + "title": "Splitter", + "scalingFactor": 1 + }, + "size": { + "width": 200, + "height": 100 + } + }, + "edd68621-780a-42df-958c-74f0a711b260": { + "id": "edd68621-780a-42df-958c-74f0a711b260", + "position": { + "x": 2634.793023003473, + "y": 145.73856209150335 + }, + "orientation": 0, + "type": "splitter", + "ports": { + "splitter_source": { + "id": "splitter_source", + "type": "left", + "properties": { + "special_node": 1, + "color": "#000000" + }, + "position": { + "x": 0.5, + "y": 49.5 + } + }, + "splitter_target": { + "id": "splitter_target", + "type": "right", + "properties": { + "special_node": 1, + "color": "#000000" + }, + "position": { + "x": 197.5, + "y": 49.5 + } + } + }, + "properties": { + "title": "Splitter", + "scalingFactor": 1 + }, + "size": { + "width": 200, + "height": 100 + } + }, + "1a85bb42-8eba-4b46-b691-ac0687985fab": { + "id": "1a85bb42-8eba-4b46-b691-ac0687985fab", + "position": { + "x": 3088.387794245301, + "y": 186.2614379084967 + }, + "orientation": 0, + "type": "input-output", + "ports": { + "list_of_paths": { + "properties": { + "color": "#79c5d2" + }, + "id": "list_of_paths", + "type": "left", + "position": { + "x": 44, + "y": 74.5 + } + } + }, + "properties": { + "title": "loop list-dev-user-name-at-domain-com", + "vars": [ + { + "name": "list_of_paths", + "direction": "input", + "type": "datatype", + "color": "#79c5d2" + } + ], + "params": [], + "inputs": [ + "list_of_paths" + ], + "outputs": [], + "og_node_id": "6ce40db" + }, + "size": { + "width": 250, + "height": 150 + } + }, + "8f94be49-2e5a-4b22-bef9-7a2d35b05d76": { + "id": "8f94be49-2e5a-4b22-bef9-7a2d35b05d76", + "position": { + "x": 2881.851846532883, + "y": 528.7450980392158 + }, + "orientation": 0, + "type": "input-output", + "ports": { + "list_of_paths": { + "properties": { + "color": "#79c5d2" + }, + "id": "list_of_paths", + "type": "left", + "position": { + "x": 44, + "y": 74.5 + } + } + }, + "properties": { + "title": "loop list-dev-user-name-at-domain-com", + "vars": [ + { + "name": "list_of_paths", + "direction": "input", + "type": "datatype", + "color": "#79c5d2" + } + ], + "params": [], + "inputs": [ + "list_of_paths" + ], + "outputs": [], + "og_node_id": "6ce40db" + }, + "size": { + "width": 250, + "height": 150 + } + }, + "01a6d66f-afb1-47c4-8f94-faace4874409": { + "id": "01a6d66f-afb1-47c4-8f94-faace4874409", + "position": { + "x": 3060.936813853145, + "y": 0 + }, + "orientation": 0, + "type": "input-output", + "ports": { + "list_of_ints": { + "properties": { + "color": "#5f1f93" + }, + "id": "list_of_ints", + "type": "left", + "position": { + "x": 37.5, + "y": 74.5 + } + }, + "a": { + "properties": { + "color": "#77862d" + }, + "id": "a", + "type": "right", + "position": { + "x": 243, + "y": 74.5 + } + } + }, + "properties": { + "title": "loop int list-dev-user-name-at-domain-com", + "vars": [ + { + "name": "list_of_ints", + "direction": "input", + "type": "datatype", + "color": "#5f1f93" + }, + { + "name": "a", + "direction": "output", + "type": "datatype", + "color": "#77862d" + } + ], + "params": [], + "inputs": [ + "list_of_ints" + ], + "outputs": [ + "a" + ], + "og_node_id": "e1578ef" + }, + "size": { + "width": 250, + "height": 150 + } + }, + "f5733922-b56b-497e-8066-c96cdb7b2a79": { + "id": "f5733922-b56b-497e-8066-c96cdb7b2a79", + "position": { + "x": 3104.074068755105, + "y": 347.0457516339869 + }, + "orientation": 0, + "type": "input-output", + "ports": { + "list_of_ints": { + "properties": { + "color": "#5f1f93" + }, + "id": "list_of_ints", + "type": "left", + "position": { + "x": 37.5, + "y": 74.5 + } + }, + "a": { + "properties": { + "color": "#77862d" + }, + "id": "a", + "type": "right", + "position": { + "x": 243, + "y": 74.5 + } + } + }, + "properties": { + "title": "loop int list-dev-user-name-at-domain-com", + "vars": [ + { + "name": "list_of_ints", + "direction": "input", + "type": "datatype", + "color": "#5f1f93" + }, + { + "name": "a", + "direction": "output", + "type": "datatype", + "color": "#77862d" + } + ], + "params": [], + "inputs": [ + "list_of_ints" + ], + "outputs": [ + "a" + ], + "og_node_id": "e1578ef" + }, + "size": { + "width": 250, + "height": 150 + } + }, + "c64cf08d-2caa-46bb-a199-c8fd25e9d590": { + "id": "c64cf08d-2caa-46bb-a199-c8fd25e9d590", + "position": { + "x": 3744.5969445720984, + "y": 345.73856209150324 + }, + "orientation": 0, + "type": "input-output", + "ports": { + "a": { + "properties": { + "color": "#77862d" + }, + "id": "a", + "type": "left", + "position": { + "x": 5, + "y": 74.5 + } + }, + "msg": { + "properties": { + "color": "#7bd279" + }, + "id": "msg", + "type": "right", + "position": { + "x": 232.5, + "y": 74.5 + } + } + }, + "properties": { + "title": "KNMI-vol-h5-to-ODIM-h5-dev-user-name-at-domain-com", + "vars": [ + { + "name": "a", + "direction": "input", + "type": "datatype", + "color": "#77862d" + }, + { + "name": "msg", + "direction": "output", + "type": "datatype", + "color": "#7bd279" + } + ], + "params": [], + "inputs": [ + "a" + ], + "outputs": [ + "msg" + ], + "og_node_id": "e25ec34" + }, + "size": { + "width": 250, + "height": 150 + } + }, + "223011e6-faff-479b-a859-a3f77db0fbb5": { + "id": "223011e6-faff-479b-a859-a3f77db0fbb5", + "position": { + "x": 3735.4466177747117, + "y": 515.6732026143787 + }, + "orientation": 0, + "type": "input-output", + "ports": { + "a": { + "properties": { + "color": "#77862d" + }, + "id": "a", + "type": "left", + "position": { + "x": 5, + "y": 74.5 + } + }, + "msg": { + "properties": { + "color": "#7bd279" + }, + "id": "msg", + "type": "right", + "position": { + "x": 232.5, + "y": 74.5 + } + } + }, + "properties": { + "title": "vol2bird-dev-user-name-at-domain-com", + "vars": [ + { + "name": "a", + "direction": "input", + "type": "datatype", + "color": "#77862d" + }, + { + "name": "msg", + "direction": "output", + "type": "datatype", + "color": "#7bd279" + } + ], + "params": [], + "inputs": [ + "a" + ], + "outputs": [ + "msg" + ], + "og_node_id": "ded63d7" + }, + "size": { + "width": 250, + "height": 150 + } + }, + "e9e3a59e-923f-4d05-b3b0-2ec79590acc9": { + "id": "e9e3a59e-923f-4d05-b3b0-2ec79590acc9", + "position": { + "x": 4251.78648705576, + "y": 340.5098039215685 + }, + "orientation": 0, + "type": "input-output", + "ports": { + "msg": { + "properties": { + "color": "#7bd279" + }, + "id": "msg", + "type": "left", + "position": { + "x": 15.5, + "y": 74.5 + } + } + }, + "properties": { + "title": "Print msg-dev-user-name-at-domain-com", + "vars": [ + { + "name": "msg", + "direction": "input", + "type": "datatype", + "color": "#7bd279" + } + ], + "params": [], + "inputs": [ + "msg" + ], + "outputs": [], + "og_node_id": "a74dc70" + }, + "size": { + "width": 250, + "height": 150 + } + }, + "cd47ca61-ba20-4826-b92f-7ee990822f3a": { + "id": "cd47ca61-ba20-4826-b92f-7ee990822f3a", + "position": { + "x": 4249.172107970791, + "y": 539.2026143790852 + }, + "orientation": 0, + "type": "input-output", + "ports": { + "msg": { + "properties": { + "color": "#7bd279" + }, + "id": "msg", + "type": "left", + "position": { + "x": 15.5, + "y": 74.5 + } + } + }, + "properties": { + "title": "Print msg-dev-user-name-at-domain-com", + "vars": [ + { + "name": "msg", + "direction": "input", + "type": "datatype", + "color": "#7bd279" + } + ], + "params": [], + "inputs": [ + "msg" + ], + "outputs": [], + "og_node_id": "a74dc70" + }, + "size": { + "width": 250, + "height": 150 + } + } + }, + "links": { + "a9475401-f066-46ec-893d-84f5ab3d1cf4": { + "id": "a9475401-f066-46ec-893d-84f5ab3d1cf4", + "from": { + "nodeId": "e05a69bc-d0a4-4f79-93e2-7c79ba2b04e1", + "portId": "file_path" + }, + "to": { + "nodeId": "7d1c8e18-880d-4bab-be5c-dd47d5eed5b4", + "portId": "file_path" + } + }, + "48a07267-919e-4852-8d08-c83a366b0c85": { + "id": "48a07267-919e-4852-8d08-c83a366b0c85", + "from": { + "nodeId": "e05a69bc-d0a4-4f79-93e2-7c79ba2b04e1", + "portId": "file_path" + }, + "to": { + "nodeId": "f847ca0a-1a4d-4e2a-85bd-6ec7be87c3ed", + "portId": "file_path" + } + }, + "43dcc09b-af0e-4738-bcba-faf9f6ae26dc": { + "id": "43dcc09b-af0e-4738-bcba-faf9f6ae26dc", + "from": { + "nodeId": "7d1c8e18-880d-4bab-be5c-dd47d5eed5b4", + "portId": "lines" + }, + "to": { + "nodeId": "8f44f21c-829c-4abc-881f-377159a2d2b1", + "portId": "splitter_source" + } + }, + "15096fc3-d464-4566-b9e7-1fd786d087a8": { + "id": "15096fc3-d464-4566-b9e7-1fd786d087a8", + "from": { + "nodeId": "8f44f21c-829c-4abc-881f-377159a2d2b1", + "portId": "splitter_target" + }, + "to": { + "nodeId": "1d67688c-536d-48ef-a40f-4b0aaa3f3638", + "portId": "lines" + } + }, + "01c23e09-97cb-4bb4-b965-c40ae9fa9b66": { + "id": "01c23e09-97cb-4bb4-b965-c40ae9fa9b66", + "from": { + "nodeId": "f847ca0a-1a4d-4e2a-85bd-6ec7be87c3ed", + "portId": "lines" + }, + "to": { + "nodeId": "473de030-05fc-4f09-9325-11b838bcc140", + "portId": "lines" + } + }, + "03a6f9db-f510-46e5-89d0-3235c2a51a6b": { + "id": "03a6f9db-f510-46e5-89d0-3235c2a51a6b", + "from": { + "nodeId": "473de030-05fc-4f09-9325-11b838bcc140", + "portId": "count" + }, + "to": { + "nodeId": "62fe958c-92d6-4be1-bbcc-73bfe88b9e2a", + "portId": "count" + } + }, + "4f74db47-a87d-4eb6-abf6-fd101ee4056c": { + "id": "4f74db47-a87d-4eb6-abf6-fd101ee4056c", + "from": { + "nodeId": "62fe958c-92d6-4be1-bbcc-73bfe88b9e2a", + "portId": "a" + }, + "to": { + "nodeId": "57c8b452-35f5-4b7b-b777-e2bee59ef99d", + "portId": "a" + } + }, + "cc20739a-d3a0-4286-a428-23ab1eaf883e": { + "id": "cc20739a-d3a0-4286-a428-23ab1eaf883e", + "from": { + "nodeId": "473de030-05fc-4f09-9325-11b838bcc140", + "portId": "count" + }, + "to": { + "nodeId": "57c8b452-35f5-4b7b-b777-e2bee59ef99d", + "portId": "count" + } + }, + "cb0a4d7c-c379-4194-9a8a-14d0adf2744c": { + "id": "cb0a4d7c-c379-4194-9a8a-14d0adf2744c", + "from": { + "nodeId": "57c8b452-35f5-4b7b-b777-e2bee59ef99d", + "portId": "msg" + }, + "to": { + "nodeId": "12ae508b-b472-4924-af9a-a3051c7612fa", + "portId": "msg" + } + }, + "1f8b4ec5-977c-4cde-9302-f43b134513e3": { + "id": "1f8b4ec5-977c-4cde-9302-f43b134513e3", + "from": { + "nodeId": "57c8b452-35f5-4b7b-b777-e2bee59ef99d", + "portId": "msg" + }, + "to": { + "nodeId": "c8cf52ae-446f-402e-8ea5-d26dbf4ff903", + "portId": "msg" + } + }, + "673af78d-9cc3-4348-83e3-cf9620d1a41d": { + "id": "673af78d-9cc3-4348-83e3-cf9620d1a41d", + "from": { + "nodeId": "12ae508b-b472-4924-af9a-a3051c7612fa", + "portId": "list_of_ints" + }, + "to": { + "nodeId": "c0f6bebc-1529-4901-85d7-42eb1342868e", + "portId": "splitter_source" + } + }, + "0508241e-6b83-46d2-be50-64b9e4b48e0a": { + "id": "0508241e-6b83-46d2-be50-64b9e4b48e0a", + "from": { + "nodeId": "12ae508b-b472-4924-af9a-a3051c7612fa", + "portId": "list_of_paths" + }, + "to": { + "nodeId": "edd68621-780a-42df-958c-74f0a711b260", + "portId": "splitter_source" + } + }, + "928092bb-0d60-4c46-9aef-0bd524cadc06": { + "id": "928092bb-0d60-4c46-9aef-0bd524cadc06", + "from": { + "nodeId": "c8cf52ae-446f-402e-8ea5-d26dbf4ff903", + "portId": "list_of_paths" + }, + "to": { + "nodeId": "8f94be49-2e5a-4b22-bef9-7a2d35b05d76", + "portId": "list_of_paths" + } + }, + "b1b4c25d-114c-4ade-bfaf-5055e8347806": { + "id": "b1b4c25d-114c-4ade-bfaf-5055e8347806", + "from": { + "nodeId": "edd68621-780a-42df-958c-74f0a711b260", + "portId": "splitter_target" + }, + "to": { + "nodeId": "1a85bb42-8eba-4b46-b691-ac0687985fab", + "portId": "list_of_paths" + } + }, + "822e43b9-bf44-4dd5-9132-2cd4301e2ce8": { + "id": "822e43b9-bf44-4dd5-9132-2cd4301e2ce8", + "from": { + "nodeId": "c0f6bebc-1529-4901-85d7-42eb1342868e", + "portId": "splitter_target" + }, + "to": { + "nodeId": "01a6d66f-afb1-47c4-8f94-faace4874409", + "portId": "list_of_ints" + } + }, + "4e0cbf61-f09e-4864-841f-c675d95daf46": { + "id": "4e0cbf61-f09e-4864-841f-c675d95daf46", + "from": { + "nodeId": "c8cf52ae-446f-402e-8ea5-d26dbf4ff903", + "portId": "list_of_ints" + }, + "to": { + "nodeId": "f5733922-b56b-497e-8066-c96cdb7b2a79", + "portId": "list_of_ints" + } + }, + "95a9c637-214c-44d4-8042-c20023e0220d": { + "id": "95a9c637-214c-44d4-8042-c20023e0220d", + "from": { + "nodeId": "f5733922-b56b-497e-8066-c96cdb7b2a79", + "portId": "a" + }, + "to": { + "nodeId": "c64cf08d-2caa-46bb-a199-c8fd25e9d590", + "portId": "a" + } + }, + "e72a8541-2a9c-4c88-afa7-0a159ea3fda4": { + "id": "e72a8541-2a9c-4c88-afa7-0a159ea3fda4", + "from": { + "nodeId": "f5733922-b56b-497e-8066-c96cdb7b2a79", + "portId": "a" + }, + "to": { + "nodeId": "223011e6-faff-479b-a859-a3f77db0fbb5", + "portId": "a" + } + }, + "94a01312-1a30-41f4-8959-e12a0c01269f": { + "id": "94a01312-1a30-41f4-8959-e12a0c01269f", + "from": { + "nodeId": "c64cf08d-2caa-46bb-a199-c8fd25e9d590", + "portId": "msg" + }, + "to": { + "nodeId": "e9e3a59e-923f-4d05-b3b0-2ec79590acc9", + "portId": "msg" + } + }, + "38d3d6f4-d54a-435d-b290-b489c8d2460b": { + "id": "38d3d6f4-d54a-435d-b290-b489c8d2460b", + "from": { + "nodeId": "223011e6-faff-479b-a859-a3f77db0fbb5", + "portId": "msg" + }, + "to": { + "nodeId": "cd47ca61-ba20-4826-b92f-7ee990822f3a", + "portId": "msg" + } + } + }, + "selected": {}, + "hovered": {} + }, + "params": {} +} diff --git a/vreapis/tests/resources/workflows/argo/argo_workflow.json b/vreapis/tests/resources/workflows/argo/argo_workflow.json new file mode 100644 index 00000000..1a430a74 --- /dev/null +++ b/vreapis/tests/resources/workflows/argo/argo_workflow.json @@ -0,0 +1,37 @@ +{ + "namespace": "argo", + "serverDryRun": false, + "workflow": { + "metadata": { + "generateName": "hello-world-", + "namespace": "argo", + "labels": { + "workflows.argoproj.io/completed": "false" + } + }, + "spec": { + "templates": [ + { + "name": "whalesay", + "arguments": {}, + "inputs": {}, + "outputs": {}, + "metadata": {}, + "container": { + "name": "", + "image": "docker/whalesay:latest", + "command": [ + "cowsay" + ], + "args": [ + "hello world" + ], + "resources": {} + } + } + ], + "entrypoint": "whalesay", + "arguments": {} + } + } +} \ No newline at end of file From 72b5b532aacb6458f9cac9c9c7e30b7e23d3fd09 Mon Sep 17 00:00:00 2001 From: AndyBRoswell Date: Thu, 6 Jun 2024 22:24:14 +0200 Subject: [PATCH 016/149] test_extractor.py migration preliminarily completed --- vreapis/containerizer/tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vreapis/containerizer/tests.py b/vreapis/containerizer/tests.py index dea9eded..79c40f8e 100644 --- a/vreapis/containerizer/tests.py +++ b/vreapis/containerizer/tests.py @@ -49,7 +49,7 @@ class ExtractorTestCase(TestCase): 'param_string_with_comment': 'param_string value', } - def setUp(self): + def setUp(self): # use setUp instead of __init__, or 'uncaught TypeError: __init__() takes 1 positional argument but 2 were given' super().__init__() self.base_path = '' if os.path.exists('resources'): From cfd21d83cc64da74086ee770fb1dafb869854c47 Mon Sep 17 00:00:00 2001 From: AndyBRoswell Date: Thu, 6 Jun 2024 23:36:52 +0200 Subject: [PATCH 017/149] test_extractor.py migration preliminarily completed --- vreapis/containerizer/tests.py | 41 +++++++++++++++---- vreapis/containerizer/urls.py | 3 +- vreapis/containerizer/views.py | 8 +++- .../resources/notebooks/test_notebook.ipynb | 1 - 4 files changed, 42 insertions(+), 11 deletions(-) diff --git a/vreapis/containerizer/tests.py b/vreapis/containerizer/tests.py index 79c40f8e..845cb6e4 100644 --- a/vreapis/containerizer/tests.py +++ b/vreapis/containerizer/tests.py @@ -10,11 +10,23 @@ import requests import nbformat from slugify import slugify +from unittest import mock from services.extractor.pyextractor import PyExtractor from services.extractor.rextractor import RExtractor from services.converter import ConverterReactFlowChart from db.cell import Cell +from .views import ExtractorHandler + +base_path = '' +if os.path.exists('resources'): + base_path = 'resources' +elif os.path.exists('tests/resources/'): + base_path = 'tests/resources/' + + +def get_auth_header() -> dict[str, str]: + return {'Authorization': f'Token {settings.NAAVRE_API_TOKEN}'} class ContainerizerTestCase(TestCase): @@ -32,7 +44,7 @@ def test_get_base_images(self): dummy_user = User.objects.create_user(dummy_username, password=dummy_password) client.login(username=dummy_username, password=dummy_password) - response = client.get('/api/containerizer/baseimagetags/', headers={'Authorization': f'Token {settings.NAAVRE_API_TOKEN}'}) + response = client.get('/api/containerizer/baseimagetags/', headers=get_auth_header()) self.assertEqual(response.status_code, 200) images = response.json() self.assertIsInstance(images, dict) @@ -51,11 +63,6 @@ class ExtractorTestCase(TestCase): def setUp(self): # use setUp instead of __init__, or 'uncaught TypeError: __init__() takes 1 positional argument but 2 were given' super().__init__() - self.base_path = '' - if os.path.exists('resources'): - self.base_path = 'resources' - elif os.path.exists('tests/resources/'): - self.base_path = 'tests/resources/' def create_cell(self, payload_path=None): with open(payload_path, 'r') as file: @@ -125,7 +132,7 @@ def extract_cell(self, payload_path): return None def test_extract_cell(self): - notebooks_json_path = os.path.join(self.base_path, 'notebooks') + notebooks_json_path = os.path.join(base_path, 'notebooks') notebooks_files = glob.glob( os.path.join(notebooks_json_path, "*.json") ) @@ -143,3 +150,23 @@ def test_extract_cell(self): if os.path.basename(notebook_file) in ['test_param_values_Python.json', 'test_param_values_R.json', ]: for param_name in cell['params']: self.assertTrue(cell['param_values'][param_name] == self.param_values_ref[param_name]) + + +class ExtractorHandlerTestCase(TestCase): + def test(self): + notebooks_json_path = os.path.join(base_path, 'notebooks') + notebooks_files = glob.glob(os.path.join(notebooks_json_path, "*.json")) + for notebook_file in notebooks_files: + with open(notebook_file, 'r') as file: + notebook = json.load(file) + # print(f'[Notebook File]{os.linesep}{notebook_file}') + # print(f'[Notebook Content]{os.linesep}{notebook}') + file.close() + client = Client() + response = client.post('/api/containerizer/extractorhandler/', headers=get_auth_header(), data=notebook, content_type="application/json") + self.assertEqual(response.status_code, 200) + # get JSON response + JSON_response = json.loads(response.data) + self.assertIsNotNone(JSON_response) + cell = notebook['notebook']['cells'][notebook['cell_index']] + print('cell: ', cell) diff --git a/vreapis/containerizer/urls.py b/vreapis/containerizer/urls.py index e67d7955..fd562b9d 100644 --- a/vreapis/containerizer/urls.py +++ b/vreapis/containerizer/urls.py @@ -2,5 +2,6 @@ from . import views urlpatterns = [ - path('baseimagetags/', views.get_base_images) + path('baseimagetags/', views.get_base_images), + path('extractorhandler/', views.ExtractorHandler.as_view()) ] diff --git a/vreapis/containerizer/views.py b/vreapis/containerizer/views.py index 4455d680..20ffa283 100644 --- a/vreapis/containerizer/views.py +++ b/vreapis/containerizer/views.py @@ -24,6 +24,7 @@ import utils.cors from auth.simple import StaticTokenAuthentication from db.catalog import Catalog +from db.cell import Cell import common @@ -45,7 +46,10 @@ def get_base_images(request): return Response(dat, headers=utils.cors.get_CORS_headers(request)) -class ExtractorHandler(APIView): +class ExtractorHandler(APIView, Catalog): + authentication_classes = [StaticTokenAuthentication] + permission_classes = [IsAuthenticated] + def extract_cell_by_index(self, notebook, cell_index): new_nb = copy.deepcopy(notebook) if cell_index < len(notebook.cells): @@ -66,7 +70,7 @@ def get(self, request: Request): def post(self, request: Request): payload = request.data - common.logger.debug('ExtractorHandler. payload: ' + json.dumps(payload, indent=4)) + # common.logger.debug('ExtractorHandler. payload: ' + json.dumps(payload, indent=4)) kernel = payload['kernel'] cell_index = payload['cell_index'] notebook = nbformat.reads(json.dumps(payload['notebook']), nbformat.NO_CONVERT) diff --git a/vreapis/tests/resources/notebooks/test_notebook.ipynb b/vreapis/tests/resources/notebooks/test_notebook.ipynb index c4b4e90b..d632ab3e 100644 --- a/vreapis/tests/resources/notebooks/test_notebook.ipynb +++ b/vreapis/tests/resources/notebooks/test_notebook.ipynb @@ -226,4 +226,3 @@ "nbformat": 4, "nbformat_minor": 5 } - From db0ee1ed7a7aebf963196b2f5de437ad79f0adee Mon Sep 17 00:00:00 2001 From: AndyBRoswell Date: Fri, 7 Jun 2024 00:14:31 +0200 Subject: [PATCH 018/149] migrating TypesHandler, BaseImageHandler, CellsHandler --- vreapis/containerizer/tests.py | 2 -- vreapis/containerizer/views.py | 56 ++++++++++++++++++++++++++++++++-- 2 files changed, 54 insertions(+), 4 deletions(-) diff --git a/vreapis/containerizer/tests.py b/vreapis/containerizer/tests.py index 845cb6e4..170cc000 100644 --- a/vreapis/containerizer/tests.py +++ b/vreapis/containerizer/tests.py @@ -159,8 +159,6 @@ def test(self): for notebook_file in notebooks_files: with open(notebook_file, 'r') as file: notebook = json.load(file) - # print(f'[Notebook File]{os.linesep}{notebook_file}') - # print(f'[Notebook Content]{os.linesep}{notebook}') file.close() client = Client() response = client.post('/api/containerizer/extractorhandler/', headers=get_auth_header(), data=notebook, content_type="application/json") diff --git a/vreapis/containerizer/views.py b/vreapis/containerizer/views.py index 20ffa283..cd7d0be4 100644 --- a/vreapis/containerizer/views.py +++ b/vreapis/containerizer/views.py @@ -66,11 +66,11 @@ def set_notebook_kernel(self, notebook, kernel): def get(self, request: Request): msg_json = dict(title="Operation not supported.") - return Response(msg_json) + return Response(msg_json, status=status.HTTP_400_BAD_REQUEST) def post(self, request: Request): payload = request.data - # common.logger.debug('ExtractorHandler. payload: ' + json.dumps(payload, indent=4)) + common.logger.debug('ExtractorHandler. payload: ' + json.dumps(payload, indent=4)) kernel = payload['kernel'] cell_index = payload['cell_index'] notebook = nbformat.reads(json.dumps(payload['notebook']), nbformat.NO_CONVERT) @@ -145,3 +145,55 @@ def post(self, request: Request): cell.chart_obj = chart Catalog.editor_buffer = copy.deepcopy(cell) return Response(cell.toJSON()) + + +class TypesHandler(APIView, Catalog): + authentication_classes = [StaticTokenAuthentication] + permission_classes = [IsAuthenticated] + + def post(self, request: Request): + payload = request.data + common.logger.debug('TypesHandler. payload: ' + str(payload)) + port = payload['port'] + p_type = payload['type'] + cell = Catalog.editor_buffer + cell.types[port] = p_type + + +class BaseImageHandler(APIView, Catalog): + authentication_classes = [StaticTokenAuthentication] + permission_classes = [IsAuthenticated] + + def post(self, request: Request): + payload = request.data + common.logger.debug('BaseImageHandler. payload: ' + str(payload)) + print('BaseImageHandler. payload: ' + str(payload)) + base_image = payload['image'] + cell = Catalog.editor_buffer + cell.base_image = base_image + + +class CellsHandler(APIView, Catalog): + authentication_classes = [StaticTokenAuthentication] + permission_classes = [IsAuthenticated] + + def get(self, request: Request): + msg_json = dict(title="Operation not supported.") + return Response(msg_json) + + def post(self, request: Request): + try: + current_cell = Catalog.editor_buffer + current_cell.clean_code() + current_cell.clean_title() + current_cell.clean_task_name() + except Exception as ex: + err_msg = 'Error setting cell: ' + str(ex) + common.logger.error(err_msg) + return Response(err_msg, status=status.HTTP_400_BAD_REQUEST) + + common.logger.debug('current_cell: ' + current_cell.toJSON()) + all_vars = current_cell.params + current_cell.inputs + current_cell.outputs + for param_name in all_vars: + if param_name not in current_cell.types: + pass From 13b8e846bf99010352c0c54318a35cd15ff61669 Mon Sep 17 00:00:00 2001 From: AndyBRoswell Date: Fri, 7 Jun 2024 11:15:12 +0200 Subject: [PATCH 019/149] migrating TypesHandler, BaseImageHandler, CellsHandler --- vreapis/containerizer/views.py | 38 +++++++++++++++++++++------------- 1 file changed, 24 insertions(+), 14 deletions(-) diff --git a/vreapis/containerizer/views.py b/vreapis/containerizer/views.py index cd7d0be4..ca97faa3 100644 --- a/vreapis/containerizer/views.py +++ b/vreapis/containerizer/views.py @@ -3,12 +3,12 @@ import traceback import copy import hashlib +from typing import Optional from rest_framework.permissions import IsAuthenticated from rest_framework.response import Response from rest_framework.request import Request from rest_framework.views import APIView -from rest_framework.status import HTTP_400_BAD_REQUEST from rest_framework import status from rest_framework.decorators import api_view, authentication_classes, permission_classes import requests @@ -29,6 +29,11 @@ import common +def return_error(err_msg: str, e: Optional[Exception], stat: status = status.HTTP_400_BAD_REQUEST) -> Response: + common.logger.error(err_msg, e) + return Response(err_msg, status=stat) + + @api_view(['GET']) @authentication_classes([StaticTokenAuthentication]) @permission_classes([IsAuthenticated]) @@ -40,9 +45,7 @@ def get_base_images(request): response.raise_for_status() dat: dict[str, dict[str, str]] = response.json() except (requests.ConnectionError, requests.HTTPError, requests.JSONDecodeError,) as e: - msg: str = f'Error loading base image tags from {url}\n{e}' - common.logger.debug(msg) - return Response({'error': msg}, status=status.HTTP_500_INTERNAL_SERVER_ERROR) + return return_error(f'Error loading base image tags from {url}', e, stat=status.HTTP_500_INTERNAL_SERVER_ERROR) return Response(dat, headers=utils.cors.get_CORS_headers(request)) @@ -65,8 +68,7 @@ def set_notebook_kernel(self, notebook, kernel): return new_nb def get(self, request: Request): - msg_json = dict(title="Operation not supported.") - return Response(msg_json, status=status.HTTP_400_BAD_REQUEST) + return return_error("Operation not supported.", stat=status.HTTP_400_BAD_REQUEST) def post(self, request: Request): payload = request.data @@ -85,7 +87,7 @@ def post(self, request: Request): try: extractor = HeaderExtractor(notebook, source) except jsonschema.ValidationError as e: - return Response({'message': f"Error in cell header: {e}", 'reason': None, 'traceback': traceback.format_exception(e), }, status=HTTP_400_BAD_REQUEST) + return return_error('Error in cell header', e, stat=status.HTTP_400_BAD_REQUEST) # Extractor based on code analysis. Used if the cell has no header, or if some values are not specified in the header if not extractor.is_complete(): @@ -167,7 +169,6 @@ class BaseImageHandler(APIView, Catalog): def post(self, request: Request): payload = request.data common.logger.debug('BaseImageHandler. payload: ' + str(payload)) - print('BaseImageHandler. payload: ' + str(payload)) base_image = payload['image'] cell = Catalog.editor_buffer cell.base_image = base_image @@ -178,8 +179,7 @@ class CellsHandler(APIView, Catalog): permission_classes = [IsAuthenticated] def get(self, request: Request): - msg_json = dict(title="Operation not supported.") - return Response(msg_json) + return return_error('Operation not supported.', stat=status.HTTP_400_BAD_REQUEST) def post(self, request: Request): try: @@ -188,12 +188,22 @@ def post(self, request: Request): current_cell.clean_title() current_cell.clean_task_name() except Exception as ex: - err_msg = 'Error setting cell: ' + str(ex) - common.logger.error(err_msg) - return Response(err_msg, status=status.HTTP_400_BAD_REQUEST) + return return_error('Error setting cell', ex, status.HTTP_400_BAD_REQUEST) common.logger.debug('current_cell: ' + current_cell.toJSON()) all_vars = current_cell.params + current_cell.inputs + current_cell.outputs for param_name in all_vars: if param_name not in current_cell.types: - pass + return return_error(f'{param_name} not in types', stat=status.HTTP_400_BAD_REQUEST) + + if not hasattr(current_cell, 'base_image'): + return return_error(f'{current_cell.task_name} has not selected base image', stat=status.HTTP_400_BAD_REQUEST) + try: + doc_cell = Catalog.get_cell_from_og_node_id(current_cell.node_id) + if doc_cell: + Catalog.update_cell(current_cell) + else: + Catalog.add_cell(current_cell) + except Exception as ex: + return return_error('Error adding or updating cell in catalog', ex, status.HTTP_400_BAD_REQUEST) + From 90ca89c0dec5f9d73b2b3b815b778576951e2e2d Mon Sep 17 00:00:00 2001 From: AndyBRoswell Date: Fri, 7 Jun 2024 15:59:03 +0200 Subject: [PATCH 020/149] migrating TypesHandler, BaseImageHandler, CellsHandler --- vreapis/containerizer/views.py | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/vreapis/containerizer/views.py b/vreapis/containerizer/views.py index ca97faa3..2c49ec64 100644 --- a/vreapis/containerizer/views.py +++ b/vreapis/containerizer/views.py @@ -4,6 +4,7 @@ import copy import hashlib from typing import Optional +from pathlib import Path from rest_framework.permissions import IsAuthenticated from rest_framework.response import Response @@ -177,6 +178,13 @@ def post(self, request: Request): class CellsHandler(APIView, Catalog): authentication_classes = [StaticTokenAuthentication] permission_classes = [IsAuthenticated] + cells_path = os.path.join(str(Path.home()), 'NaaVRE', 'cells') + + def write_cell_to_file(self, current_cell): + Path('/tmp/workflow_cells/cells').mkdir(parents=True, exist_ok=True) + with open('/tmp/workflow_cells/cells/' + current_cell.task_name + '.json', 'w') as f: + f.write(current_cell.toJSON()) + f.close() def get(self, request: Request): return return_error('Operation not supported.', stat=status.HTTP_400_BAD_REQUEST) @@ -207,3 +215,26 @@ def post(self, request: Request): except Exception as ex: return return_error('Error adding or updating cell in catalog', ex, status.HTTP_400_BAD_REQUEST) + if os.getenv('DEBUG'): + self.write_cell_to_file(current_cell) + + if not os.path.exists(self.cells_path): + os.mkdir(self.cells_path) + + cell_path = os.path.join(self.cells_path, current_cell.task_name) + + if os.path.exists(cell_path): + for files in os.listdir(cell_path): + path = os.path.join(cell_path, files) + if os.path.isfile(path): + os.remove(path) + else: + os.mkdir(cell_path) + + registry_credentials = Catalog.get_registry_credentials() + if not registry_credentials or len(registry_credentials) <= 0: + return return_error('Registry credentials not found', stat=status.HTTP_400_BAD_REQUEST) + image_repo = registry_credentials[0]['url'] + if not image_repo: + return return_error('Registry not found') + From fd5cbdd3b3931802e02777ef3fe562c0616acc23 Mon Sep 17 00:00:00 2001 From: AndyBRoswell Date: Mon, 10 Jun 2024 21:01:54 +0200 Subject: [PATCH 021/149] migrating TypesHandler, BaseImageHandler, CellsHandler --- vreapis/containerizer/RContainerizer.py | 98 +++++++++ vreapis/containerizer/tests.py | 276 +++++++++++++++++++++++- vreapis/containerizer/urls.py | 3 +- vreapis/containerizer/views.py | 262 +++++++++++++++++++++- vreapis/requirements.txt | 3 + 5 files changed, 626 insertions(+), 16 deletions(-) create mode 100644 vreapis/containerizer/RContainerizer.py diff --git a/vreapis/containerizer/RContainerizer.py b/vreapis/containerizer/RContainerizer.py new file mode 100644 index 00000000..241b841c --- /dev/null +++ b/vreapis/containerizer/RContainerizer.py @@ -0,0 +1,98 @@ +import logging +import os + +from jinja2 import Environment, PackageLoader + +logger = logging.getLogger(__name__) + +handler = logging.StreamHandler() +handler.setLevel(logging.DEBUG) + +# Create a formatter for the log messages +formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') + +# Add the formatter to the handler +handler.setFormatter(formatter) + +# Add the handler to the logger +logger.addHandler(handler) + + +class RContainerizer: + @staticmethod + def get_files_info(cell=None, cells_path=None): + if not os.path.exists(cells_path): + os.mkdir(cells_path) + cell_path = os.path.join(cells_path, cell.task_name) + + cell_file_name = 'task.R' + dockerfile_name = 'Dockerfile' + environment_file_name = 'environment.yaml' + + if os.path.exists(cell_path): + for files in os.listdir(cell_path): + path = os.path.join(cell_path, files) + if os.path.isfile(path): + os.remove(path) + else: + os.mkdir(cell_path) + + cell_file_path = os.path.join(cell_path, cell_file_name) + dockerfile_file_path = os.path.join(cell_path, dockerfile_name) + env_file_path = os.path.join(cell_path, environment_file_name) + return { + 'cell': {'file_name': cell_file_name, 'path': cell_file_path}, + 'dockerfile': {'file_name': dockerfile_name, 'path': dockerfile_file_path}, + 'environment': {'file_name': environment_file_name, 'path': env_file_path}, + } + + @staticmethod + def map_dependencies(dependencies, module_name_mapping): + dependencies = map(lambda x: 'r-' + x['name'], dependencies) + dependencies = map(lambda x: module_name_mapping.get('r', {}).get(x, x), dependencies) + set_conda_deps = set(dependencies) + set_pip_deps = set() + set_conda_deps.discard(None) + set_conda_deps.discard(None) + return set_conda_deps, set_pip_deps + + @staticmethod + def build_templates(cell=None, files_info=None, module_name_mapping=None): + # we also want to always add the id to the input parameters + inputs = cell.inputs + types = cell.types + inputs.append('id') + cell.concatenate_all_inputs() + types['id'] = 'str' + logger.debug("inputs: " + str(cell.inputs)) + logger.debug("types: " + str(cell.types)) + logger.debug("params: " + str(cell.params)) + logger.debug("outputs: " + str(cell.outputs)) + + logger.debug('files_info: ' + str(files_info)) + logger.debug('cell.dependencies: ' + str(cell.dependencies)) + + loader = PackageLoader('jupyterlab_vre', 'templates') + template_env = Environment(loader=loader, trim_blocks=True, lstrip_blocks=True) + + template_cell = template_env.get_template('R_cell_template.jinja2') + template_dockerfile = template_env.get_template('dockerfile_template_conda.jinja2') + + compiled_code = template_cell.render(cell=cell, deps=cell.generate_dependencies(), types=cell.types, confs=cell.generate_configuration()) + cell.container_source = compiled_code + dependencies = cell.generate_dependencies() + r_dependencies = [] + for dep in dependencies: + r_dep = dep.replace('import ', '') + install_packages = 'if (!requireNamespace("' + r_dep + '", quietly = TRUE)) {\n\tinstall.packages("' + r_dep + '", repos="http://cran.us.r-project.org")\n}' + r_dependencies.append(install_packages) + library = 'library(' + r_dep + ')' + r_dependencies.append(library) + + template_cell.stream(cell=cell, deps=r_dependencies, types=cell.types, confs=cell.generate_configuration()).dump(files_info['cell']['path']) + template_dockerfile.stream(task_name=cell.task_name, base_image=cell.base_image).dump(files_info['dockerfile']['path']) + + set_conda_deps, set_pip_deps = RContainerizer.map_dependencies(cell.dependencies, module_name_mapping) + logger.debug('cell.dependencies.conda: ' + str(cell.dependencies)) + template_conda = template_env.get_template('conda_env_template.jinja2') + template_conda.stream(base_image=cell.base_image, conda_deps=list(set_conda_deps), pip_deps=list(set_pip_deps)).dump(files_info['environment']['path']) diff --git a/vreapis/containerizer/tests.py b/vreapis/containerizer/tests.py index 170cc000..14d669a1 100644 --- a/vreapis/containerizer/tests.py +++ b/vreapis/containerizer/tests.py @@ -1,7 +1,13 @@ +import datetime import glob import os import json +import shlex +import shutil +import subprocess +import sys import uuid +from pathlib import Path from django.test import TestCase, Client from django.conf import settings @@ -9,14 +15,16 @@ from urllib.parse import urlencode import requests import nbformat +from github import Github from slugify import slugify -from unittest import mock +from tornado.gen import sleep +import common +from db.catalog import Catalog from services.extractor.pyextractor import PyExtractor from services.extractor.rextractor import RExtractor from services.converter import ConverterReactFlowChart from db.cell import Cell -from .views import ExtractorHandler base_path = '' if os.path.exists('resources'): @@ -29,7 +37,7 @@ def get_auth_header() -> dict[str, str]: return {'Authorization': f'Token {settings.NAAVRE_API_TOKEN}'} -class ContainerizerTestCase(TestCase): +class GetBaseImagesTestCase(TestCase): @staticmethod def Keycloak_login() -> dict[str, any]: return requests.post(settings.KEYCLOAK_LOGIN_URL, headers={'Content-Type': 'application/x-www-form-urlencoded'}, data=urlencode({'client_id': 'myclient', 'grant_type': 'password', 'scope': 'openid', 'username': 'u', 'password': 'u'}), verify=settings.ALLOW_INSECURE_TLS).json() @@ -122,20 +130,15 @@ def extract_cell(self, payload_path): # Check if file exists if os.path.exists(payload_path): cell = self.create_cell(payload_path) - node = ConverterReactFlowChart.get_node(cell.node_id, cell.title, cell.inputs, cell.outputs, cell.params, ) - chart = {'offset': {'x': 0, 'y': 0, }, 'scale': 1, 'nodes': {cell.node_id: node}, 'links': {}, 'selected': {}, 'hovered': {}, } - cell.chart_obj = chart return cell.toJSON() return None def test_extract_cell(self): notebooks_json_path = os.path.join(base_path, 'notebooks') - notebooks_files = glob.glob( - os.path.join(notebooks_json_path, "*.json") - ) + notebooks_files = glob.glob(os.path.join(notebooks_json_path, "*.json")) for notebook_file in notebooks_files: cell = self.extract_cell(notebook_file) if cell: @@ -168,3 +171,258 @@ def test(self): self.assertIsNotNone(JSON_response) cell = notebook['notebook']['cells'][notebook['cell_index']] print('cell: ', cell) + + +class CellsHandlerTestCase(TestCase): + cells_path = os.path.join(str(Path.home()), 'NaaVRE', 'cells') + github_url_repos = 'https://api.github.com/repos' + + def setUp(self): + self.client = Client() + + def create_cell_and_add_to_cat(self, cell_path=None): + print('Creating cell from: ', cell_path) + with open(cell_path, 'r') as file: + cell = json.load(file) + file.close() + notebook_dict = {} + if 'notebook_dict' in cell: + notebook_dict = cell['notebook_dict'] + test_cell = Cell(cell['title'], cell['task_name'], cell['original_source'], cell['inputs'], + cell['outputs'], + cell['params'], cell['confs'], cell['dependencies'], cell['container_source'], + cell['chart_obj'], cell['node_id'], cell['kernel'], notebook_dict) + test_cell.types = cell['types'] + test_cell.base_image = cell['base_image'] + Catalog.editor_buffer = test_cell + return test_cell, cell + + def call_cell_handler(self): + return self.client.post('/api/containerizer/cellshandler', data='', content_type='application/json') + + def delete_text(self, file_path, text_to_delete): + # Read the file + with open(file_path, 'r') as file: + lines = file.readlines() + + # Remove the text from each line + updated_lines = [] + for line in lines: + updated_line = line.replace(text_to_delete, '') + updated_lines.append(updated_line) + + # Write the updated lines to the file + with open(file_path, 'w') as file: + file.writelines(updated_lines) + + def wait_for_github_api_resources(self): + github = Github(Catalog.get_repositories()[0]['token']) + rate_limit = github.get_rate_limit() + while rate_limit.core.remaining <= 0: + reset = rate_limit.core.reset + # Calculate remaining time for reset + remaining_time = reset.timestamp() - datetime.datetime.now().timestamp() + common.logger.debug(f'Remaining time for reset: {remaining_time} s') + common.logger.debug(f'API rate exceeded, waiting') + common.logger.debug(f'Sleeping for: {remaining_time + 1}') + sleep(remaining_time + 1) + rate_limit = github.get_rate_limit() + + def get_github_workflow_runs(self, owner=None, repository_name=None, t_utc=None, token=None): + workflow_runs_url = CellsHandlerTestCase.github_url_repos + '/' + owner + '/' + repository_name + '/actions/runs' + if t_utc: + t_start = (t_utc - datetime.timedelta(minutes=1)).strftime("%Y-%m-%dT%H:%M:%SZ") + t_stop = (t_utc + datetime.timedelta(minutes=1)).strftime("%Y-%m-%dT%H:%M:%SZ") + workflow_runs_url += f"?created={t_start}..{t_stop}" + headers = {'Accept': 'application/vnd.github.v3+json'} + if token: + headers['Authorization'] = 'Bearer ' + token + workflow_runs = common.session.get(url=workflow_runs_url, verify=False, headers=headers) + if workflow_runs.status_code != 200: + return None + workflow_runs_json = json.loads(workflow_runs.text) + return workflow_runs_json + + def get_github_workflow_jobs(self, jobs_url=None, token=None): + headers = {'Accept': 'application/vnd.github.v3+json'} + if token: + headers['Authorization'] = 'Bearer ' + token + jobs = common.session.get(url=jobs_url, verify=False, headers=headers) + if jobs.status_code == 200: + return json.loads(jobs.text) + else: + raise Exception('Error getting jobs for workflow run: ' + jobs.text) + + def find_job(self, + wf_id=None, + wf_creation_utc=None, + owner=None, + repository_name=None, + token=None, + job_id=None, + ): + f""" Find Github workflow job + + If job_id is set, retrieve it through + https://api.github.com/repos/{owner}/{repository_name}/actions/jobs/{job_id} + + Else, get all workflows runs created around wf_creation_utc through + https://api.github.com/repos/{owner}/{repository_name}/actions/runs + and find the one matching {wf_id} + """ + if job_id: + jobs_url = CellsHandlerTestCase.github_url_repos + '/' + owner + '/' + repository_name + '/actions/jobs/' + str(job_id) + self.wait_for_github_api_resources() + job = self.get_github_workflow_jobs(jobs_url, token=token) + return job + self.wait_for_github_api_resources() + runs = self.get_github_workflow_runs( + owner=owner, + repository_name=repository_name, + t_utc=wf_creation_utc, + token=token) + if not runs: + return None + for run in runs['workflow_runs']: + jobs_url = run['jobs_url'] + self.wait_for_github_api_resources() + jobs = self.get_github_workflow_jobs(jobs_url, token=token) + for job in jobs['jobs']: + if job['name'] == wf_id: + job['head_sha'] = run['head_sha'] + return job + return None + + def wait_for_job(self, + wf_id=None, + wf_creation_utc=None, + owner=None, + repository_name=None, + token=None, + job_id=None, + timeout=200, + wait_for_completion=False, + ): + """ Call find_job until something is returned or timeout is reached + + :param wf_id: passed to find_job + :param wf_creation_utc: passed to find_job + :param owner: passed to find_job + :param repository_name: passed to find_job + :param token: passed to find_job + :param job_id: passed to find_job + :param timeout: timeout in seconds + :param wait_for_completion: wait for the job's status to be 'complete' + + :return: job or None + """ + start_time = datetime.datetime.now().timestamp() # seconds + stop_time = start_time + timeout + while datetime.datetime.now().timestamp() < stop_time: + job = self.find_job( + wf_id=wf_id, + wf_creation_utc=wf_creation_utc, + owner=owner, + repository_name=repository_name, + token=token, + job_id=job_id, + ) + if job: + if not wait_for_completion: + return job + if wait_for_completion and (job['status'] == 'completed'): + return job + sleep(5) + + def test(self): + cells_json_path = os.path.join(base_path, 'cells') + cells_files = os.listdir(cells_json_path) + test_cells = [] + for cell_file in cells_files: + cell_path = os.path.join(cells_json_path, cell_file) + test_cell, cell = self.create_cell_and_add_to_cat(cell_path=cell_path) + response = self.call_cell_handler() + self.assertEqual(200, response.status_code) + wf_id = json.loads(response.body.decode('utf-8'))['wf_id'] + wf_creation_utc = datetime.datetime.now(tz=datetime.timezone.utc) + dispatched_github_workflow = json.loads(response.body.decode('utf-8'))['dispatched_github_workflow'] + test_cells.append({ + 'wf_id': wf_id, + 'wf_creation_utc': wf_creation_utc, + 'dispatched_github_workflow': dispatched_github_workflow, + }) + if 'skip_exec' in cell and cell['skip_exec']: + continue + if 'python' in test_cell.kernel and 'skip_exec': + cell_path = os.path.join(CellsHandlerTestCase.cells_path, test_cell.task_name, 'task.py') + print('---------------------------------------------------') + print('Executing cell: ', cell_path) + if 'example_inputs' in cell: + exec_args = [sys.executable, cell_path] + cell['example_inputs'] + else: + exec_args = [sys.executable, cell_path] + + cell_exec = subprocess.Popen(exec_args, stdout=subprocess.PIPE) + text = cell_exec.communicate()[0] + print(text) + print("stdout:", cell_exec.stdout) + print("stderr:", cell_exec.stderr) + print("return code:", cell_exec.returncode) + print('---------------------------------------------------') + self.assertEqual(0, cell_exec.returncode, 'Cell execution failed: ' + cell_file) + elif test_cell.kernel == 'IRkernel' and 'skip_exec': + cell_path = os.path.join(CellsHandlerTestCase.cells_path, test_cell.task_name, 'task.R') + run_local_cell_path = os.path.join(CellsHandlerTestCase.cells_path, test_cell.task_name, 'run_local.R') + shutil.copy(cell_path, run_local_cell_path) + self.delete_text(run_local_cell_path, 'setwd(\'/app\')') + example_inputs = '' + if 'example_inputs' in cell: + example_inputs = ' '.join(cell['example_inputs']) + command = 'Rscript ' + run_local_cell_path + ' ' + example_inputs + result = subprocess.run(shlex.split(command), capture_output=True, text=True) + self.assertEqual(0, result.returncode, result.stderr) + + cat_repositories = Catalog.get_repositories() + repo = cat_repositories[0] + repo_token = repo['token'] + owner, repository_name = repo['url'].removeprefix('https://github.com/').split('/') + if '.git' in repository_name: + repository_name = repository_name.split('.git')[0] + + updated_cells = list(filter( + lambda cell: cell['dispatched_github_workflow'], + test_cells, + )) + + for cell in updated_cells: + # Get job id (many calls to the GitHub API) + job = self.wait_for_job( + wf_id=cell['wf_id'], + wf_creation_utc=cell['wf_creation_utc'], + owner=owner, + repository_name=repository_name, + token=repo_token, + job_id=None, + timeout=300, + wait_for_completion=False, + ) + cell['job'] = job + + for cell in updated_cells: + # Wait for job completion (fewer calls) + job = self.wait_for_job( + wf_id=cell['wf_id'], + wf_creation_utc=None, + owner=owner, + repository_name=repository_name, + token=repo_token, + job_id=cell['job']['id'], + timeout=300, + wait_for_completion=True, + ) + cell['job'] = job + + for cell in updated_cells: + self.assertIsNotNone(cell['job'], 'Job not found') + self.assertEqual('completed', cell['job']['status'], 'Job not completed') + self.assertEqual('success', cell['job']['conclusion'], 'Job not successful') diff --git a/vreapis/containerizer/urls.py b/vreapis/containerizer/urls.py index fd562b9d..c8c384b4 100644 --- a/vreapis/containerizer/urls.py +++ b/vreapis/containerizer/urls.py @@ -3,5 +3,6 @@ urlpatterns = [ path('baseimagetags/', views.get_base_images), - path('extractorhandler/', views.ExtractorHandler.as_view()) + path('extractorhandler/', views.ExtractorHandler.as_view()), + path('cellshandler/', views.CellsHandler.as_view()), ] diff --git a/vreapis/containerizer/views.py b/vreapis/containerizer/views.py index 2c49ec64..1293979c 100644 --- a/vreapis/containerizer/views.py +++ b/vreapis/containerizer/views.py @@ -1,11 +1,18 @@ +import importlib import os import json +import re +import sys import traceback import copy import hashlib +import uuid from typing import Optional from pathlib import Path +import autopep8 +from distro import distro +from jinja2 import PackageLoader, Environment from rest_framework.permissions import IsAuthenticated from rest_framework.response import Response from rest_framework.request import Request @@ -16,7 +23,9 @@ import nbformat import jsonschema from slugify import slugify +from github import Github, UnknownObjectException +from containerizer.RContainerizer import RContainerizer from services.extractor.extractor import DummyExtractor from services.extractor.headerextractor import HeaderExtractor from services.extractor.pyextractor import PyExtractor @@ -179,6 +188,8 @@ class CellsHandler(APIView, Catalog): authentication_classes = [StaticTokenAuthentication] permission_classes = [IsAuthenticated] cells_path = os.path.join(str(Path.home()), 'NaaVRE', 'cells') + github_url_repos = 'https://api.github.com/repos' + github_workflow_file_name = 'build-push-docker.yml' def write_cell_to_file(self, current_cell): Path('/tmp/workflow_cells/cells').mkdir(parents=True, exist_ok=True) @@ -186,8 +197,200 @@ def write_cell_to_file(self, current_cell): f.write(current_cell.toJSON()) f.close() + def load_module_name_mapping(self): + module_mapping_url = os.getenv('MODULE_MAPPING_URL') + module_mapping = {} + if module_mapping_url: + resp = common.session.get(module_mapping_url) + module_mapping = json.loads(resp.text) + module_name_mapping_path = os.path.join(str(Path.home()), 'NaaVRE', 'module_name_mapping.json') + if not os.path.exists(module_name_mapping_path): + with open(module_name_mapping_path, 'w') as module_name_mapping_file: + json.dump(module_mapping, module_name_mapping_file, indent=4) + module_name_mapping_file.close() + + module_name_mapping_file = open(module_name_mapping_path) + loaded_module_name_mapping = json.load(module_name_mapping_file) + loaded_module_name_mapping.update(module_mapping) + module_name_mapping_file.close() + return loaded_module_name_mapping + + def get_files_info(self, cell=None): + if not os.path.exists(self.cells_path): + os.mkdir(self.cells_path) + cell_path = os.path.join(self.cells_path, cell.task_name) + + cell_file_name = 'task.py' + dockerfile_name = 'Dockerfile' + environment_file_name = 'environment.yaml' + + notebook_file_name = None + if 'visualize-' in cell.task_name: + notebook_file_name = 'task.ipynb' + if os.path.exists(cell_path): + for files in os.listdir(cell_path): + path = os.path.join(cell_path, files) + if os.path.isfile(path): + os.remove(path) + else: + os.mkdir(cell_path) + + cell_file_path = os.path.join(cell_path, cell_file_name) + dockerfile_file_path = os.path.join(cell_path, dockerfile_name) + env_file_path = os.path.join(cell_path, environment_file_name) + info = { + 'cell': {'file_name': cell_file_name, 'path': cell_file_path}, + 'dockerfile': {'file_name': dockerfile_name, 'path': dockerfile_file_path}, + 'environment': {'file_name': environment_file_name, 'path': env_file_path} + } + if notebook_file_name: + info['notebook'] = {'file_name': notebook_file_name, 'path': os.path.join(cell_path, notebook_file_name)} + return info + + def is_standard_module(self, module_name): + if module_name in sys.builtin_module_names: + return True + installation_path = None + try: + installation_path = importlib.import_module(module_name).__file__ + except ImportError: + return False + linux_os = distro.id() + return 'dist-packages' not in installation_path if linux_os == 'Ubuntu' else 'site-packages' not in installation_path + + def map_dependencies(self, dependencies=None, module_name_mapping=None): + set_conda_deps = set([]) + set_pip_deps = set([]) + for dep in dependencies: + if 'module' in dep and dep['module']: + if '.' in dep['module']: + module_name = dep['module'].split('.')[0] + else: + module_name = dep['module'] + elif 'name' in dep and dep['name']: + module_name = dep['name'] + if module_name: + conda_package = True + pip_package = False + if module_name in module_name_mapping['conda'].keys(): + module_name = module_name_mapping['conda'][module_name] + pip_package = False + conda_package = True + if module_name in module_name_mapping['pip'].keys(): + module_name = module_name_mapping['pip'][module_name] + pip_package = True + conda_package = False + if module_name is None: + continue + if not self.is_standard_module(module_name): + if conda_package: + set_conda_deps.add(module_name) + if pip_package: + set_pip_deps.add(module_name) + return set_conda_deps, set_pip_deps + + def build_templates(self, cell=None, files_info=None, module_name_mapping=None): + common.logger.debug('files_info: ' + str(files_info)) + common.logger.debug('cell.dependencies: ' + str(cell.dependencies)) + set_conda_deps, set_pip_deps = self.map_dependencies(dependencies=cell.dependencies, module_name_mapping=module_name_mapping, ) + loader = PackageLoader('jupyterlab_vre', 'templates') + template_env = Environment(loader=loader, trim_blocks=True, lstrip_blocks=True) + + if cell.title.startswith('visualize-'): + template_cell = template_env.get_template('vis_cell_template.jinja2') + if cell.notebook_dict: + notebook_path = os.path.join(files_info['cell']['path']).replace('.py', '.ipynb') + with open(notebook_path, 'w') as f: + f.write(json.dumps(cell.notebook_dict, indent=4)) + f.close() + else: + template_cell = template_env.get_template('py_cell_template.jinja2') + template_dockerfile = template_env.get_template('dockerfile_template_conda.jinja2') + + compiled_code = template_cell.render(cell=cell, deps=cell.generate_dependencies(), types=cell.types, confs=cell.generate_configuration_dict()) + + compiled_code = autopep8.fix_code(compiled_code) + cell.container_source = compiled_code + + template_cell.stream(cell=cell, deps=cell.generate_dependencies(), types=cell.types, confs=cell.generate_configuration_dict()).dump(files_info['cell']['path']) + template_dockerfile.stream(task_name=cell.task_name, base_image=cell.base_image).dump( + files_info['dockerfile']['path']) + + template_conda = template_env.get_template('conda_env_template.jinja2') + template_conda.stream(base_image=cell.base_image, conda_deps=list(set_conda_deps), pip_deps=list(set_pip_deps)).dump(files_info['environment']['path']) + + def git_hash(self, contents): + s = hashlib.sha1() + s.update(('blob %u\0' % len(contents)).encode('utf-8')) + s.update(contents) + return s.hexdigest() + + def create_or_update_cell_in_repository(self, task_name, repository, files_info): + files_updated = False + code_content_hash = None + for f_type, f_info in files_info.items(): + f_name = f_info['file_name'] + f_path = f_info['path'] + with open(f_path, 'rb') as f: + local_content = f.read() + local_hash = self.git_hash(local_content) + try: + remote_hash = repository.get_contents(path=task_name + '/' + f_name).sha + except UnknownObjectException: + remote_hash = None + common.logger.debug(f'local_hash: {local_hash}; remote_hash: {remote_hash}') + if remote_hash is None: + repository.create_file(path=task_name + '/' + f_name, message=task_name + ' creation', content=local_content, ) + elif remote_hash != local_hash: + repository.update_file(path=task_name + '/' + f_name, message=task_name + ' update', content=local_content, sha=remote_hash, ) + files_updated = True + if f_type == 'cell': + code_content_hash = local_hash + return files_updated, code_content_hash + + def query_registry_for_image(self, image_repo, image_name): + m = re.match(r'^docker.io/(\w+)', image_name) + if m: + # Docker Hub + url = f'https://hub.docker.com/v2/repositories/{m.group(1)}/{image_name}' + headers = {} + else: + # OCI registries + domain = image_repo.split('/')[0] + path = '/'.join(image_repo.split('/')[1:]) + url = f'https://{domain}/v2/{path}/{image_name}/tags/list' + # OCI registries require authentication, even for public registries. + # The token should be set in the $OCI_TOKEN environment variable. + # For ghcr.io, connections still succeed when $OCI_TOKEN is unset (this results in header "Authorization: Bearer None", which grants access to public registries, although it is not officially documented). If this fails, or when accessing private registries, OCI_TOKEN should be a base64-encoded GitHub classic access token with the read:packages scope. + headers = {"Authorization": f"Bearer {os.getenv('OCI_TOKEN')}", } + response = common.session.get(url, headers=headers) + if response.status_code == 200: + return json.loads(response.content.decode('utf-8')) + else: + return None + + def dispatch_github_workflow(owner, repository_name, task_name, files_info, repository_token, image, wf_id=None, image_version=None): + url = CellsHandler.github_url_repos + '/' + owner + '/' + repository_name + '/actions/workflows/' + CellsHandler.github_workflow_file_name + '/dispatches' + resp = common.session.post( + url=url, + json={ + 'ref': 'refs/heads/main', + 'inputs': { + 'build_dir': task_name, + 'dockerfile': files_info['dockerfile']['file_name'], + 'image_repo': image, + 'image_tag': task_name, + 'id': wf_id, + 'image_version': image_version, + } + }, + verify=False, + headers={'Accept': 'application/vnd.github.v3+json', 'Authorization': 'token ' + repository_token} + ) + return resp + def get(self, request: Request): - return return_error('Operation not supported.', stat=status.HTTP_400_BAD_REQUEST) + return return_error('Operation not supported.') def post(self, request: Request): try: @@ -196,16 +399,16 @@ def post(self, request: Request): current_cell.clean_title() current_cell.clean_task_name() except Exception as ex: - return return_error('Error setting cell', ex, status.HTTP_400_BAD_REQUEST) + return return_error('Error setting cell', ex) common.logger.debug('current_cell: ' + current_cell.toJSON()) all_vars = current_cell.params + current_cell.inputs + current_cell.outputs for param_name in all_vars: if param_name not in current_cell.types: - return return_error(f'{param_name} not in types', stat=status.HTTP_400_BAD_REQUEST) + return return_error(f'{param_name} not in types') if not hasattr(current_cell, 'base_image'): - return return_error(f'{current_cell.task_name} has not selected base image', stat=status.HTTP_400_BAD_REQUEST) + return return_error(f'{current_cell.task_name} has not selected base image') try: doc_cell = Catalog.get_cell_from_og_node_id(current_cell.node_id) if doc_cell: @@ -213,7 +416,7 @@ def post(self, request: Request): else: Catalog.add_cell(current_cell) except Exception as ex: - return return_error('Error adding or updating cell in catalog', ex, status.HTTP_400_BAD_REQUEST) + return return_error('Error adding or updating cell in catalog', ex) if os.getenv('DEBUG'): self.write_cell_to_file(current_cell) @@ -233,8 +436,55 @@ def post(self, request: Request): registry_credentials = Catalog.get_registry_credentials() if not registry_credentials or len(registry_credentials) <= 0: - return return_error('Registry credentials not found', stat=status.HTTP_400_BAD_REQUEST) + return return_error('Registry credentials not found') image_repo = registry_credentials[0]['url'] if not image_repo: return return_error('Registry not found') + if current_cell.kernel == "IRkernel": + files_info = RContainerizer.get_files_info(cell=current_cell, cells_path=CellsHandler.cells_path) + RContainerizer.build_templates(cell=current_cell, files_info=files_info, module_name_mapping=self.load_module_name_mapping(), ) + elif 'python' in current_cell.kernel.lower(): + files_info = self.get_files_info(cell=current_cell) + self.build_templates(cell=current_cell, files_info=files_info, module_name_mapping=self.load_module_name_mapping(), ) + else: + return return_error(f'Kernel {current_cell.kernel} not supported') + + # upload to GIT + cat_repositories = Catalog.get_repositories() + + repo_token = cat_repositories[0]['token'] + if not repo_token: + return return_error('Repository token not found') + + gh_token = Github(cat_repositories[0]['token']) + url_repos = cat_repositories[0]['url'] + if not url_repos: + return return_error('Repository url not found') + + owner = url_repos.split('https://github.com/')[1].split('/')[0] + repository_name = url_repos.split('https://github.com/')[1].split('/')[1] + if '.git' in repository_name: + repository_name = repository_name.split('.git')[0] + try: + gh_repository = gh_token.get_repo(owner + '/' + repository_name) + except Exception as ex: + return return_error(f'Error getting repository', ex) + do_dispatch_github_workflow, image_version = self.create_or_update_cell_in_repository(task_name=current_cell.task_name, repository=gh_repository, files_info=files_info, ) + wf_id = str(uuid.uuid4()) + + if os.getenv('DEBUG') and os.getenv('DEBUG').lower() == 'true': + do_dispatch_github_workflow = True + else: + image_info = self.query_registry_for_image(image_repo=image_repo, image_name=current_cell.task_name, ) + if not image_info: + do_dispatch_github_workflow = True + + image_version = image_version[:7] + if do_dispatch_github_workflow: + resp = self.dispatch_github_workflow(owner, repository_name, current_cell.task_name, files_info, repo_token, image_repo, wf_id=wf_id, image_version=image_version) + if resp.status_code != 201 and resp.status_code != 200 and resp.status_code != 204: + return return_error(resp.text) + current_cell.set_image_version(image_version) + Catalog.delete_cell_from_task_name(current_cell.task_name) + Catalog.add_cell(current_cell) diff --git a/vreapis/requirements.txt b/vreapis/requirements.txt index 7c790334..d0394b7d 100644 --- a/vreapis/requirements.txt +++ b/vreapis/requirements.txt @@ -20,3 +20,6 @@ rpy2==3.5.11 pyflakes pytype tinydb +autopep8 +distro +pygithub From deaaf0a76c347419d44538e6ad6ce069d34304f0 Mon Sep 17 00:00:00 2001 From: AndyBRoswell Date: Mon, 10 Jun 2024 21:05:09 +0200 Subject: [PATCH 022/149] [bugfix] migrating TypesHandler, BaseImageHandler, CellsHandler --- vreapis/containerizer/tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vreapis/containerizer/tests.py b/vreapis/containerizer/tests.py index 14d669a1..9a8d16cc 100644 --- a/vreapis/containerizer/tests.py +++ b/vreapis/containerizer/tests.py @@ -198,7 +198,7 @@ def create_cell_and_add_to_cat(self, cell_path=None): return test_cell, cell def call_cell_handler(self): - return self.client.post('/api/containerizer/cellshandler', data='', content_type='application/json') + return self.client.post('/api/containerizer/cellshandler/', content_type='application/json') def delete_text(self, file_path, text_to_delete): # Read the file From 2785dc27bf61bca93747d2b704162177e70e3618 Mon Sep 17 00:00:00 2001 From: AndyBRoswell Date: Mon, 10 Jun 2024 21:21:56 +0200 Subject: [PATCH 023/149] [bugfix] migrating TypesHandler, BaseImageHandler, CellsHandler --- vreapis/containerizer/tests.py | 2 +- vreapis/containerizer/views.py | 2 +- vreapis/templates/R_cell_template.jinja2 | 71 +++++++ vreapis/templates/conda_env_template.jinja2 | 17 ++ .../dockerfile_template_conda.jinja2 | 18 ++ vreapis/templates/py_cell_template.jinja2 | 62 ++++++ vreapis/templates/vis_cell_template.jinja2 | 72 +++++++ vreapis/templates/workflow_template_v2.jinja2 | 199 ++++++++++++++++++ 8 files changed, 441 insertions(+), 2 deletions(-) create mode 100644 vreapis/templates/R_cell_template.jinja2 create mode 100644 vreapis/templates/conda_env_template.jinja2 create mode 100644 vreapis/templates/dockerfile_template_conda.jinja2 create mode 100644 vreapis/templates/py_cell_template.jinja2 create mode 100644 vreapis/templates/vis_cell_template.jinja2 create mode 100644 vreapis/templates/workflow_template_v2.jinja2 diff --git a/vreapis/containerizer/tests.py b/vreapis/containerizer/tests.py index 9a8d16cc..a19db4b7 100644 --- a/vreapis/containerizer/tests.py +++ b/vreapis/containerizer/tests.py @@ -198,7 +198,7 @@ def create_cell_and_add_to_cat(self, cell_path=None): return test_cell, cell def call_cell_handler(self): - return self.client.post('/api/containerizer/cellshandler/', content_type='application/json') + return self.client.post('/api/containerizer/cellshandler/', content_type='application/json', headers=get_auth_header()) def delete_text(self, file_path, text_to_delete): # Read the file diff --git a/vreapis/containerizer/views.py b/vreapis/containerizer/views.py index 1293979c..97894d78 100644 --- a/vreapis/containerizer/views.py +++ b/vreapis/containerizer/views.py @@ -293,7 +293,7 @@ def build_templates(self, cell=None, files_info=None, module_name_mapping=None): common.logger.debug('files_info: ' + str(files_info)) common.logger.debug('cell.dependencies: ' + str(cell.dependencies)) set_conda_deps, set_pip_deps = self.map_dependencies(dependencies=cell.dependencies, module_name_mapping=module_name_mapping, ) - loader = PackageLoader('jupyterlab_vre', 'templates') + loader = PackageLoader('.', 'templates') template_env = Environment(loader=loader, trim_blocks=True, lstrip_blocks=True) if cell.title.startswith('visualize-'): diff --git a/vreapis/templates/R_cell_template.jinja2 b/vreapis/templates/R_cell_template.jinja2 new file mode 100644 index 00000000..c99660b5 --- /dev/null +++ b/vreapis/templates/R_cell_template.jinja2 @@ -0,0 +1,71 @@ +setwd('/app') + +# retrieve input parameters + +library(optparse) +library(jsonlite) +{% for dep in deps%} +{{ dep }} +{% endfor %} + + +option_list = list( + +{% for in_var in cell.all_inputs|sort %} +{% if types[in_var] == list or types[in_var] == 'list' %} +{% set type = 'character' %} +{% elif types[in_var] == str or types[in_var] == 'str' %} +{% set type = 'character' %} +{% elif types[in_var] == int or types[in_var] == 'int' %} +{% set type = 'integer' %} +{% elif types[in_var] == float or types[in_var] == 'float' %} +{% set type = 'numeric' %} +{% elif types[in_var] == bool or types[in_var] == 'bool' %} +{% set type = 'logical' %} +{% else %} +{% set type = 'character' %} +{% endif %} +{% set option = 'make_option(c("--' + in_var+'"), action="store", default=NA, type="'+type+'", help="my description")' %} +{% if not loop.last %} +{% set option = option + ', '%} +{% endif %} +{{ option }} +{% endfor %} + +) + +# set input parameters accordingly +opt = parse_args(OptionParser(option_list=option_list)) + +{% for in_var in cell.inputs|sort %} +{% if types[in_var] == list or types[in_var] == 'list' %} +{{ in_var }} = fromJSON(opt${{ in_var }}) +{% elif types[in_var] == str or types[in_var] == 'str' %} +{{ in_var }} <- gsub('"', '', opt${{ in_var }}) +{% else %} +{{ in_var }} = opt${{ in_var }} +{% endif %} +{% endfor %} + +{% for param in cell.params|sort %} +{{ param }} = opt${{ param }} +{% endfor %} + + +{% for c in confs %} +{{ c }} +{% endfor %} + + +{{ cell.original_source }} + + + +{% if cell.outputs|length > 0 %} +# capturing outputs +{% for out in cell.outputs %} +file <- file(paste0('/tmp/{{out}}_', id, '.json')) +writeLines(toJSON({{out}}, auto_unbox=TRUE), file) +close(file) +{% endfor %} +{% endif %} \ No newline at end of file diff --git a/vreapis/templates/conda_env_template.jinja2 b/vreapis/templates/conda_env_template.jinja2 new file mode 100644 index 00000000..6506adaf --- /dev/null +++ b/vreapis/templates/conda_env_template.jinja2 @@ -0,0 +1,17 @@ +name: venv +channels: + - conda-forge +dependencies: + - pip + - python>=3.8 +{% for d in conda_deps %} + - {{ d }} +{% endfor %} +{% if pip_deps|length > 0 %} + - pip: + {% for d in pip_deps %} + - {{ d }} + {% endfor %} +{% endif %} + - papermill + - ipykernel \ No newline at end of file diff --git a/vreapis/templates/dockerfile_template_conda.jinja2 b/vreapis/templates/dockerfile_template_conda.jinja2 new file mode 100644 index 00000000..4c0c53f2 --- /dev/null +++ b/vreapis/templates/dockerfile_template_conda.jinja2 @@ -0,0 +1,18 @@ +FROM {{ base_image.build }} AS build +COPY --chown=$MAMBA_USER:$MAMBA_USER environment.yaml . +RUN micromamba install -y -n venv -f environment.yaml +ARG MAMBA_DOCKERFILE_ACTIVATE=1 +USER root +RUN conda-pack -p /opt/conda/envs/venv -o /tmp/env.tar && \ + mkdir /venv && cd /venv && tar xf /tmp/env.tar && \ + rm /tmp/env.tar +RUN /venv/bin/conda-unpack + +{% if task_name.startswith('visualize-')%} +FROM jupyter/base-notebook AS runtime +{% else %} +FROM {{ base_image.runtime }} AS runtime +{% endif %} +COPY --from=build /venv /venv +WORKDIR /app +COPY . . \ No newline at end of file diff --git a/vreapis/templates/py_cell_template.jinja2 b/vreapis/templates/py_cell_template.jinja2 new file mode 100644 index 00000000..0c6ee9c5 --- /dev/null +++ b/vreapis/templates/py_cell_template.jinja2 @@ -0,0 +1,62 @@ +{% if deps|length >0 %} +{% for d in deps %} +{{ d }} +{% endfor %} +{% endif %} + +import argparse +arg_parser = argparse.ArgumentParser() + +arg_parser.add_argument('--id', action='store', type=str, required=True, dest='id') + +{% for in_var in cell.inputs|sort %} + +{% if types[in_var] == list or types[in_var] == 'list' %} +arg_parser.add_argument('--{{ in_var }}', action='store', type=str, required=True, dest='{{ in_var }}') +{% else %} +arg_parser.add_argument('--{{ in_var }}', action='store', type={{ types[in_var] }}, required=True, dest='{{ in_var }}') +{% endif %} +{% endfor %} + +{% for param in cell.params|sort %} +arg_parser.add_argument('--{{ param }}', action='store', type={{ types[param] }}, required=True, dest='{{ param }}') +{% endfor %} + +args = arg_parser.parse_args() +print(args) + +id = args.id + +{% for in_var in cell.inputs|sort %} +{% if types[in_var] == list or types[in_var] == 'list' %} +import json +{{ in_var }} = json.loads(args.{{ in_var }}) +{% elif types[in_var] == str or types[in_var] == 'str' %} +{{ in_var }} = args.{{ in_var }}.replace('"','') +{% else %} +{{ in_var }} = args.{{ in_var }} +{% endif %} +{% endfor %} + +{% for param in cell.params|sort %} +{{ param }} = args.{{ param }} +{% endfor %} + +{% for c in confs %} +{% for c_name in c %} +{{ c_name }} = {{ c[c_name] }} +{% endfor %} + +{% endfor %} + +{{ cell.original_source }} + +{% if cell.outputs|length > 0 %} +import json +{% for out in cell.outputs %} +filename = "/tmp/{{out}}_" + id + ".json" +file_{{out}} = open(filename, "w") +file_{{out}}.write(json.dumps({{out}})) +file_{{out}}.close() +{% endfor %} +{% endif %} \ No newline at end of file diff --git a/vreapis/templates/vis_cell_template.jinja2 b/vreapis/templates/vis_cell_template.jinja2 new file mode 100644 index 00000000..26e5a3cf --- /dev/null +++ b/vreapis/templates/vis_cell_template.jinja2 @@ -0,0 +1,72 @@ +{% if deps|length >0 %} +{% for d in deps %} +{{ d }} +{% endfor %} +{% endif %} + +import argparse +import papermill as pm + +arg_parser = argparse.ArgumentParser() + +arg_parser.add_argument('--id', action='store', type=str, required=True, dest='id') + +{% for in_var in cell.inputs|sort %} + +{% if types[in_var] == list or types[in_var] == 'list' %} +arg_parser.add_argument('--{{ in_var }}', action='store', type=str, required=True, dest='{{ in_var }}') +{% else %} +arg_parser.add_argument('--{{ in_var }}', action='store', type={{ types[in_var] }}, required=True, dest='{{ in_var }}') +{% endif %} +{% endfor %} + +{% for param in cell.params|sort %} +arg_parser.add_argument('--{{ param }}', action='store', type={{ types[param] }}, required=True, dest='{{ param }}') +{% endfor %} + +args = arg_parser.parse_args() +print(args) + +id = args.id +parameters = {} + +{% for in_var in cell.inputs|sort %} +{% if types[in_var] == list or types[in_var] == 'list' %} +import json +{{ in_var }} = json.loads(args.{{ in_var }}) +{% elif types[in_var] == str or types[in_var] == 'str' %} +{{ in_var }} = args.{{ in_var }}.replace('"','') +{% else %} +{{ in_var }} = args.{{ in_var }} +{% endif %} +parameters['{{ in_var}}'] = {{ in_var }} +{% endfor %} + +{% for param in cell.params|sort %} +{{ param }} = args.{{ param }} +parameters['{{ param}}'] = {{ param }} +{% endfor %} + +{% for c in confs %} +{% for c_name in c %} +{{ c_name }} = {{ c[c_name] }} +parameters['{{ c_name }}'] = {{ c_name }} +{% endfor %} +{% endfor %} + +pm.execute_notebook( + '{{ cell.title }}.ipynb', + '{{ cell.title }}-output.ipynb', + prepare_only=True, + parameters=dict(parameters) +) + +{% if cell.outputs|length > 0 %} +import json +{% for out in cell.outputs %} +filename = "/tmp/{{out}}_" + id + ".json" +file_{{out}} = open(filename, "w") +file_{{out}}.write(json.dumps({{out}})) +file_{{out}}.close() +{% endfor %} +{% endif %} \ No newline at end of file diff --git a/vreapis/templates/workflow_template_v2.jinja2 b/vreapis/templates/workflow_template_v2.jinja2 new file mode 100644 index 00000000..e78c57d9 --- /dev/null +++ b/vreapis/templates/workflow_template_v2.jinja2 @@ -0,0 +1,199 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Workflow +metadata: + generateName: {{ workflow_name }}- + labels: + vlab_slug: {{ vlab_slug }} +spec: + entrypoint: {{ workflow_name }} + serviceAccountName: {{ workflow_service_account }} + volumeClaimTemplates: + - metadata: + name: workdir + spec: + accessModes: [ "ReadWriteMany" ] + resources: + requests: + storage: {{ workdir_storage_size }}Gi + arguments: + parameters: + {% for p, value in global_params.items() %} + - name: {{ p }} + value: {{ value }} + {% endfor %} + templates: + - name: {{ workflow_name }} + dag: + tasks: + {% for nid in nodes %} + {% set is_special_node = nodes[nid]['type'] == 'splitter' or nodes[nid]['type'] == 'merger' or nodes[nid]['type'] == 'visualizer' %} + {% set is_splitter = nodes[nid]['type'] == 'splitter' %} + {% set is_visualizer = nodes[nid]['type'] == 'visualizer' %} + {% set task_name = nodes[nid]['type'] + "-" + nid[:7] if is_special_node else cells[nid]['task_name']+ "-" + nid[:7] %} + {% set params = [] if is_special_node else cells[nid]['params'] %} + {# {% if 'visualize-' in task_name %} + - name: {{ task_name }}-server + dependencies: [{{ task_name }}] + template: {{ task_name }}-server-tmp + {% endif %} #} + + - name: {{ task_name }} + {% if deps_dag[nid]|length > 0 %} + dependencies: [{% for d in deps_dag[nid] %} {{ d['task_name'] }}{% if not loop.last %},{% endif %} {% endfor %}] + {% endif %} + template: {{ task_name }}-tmp + {% if deps_dag[nid]|length > 0 or params|length > 0 %} + arguments: + parameters: + {% for d in deps_dag[nid] %} + - {name: {{ d['port_id'] }}, value: {% if d['type'] == "splitter" %}{{ '"{{item}}"' }}{% else %}{{ '"{{tasks.' }}{{ d['task_name'] }}{{ '.outputs.parameters.' }}{{ d['port_id'] }}{{'}}"'}}{% endif %}} + {% endfor %} + {% for p in params %} + - {name: {{ p }}, value: {{ '"{{workflow.parameters.' }}{{ p }}{{ '}}"' }}} + {% endfor %} + {% for d in deps_dag[nid] %} + {% if d['type'] == "splitter" %} + withParam: {{ '"{{tasks.' }}{{ d['task_name'] }}{{ '.outputs.parameters.' }}{{ d['port_id'] }}{{'}}"'}} + {% endif %} + {% endfor %} + {% endif %} + {% endfor %} + {% for nid in nodes %} + {% set is_special_node = nodes[nid]['type'] == 'splitter' or nodes[nid]['type'] == 'merger' or nodes[nid]['type'] == 'visualizer' %} + {% set is_splitter = nodes[nid]['type'] == 'splitter' %} + {% set is_visualizer = nodes[nid]['type'] == 'visualizer' %} + {% set task_name = nodes[nid]['type'] + "-" + nid[:7] if is_special_node else cells[nid]['task_name'] + "-" + nid[:7] %} + {% set params = [] if is_special_node else cells[nid]['params'] %} + {% set ports = nodes[nid]['ports'] %} + {# {% if 'visualize-' in task_name %} + - name: {{ task_name }}-server-tmp + container: + image: busybox + imagePullPolicy: IfNotPresent + command: ["/bin/bash", "-c"] + args: + - echo "Starting server" + {% endif %} #} + - name: {{ task_name }}-tmp + {% if deps_dag[nid]|length > 0 %} + inputs: + parameters: + {% for d in deps_dag[nid] %} + - name: {{ d['port_id'] }} + {% endfor %} + {% for p in params %} + - name: {{ p }} + {% endfor %} + {% endif %} + outputs: + parameters: + {% for p in ports %} + {% if ports[p]['type'] == 'right' %} + - name: {{ ports[p]['id'] }} + valueFrom: + path: /tmp/{{ ports[p]['id'] }}.json + {% endif %} + {% endfor %} + {% if is_special_node and not is_visualizer %} + {% set special_dep = deps_dag[nid][0] %} + script: + image: python:alpine3.9 + imagePullPolicy: IfNotPresent + command: [python] + source: | + import json + {{ special_dep['port_id'] }} = {{ '{{inputs.parameters.' }}{{ special_dep['port_id'] }}{{ '}}' }} + {% for p in ports %} + {% if ports[p]['type'] == 'right' %} + {% if is_splitter %} + {# splitter logic #} + list_of_lists = [] + for elem in {{special_dep['port_id']}}: + list = [elem] + list_of_lists.append(list) + f_out = open("/tmp/{{ports[p]['id']}}.json", "w") + f_out.write(json.dumps(list_of_lists)) + f_out.close() + {% else %} + {# Merger logic #} + {{special_dep['port_id']}} = [item for items in {{special_dep['port_id']}} for item in json.loads(items)] + f_out = open("/tmp/{{ports[p]['id']}}.json", "w") + f_out.write(json.dumps({{special_dep['port_id']}})) + f_out.close() + {% endif %} + {% endif %} + {% endfor %} + {% elif is_visualizer %} + container: + image: "qcdis/geotiff_viewer:v0.2.6" + imagePullPolicy: IfNotPresent + command: ["/bin/sh", "-c"] + args: + - python /app/python_scripts/main.py + {% for d in deps_dag[nid] %} + {% set is_from_special = d['type'] == 'splitter' or d['type'] == 'merger' %} + {% set stdinname = d['og_port_id'] if is_from_special else d['port_id'] %} + {% set stdinname_base = stdinname.split('_')[0] %} + --{{ stdinname_base }} '{{ '{{inputs.parameters.' }}{{ d['port_id'] }}{{ '}}' }}' + {% endfor %} + {% for p in params %} + --{{ p }} '{{ '{{workflow.parameters.' }}{{ p }}{{ '}}' }}' + {% endfor %} + && npm start + metadata: + labels: + app: naavre-visualizer + {% else %} + container: + image: "{{ image_repo }}/{{ cells[nid]['task_name'] }}:{{ cells[nid]['image_version'] }}" + imagePullPolicy: Always + command: ["/bin/bash", "-c"] + args: + {% if 'visualize-' in cells[nid]['task_name']%} + - source /venv/bin/activate; ipython kernel install --user; python /app/task.py + {% for d in deps_dag[nid] %} + {% set is_from_special = d['type'] == 'splitter' or d['type'] == 'merger' %} + {% set stdinname = d['og_port_id'] if is_from_special else d['port_id'] %} + {% set stdinname_base = stdinname.split('_')[0] %} + --{{ stdinname_base }} '{{ '{{inputs.parameters.' }}{{ d['port_id'] }}{{ '}}' }}' + {% endfor %} + {% for p in params %} + --{{ p }} '{{ '{{workflow.parameters.' }}{{ p }}{{ '}}' }}' + {% endfor %} + --id {{' '}}{{cells[nid]['node_id']}}{{''}}{{";"}} + jupyter execute /app/task-output.ipynb --allow-errors; + jupyter nbconvert --no-input --execute /app/task-output.ipynb --to html; + jupyter notebook --port 5173 --NotebookApp.ip='0.0.0.0' --NotebookApp.allow_origin='*' --NotebookApp.base_url=/naavre-visualizer-notebook + metadata: + labels: + app: naavre-visualizer-notebook + {% elif cells[nid]['kernel'] == 'ipython' %} + - source /venv/bin/activate; python /app/task.py + {% for d in deps_dag[nid] %} + {% set is_from_special = d['type'] == 'splitter' or d['type'] == 'merger' %} + {% set stdinname = d['og_port_id'] if is_from_special else d['port_id'] %} + {% set stdinname_base = stdinname.split('_')[0] %} + --{{ stdinname_base }} '{{ '{{inputs.parameters.' }}{{ d['port_id'] }}{{ '}}' }}' + {% endfor %} + {% for p in params %} + --{{ p }} '{{ '{{workflow.parameters.' }}{{ p }}{{ '}}' }}' + {% endfor %} + --id {{' '}}{{cells[nid]['node_id']}}{{''}}{{";"}} + {% elif cells[nid]['kernel'] == 'IRkernel' %} + - source /venv/bin/activate; Rscript /app/task.R + {% for d in deps_dag[nid] %} + {% set is_from_special = d['type'] == 'splitter' or d['type'] == 'merger' %} + {% set stdinname = d['og_port_id'] if is_from_special else d['port_id'] %} + {% set stdinname_base = stdinname.split('_')[0] %} + --{{ stdinname_base }} '{{ '{{inputs.parameters.' }}{{ d['port_id'] }}{{ '}}' }}' + {% endfor %} + {% for p in params %} + --{{ p }} '{{ '{{workflow.parameters.' }}{{ p }}{{ '}}' }}' + {% endfor %} + --id {{' '}}{{cells[nid]['node_id']}}{{''}}{{";"}} + {% endif %} + volumeMounts: + - name: workdir + mountPath: /tmp/data + {% endif %} + {% endfor %} \ No newline at end of file From 77e92f83cc5eae90c98b3f9413da85f82239d418 Mon Sep 17 00:00:00 2001 From: AndyBRoswell Date: Tue, 11 Jun 2024 12:46:46 +0200 Subject: [PATCH 024/149] [bugfix] migrating TypesHandler, BaseImageHandler, CellsHandler --- vreapis/common.py | 3 +++ vreapis/containerizer/RContainerizer.py | 24 ++++++++++++------------ vreapis/containerizer/tests.py | 10 ++++++++-- vreapis/containerizer/views.py | 9 +++++---- 4 files changed, 28 insertions(+), 18 deletions(-) diff --git a/vreapis/common.py b/vreapis/common.py index bbcefdc7..d965ac3c 100644 --- a/vreapis/common.py +++ b/vreapis/common.py @@ -1,7 +1,10 @@ import logging import urllib3 +import os import requests.adapters +project_root = os.path.dirname(os.path.abspath(__file__)) + # customized requests.Session [w/ auto retry] session = requests.Session() retry_adapter = requests.adapters.HTTPAdapter(max_retries=urllib3.Retry(total=10, backoff_factor=0.1, backoff_max=2, status_forcelist=[500, 502, 503, 504])) diff --git a/vreapis/containerizer/RContainerizer.py b/vreapis/containerizer/RContainerizer.py index 241b841c..cb1de17e 100644 --- a/vreapis/containerizer/RContainerizer.py +++ b/vreapis/containerizer/RContainerizer.py @@ -1,9 +1,9 @@ import logging import os -from jinja2 import Environment, PackageLoader +import jinja2 -logger = logging.getLogger(__name__) +import common handler = logging.StreamHandler() handler.setLevel(logging.DEBUG) @@ -15,7 +15,7 @@ handler.setFormatter(formatter) # Add the handler to the logger -logger.addHandler(handler) +common.logger.addHandler(handler) class RContainerizer: @@ -64,16 +64,16 @@ def build_templates(cell=None, files_info=None, module_name_mapping=None): inputs.append('id') cell.concatenate_all_inputs() types['id'] = 'str' - logger.debug("inputs: " + str(cell.inputs)) - logger.debug("types: " + str(cell.types)) - logger.debug("params: " + str(cell.params)) - logger.debug("outputs: " + str(cell.outputs)) + common.logger.debug("inputs: " + str(cell.inputs)) + common.logger.debug("types: " + str(cell.types)) + common.logger.debug("params: " + str(cell.params)) + common.logger.debug("outputs: " + str(cell.outputs)) - logger.debug('files_info: ' + str(files_info)) - logger.debug('cell.dependencies: ' + str(cell.dependencies)) + common.logger.debug('files_info: ' + str(files_info)) + common.logger.debug('cell.dependencies: ' + str(cell.dependencies)) - loader = PackageLoader('jupyterlab_vre', 'templates') - template_env = Environment(loader=loader, trim_blocks=True, lstrip_blocks=True) + loader = jinja2.FileSystemLoader(searchpath=f'{common.project_root}/templates') + template_env = jinja2.Environment(loader=loader, trim_blocks=True, lstrip_blocks=True) template_cell = template_env.get_template('R_cell_template.jinja2') template_dockerfile = template_env.get_template('dockerfile_template_conda.jinja2') @@ -93,6 +93,6 @@ def build_templates(cell=None, files_info=None, module_name_mapping=None): template_dockerfile.stream(task_name=cell.task_name, base_image=cell.base_image).dump(files_info['dockerfile']['path']) set_conda_deps, set_pip_deps = RContainerizer.map_dependencies(cell.dependencies, module_name_mapping) - logger.debug('cell.dependencies.conda: ' + str(cell.dependencies)) + common.logger.debug('cell.dependencies.conda: ' + str(cell.dependencies)) template_conda = template_env.get_template('conda_env_template.jinja2') template_conda.stream(base_image=cell.base_image, conda_deps=list(set_conda_deps), pip_deps=list(set_pip_deps)).dump(files_info['environment']['path']) diff --git a/vreapis/containerizer/tests.py b/vreapis/containerizer/tests.py index a19db4b7..67c9cf28 100644 --- a/vreapis/containerizer/tests.py +++ b/vreapis/containerizer/tests.py @@ -343,9 +343,9 @@ def test(self): test_cell, cell = self.create_cell_and_add_to_cat(cell_path=cell_path) response = self.call_cell_handler() self.assertEqual(200, response.status_code) - wf_id = json.loads(response.body.decode('utf-8'))['wf_id'] + wf_id = response.data['wf_id'] wf_creation_utc = datetime.datetime.now(tz=datetime.timezone.utc) - dispatched_github_workflow = json.loads(response.body.decode('utf-8'))['dispatched_github_workflow'] + dispatched_github_workflow = response.data['dispatched_github_workflow'] test_cells.append({ 'wf_id': wf_id, 'wf_creation_utc': wf_creation_utc, @@ -379,6 +379,12 @@ def test(self): if 'example_inputs' in cell: example_inputs = ' '.join(cell['example_inputs']) command = 'Rscript ' + run_local_cell_path + ' ' + example_inputs + R_dependencies: list[str] = ['optparse', 'jsonlite', ] + for dependency in R_dependencies: + result = subprocess.run(['Rscript', '-e', f'if(!require("{dependency}")) install.packages("{dependency}")'], capture_output=True, text=True) + print(result.stdout) + print(result.stderr) + self.assertEqual(0, result.returncode, result.stderr) result = subprocess.run(shlex.split(command), capture_output=True, text=True) self.assertEqual(0, result.returncode, result.stderr) diff --git a/vreapis/containerizer/views.py b/vreapis/containerizer/views.py index 97894d78..3132eaff 100644 --- a/vreapis/containerizer/views.py +++ b/vreapis/containerizer/views.py @@ -3,7 +3,6 @@ import json import re import sys -import traceback import copy import hashlib import uuid @@ -12,7 +11,7 @@ import autopep8 from distro import distro -from jinja2 import PackageLoader, Environment +import jinja2 from rest_framework.permissions import IsAuthenticated from rest_framework.response import Response from rest_framework.request import Request @@ -293,8 +292,8 @@ def build_templates(self, cell=None, files_info=None, module_name_mapping=None): common.logger.debug('files_info: ' + str(files_info)) common.logger.debug('cell.dependencies: ' + str(cell.dependencies)) set_conda_deps, set_pip_deps = self.map_dependencies(dependencies=cell.dependencies, module_name_mapping=module_name_mapping, ) - loader = PackageLoader('.', 'templates') - template_env = Environment(loader=loader, trim_blocks=True, lstrip_blocks=True) + loader = jinja2.FileSystemLoader(searchpath=f'{common.project_root}/templates') + template_env = jinja2.Environment(loader=loader, trim_blocks=True, lstrip_blocks=True) if cell.title.startswith('visualize-'): template_cell = template_env.get_template('vis_cell_template.jinja2') @@ -488,3 +487,5 @@ def post(self, request: Request): current_cell.set_image_version(image_version) Catalog.delete_cell_from_task_name(current_cell.task_name) Catalog.add_cell(current_cell) + + return Response({'wf_id': wf_id, 'dispatched_github_workflow': do_dispatch_github_workflow, 'image_version': image_version}) From 32c7d562452d37a79ddde273cc3cd15bd13890f0 Mon Sep 17 00:00:00 2001 From: AndyBRoswell Date: Tue, 11 Jun 2024 23:18:24 +0200 Subject: [PATCH 025/149] migrating TypesHandler, BaseImageHandler, CellsHandler --- vreapis/containerizer/tests.py | 2 +- vreapis/containerizer/urls.py | 6 ++- .../tests/emulated-frontend/dat/extract.json | 52 +++++++++++++++++++ vreapis/tests/emulated-frontend/extract.py | 14 +++++ 4 files changed, 71 insertions(+), 3 deletions(-) create mode 100644 vreapis/tests/emulated-frontend/dat/extract.json create mode 100644 vreapis/tests/emulated-frontend/extract.py diff --git a/vreapis/containerizer/tests.py b/vreapis/containerizer/tests.py index 67c9cf28..18b90867 100644 --- a/vreapis/containerizer/tests.py +++ b/vreapis/containerizer/tests.py @@ -379,7 +379,7 @@ def test(self): if 'example_inputs' in cell: example_inputs = ' '.join(cell['example_inputs']) command = 'Rscript ' + run_local_cell_path + ' ' + example_inputs - R_dependencies: list[str] = ['optparse', 'jsonlite', ] + R_dependencies: list[str] = ['optparse', 'jsonlite', ] # Some versions [e.g., older ones] may make this test fail for dependency in R_dependencies: result = subprocess.run(['Rscript', '-e', f'if(!require("{dependency}")) install.packages("{dependency}")'], capture_output=True, text=True) print(result.stdout) diff --git a/vreapis/containerizer/urls.py b/vreapis/containerizer/urls.py index c8c384b4..c9633291 100644 --- a/vreapis/containerizer/urls.py +++ b/vreapis/containerizer/urls.py @@ -3,6 +3,8 @@ urlpatterns = [ path('baseimagetags/', views.get_base_images), - path('extractorhandler/', views.ExtractorHandler.as_view()), - path('cellshandler/', views.CellsHandler.as_view()), + path('extract/', views.ExtractorHandler.as_view()), + path('types/', views.TypesHandler.as_view()), + path('baseimage/', views.BaseImageHandler.as_view()), + path('addcell/', views.CellsHandler.as_view()), ] diff --git a/vreapis/tests/emulated-frontend/dat/extract.json b/vreapis/tests/emulated-frontend/dat/extract.json new file mode 100644 index 00000000..57768136 --- /dev/null +++ b/vreapis/tests/emulated-frontend/dat/extract.json @@ -0,0 +1,52 @@ +[ + { + "save": false, + "kernel": "ipython", + "cell_index": 0, + "notebook": { + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.9" + } + }, + "nbformat_minor": 5, + "nbformat": 4, + "cells": [ + { + "cell_type": "code", + "source": "# input names\nnames = [\"Alice\", \"Bob\"]", + "metadata": { + "trusted": true + }, + "execution_count": null, + "outputs": [], + "id": "87f87efd-50e8-41cd-b4f6-2ba14d442cfb" + }, + { + "cell_type": "code", + "source": "# print names\nfor name in names:\n print(f\"Hello, {name}!\")", + "metadata": { + "trusted": true + }, + "execution_count": null, + "outputs": [], + "id": "848c006a-bf2b-439d-8fec-ed16c5eae66d" + } + ] + } + } +] \ No newline at end of file diff --git a/vreapis/tests/emulated-frontend/extract.py b/vreapis/tests/emulated-frontend/extract.py new file mode 100644 index 00000000..36b09eba --- /dev/null +++ b/vreapis/tests/emulated-frontend/extract.py @@ -0,0 +1,14 @@ +import requests +import json +import os + +script_path: str = os.path.dirname(os.path.realpath(__file__)) +endpoint: str = f"{os.getenv('API_ENDPOINT')}/api/containerizer/extract" +session = requests.Session() + +with open(f'{script_path}/dat/extract.json') as f: + bodies: list[dict[str, any]] = json.load(f) + +for body in bodies: + response = session.post(endpoint, body, verify=False) + print(response.json()) From fe4c9d2d64692772889f1fd6c8d841d91f3ae7bd Mon Sep 17 00:00:00 2001 From: AndyBRoswell Date: Wed, 12 Jun 2024 00:32:33 +0200 Subject: [PATCH 026/149] migrating TypesHandler, BaseImageHandler, CellsHandler --- vreapis/tests/emulated-frontend/extract.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/vreapis/tests/emulated-frontend/extract.py b/vreapis/tests/emulated-frontend/extract.py index 36b09eba..a63e269e 100644 --- a/vreapis/tests/emulated-frontend/extract.py +++ b/vreapis/tests/emulated-frontend/extract.py @@ -3,12 +3,13 @@ import os script_path: str = os.path.dirname(os.path.realpath(__file__)) -endpoint: str = f"{os.getenv('API_ENDPOINT')}/api/containerizer/extract" +endpoint: str = f"{os.getenv('API_ENDPOINT')}/api/containerizer/extract/" +headers: dict = {"Authorization": f"Token {os.getenv('NAAVRE_API_TOKEN')}"} session = requests.Session() with open(f'{script_path}/dat/extract.json') as f: bodies: list[dict[str, any]] = json.load(f) for body in bodies: - response = session.post(endpoint, body, verify=False) - print(response.json()) + response = session.post(endpoint, body, headers=headers, verify=False) + print(response.text) From 53b9fdbebaad65851ee8248fc4ec1c59f927e0d2 Mon Sep 17 00:00:00 2001 From: AndyBRoswell Date: Wed, 12 Jun 2024 11:37:55 +0200 Subject: [PATCH 027/149] [bugfix] containerizer/tests.py --- vreapis/containerizer/tests.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vreapis/containerizer/tests.py b/vreapis/containerizer/tests.py index 18b90867..fb56a8c7 100644 --- a/vreapis/containerizer/tests.py +++ b/vreapis/containerizer/tests.py @@ -164,7 +164,7 @@ def test(self): notebook = json.load(file) file.close() client = Client() - response = client.post('/api/containerizer/extractorhandler/', headers=get_auth_header(), data=notebook, content_type="application/json") + response = client.post('/api/containerizer/extract/', headers=get_auth_header(), data=notebook, content_type="application/json") self.assertEqual(response.status_code, 200) # get JSON response JSON_response = json.loads(response.data) @@ -198,7 +198,7 @@ def create_cell_and_add_to_cat(self, cell_path=None): return test_cell, cell def call_cell_handler(self): - return self.client.post('/api/containerizer/cellshandler/', content_type='application/json', headers=get_auth_header()) + return self.client.post('/api/containerizer/addcell/', content_type='application/json', headers=get_auth_header()) def delete_text(self, file_path, text_to_delete): # Read the file From 61a757d3823c216d86a616cf148cc4fc4c29b744 Mon Sep 17 00:00:00 2001 From: AndyBRoswell Date: Wed, 12 Jun 2024 19:01:15 +0200 Subject: [PATCH 028/149] [bugfix] APPEND_SLASH=False urlconf modified --- vreapis/containerizer/tests.py | 4 ++-- vreapis/containerizer/urls.py | 10 +++++----- vreapis/tests/emulated-frontend/extract.py | 9 ++++++--- vreapis/vreapis/settings/base.py | 1 + 4 files changed, 14 insertions(+), 10 deletions(-) diff --git a/vreapis/containerizer/tests.py b/vreapis/containerizer/tests.py index fb56a8c7..759beccd 100644 --- a/vreapis/containerizer/tests.py +++ b/vreapis/containerizer/tests.py @@ -164,7 +164,7 @@ def test(self): notebook = json.load(file) file.close() client = Client() - response = client.post('/api/containerizer/extract/', headers=get_auth_header(), data=notebook, content_type="application/json") + response = client.post('/api/containerizer/extract', headers=get_auth_header(), data=notebook, content_type="application/json") self.assertEqual(response.status_code, 200) # get JSON response JSON_response = json.loads(response.data) @@ -198,7 +198,7 @@ def create_cell_and_add_to_cat(self, cell_path=None): return test_cell, cell def call_cell_handler(self): - return self.client.post('/api/containerizer/addcell/', content_type='application/json', headers=get_auth_header()) + return self.client.post('/api/containerizer/addcell', content_type='application/json', headers=get_auth_header()) def delete_text(self, file_path, text_to_delete): # Read the file diff --git a/vreapis/containerizer/urls.py b/vreapis/containerizer/urls.py index c9633291..c092c654 100644 --- a/vreapis/containerizer/urls.py +++ b/vreapis/containerizer/urls.py @@ -2,9 +2,9 @@ from . import views urlpatterns = [ - path('baseimagetags/', views.get_base_images), - path('extract/', views.ExtractorHandler.as_view()), - path('types/', views.TypesHandler.as_view()), - path('baseimage/', views.BaseImageHandler.as_view()), - path('addcell/', views.CellsHandler.as_view()), + path('baseimagetags', views.get_base_images), + path('extract', views.ExtractorHandler.as_view()), + path('types', views.TypesHandler.as_view()), + path('baseimage', views.BaseImageHandler.as_view()), + path('addcell', views.CellsHandler.as_view()), ] diff --git a/vreapis/tests/emulated-frontend/extract.py b/vreapis/tests/emulated-frontend/extract.py index a63e269e..9baee585 100644 --- a/vreapis/tests/emulated-frontend/extract.py +++ b/vreapis/tests/emulated-frontend/extract.py @@ -3,13 +3,16 @@ import os script_path: str = os.path.dirname(os.path.realpath(__file__)) -endpoint: str = f"{os.getenv('API_ENDPOINT')}/api/containerizer/extract/" -headers: dict = {"Authorization": f"Token {os.getenv('NAAVRE_API_TOKEN')}"} +endpoint: str = f"{os.getenv('API_ENDPOINT')}/api/containerizer/extract" +headers: dict = { + "Content-Type": "application/json", + "Authorization": f"Token {os.getenv('NAAVRE_API_TOKEN')}", +} session = requests.Session() with open(f'{script_path}/dat/extract.json') as f: bodies: list[dict[str, any]] = json.load(f) for body in bodies: - response = session.post(endpoint, body, headers=headers, verify=False) + response = session.post(endpoint, json.dumps(body), headers=headers, verify=False) print(response.text) diff --git a/vreapis/vreapis/settings/base.py b/vreapis/vreapis/settings/base.py index 654a7f3c..075aa5ee 100644 --- a/vreapis/vreapis/settings/base.py +++ b/vreapis/vreapis/settings/base.py @@ -179,6 +179,7 @@ }, } +APPEND_SLASH = False BASE_PATH = os.environ.get('BASE_PATH', '').strip('/') From 9f09daad471c996cd5e6360f112b557391a9f84d Mon Sep 17 00:00:00 2001 From: AndyBRoswell Date: Thu, 13 Jun 2024 00:03:13 +0200 Subject: [PATCH 029/149] [bugfix] refined tests --- tilt/vreapis/Dockerfile | 4 +- vreapis/containerizer/tests.py | 2 +- .../{extract.py => containerizer.py} | 10 +- .../emulated-frontend/dat/baseimage.json | 1 + .../tests/emulated-frontend/dat/extract.json | 94 +++++++++---------- 5 files changed, 56 insertions(+), 55 deletions(-) rename vreapis/tests/emulated-frontend/{extract.py => containerizer.py} (70%) create mode 100644 vreapis/tests/emulated-frontend/dat/baseimage.json diff --git a/tilt/vreapis/Dockerfile b/tilt/vreapis/Dockerfile index cee789f9..fe99d3b8 100644 --- a/tilt/vreapis/Dockerfile +++ b/tilt/vreapis/Dockerfile @@ -1,4 +1,6 @@ -FROM python:3.12.2-slim +# FROM python:3.12.2-slim +FROM python:3.11.9-slim +# Dependency pytype [used by PyExtractor] complains 'Python versions > 3.11 are not yet supported' WORKDIR /app diff --git a/vreapis/containerizer/tests.py b/vreapis/containerizer/tests.py index 759beccd..e3674de6 100644 --- a/vreapis/containerizer/tests.py +++ b/vreapis/containerizer/tests.py @@ -52,7 +52,7 @@ def test_get_base_images(self): dummy_user = User.objects.create_user(dummy_username, password=dummy_password) client.login(username=dummy_username, password=dummy_password) - response = client.get('/api/containerizer/baseimagetags/', headers=get_auth_header()) + response = client.get('/api/containerizer/baseimagetags', headers=get_auth_header()) self.assertEqual(response.status_code, 200) images = response.json() self.assertIsInstance(images, dict) diff --git a/vreapis/tests/emulated-frontend/extract.py b/vreapis/tests/emulated-frontend/containerizer.py similarity index 70% rename from vreapis/tests/emulated-frontend/extract.py rename to vreapis/tests/emulated-frontend/containerizer.py index 9baee585..83977320 100644 --- a/vreapis/tests/emulated-frontend/extract.py +++ b/vreapis/tests/emulated-frontend/containerizer.py @@ -1,6 +1,7 @@ import requests import json import os +import sys script_path: str = os.path.dirname(os.path.realpath(__file__)) endpoint: str = f"{os.getenv('API_ENDPOINT')}/api/containerizer/extract" @@ -10,9 +11,8 @@ } session = requests.Session() -with open(f'{script_path}/dat/extract.json') as f: - bodies: list[dict[str, any]] = json.load(f) - -for body in bodies: +for i in range(1, len(sys.argv)): + with open(f'{script_path}/dat/{sys.argv[i]}.json') as f: + body: dict[str, any] = json.load(f) response = session.post(endpoint, json.dumps(body), headers=headers, verify=False) - print(response.text) + print(response.text) \ No newline at end of file diff --git a/vreapis/tests/emulated-frontend/dat/baseimage.json b/vreapis/tests/emulated-frontend/dat/baseimage.json new file mode 100644 index 00000000..a5f93371 --- /dev/null +++ b/vreapis/tests/emulated-frontend/dat/baseimage.json @@ -0,0 +1 @@ +{"image": {"build": "ghcr.io/qcdis/naavre/naavre-cell-build-python:v0.14", "runtime": "ghcr.io/qcdis/naavre/naavre-cell-runtime-python:v0.14"}} \ No newline at end of file diff --git a/vreapis/tests/emulated-frontend/dat/extract.json b/vreapis/tests/emulated-frontend/dat/extract.json index 57768136..6350a958 100644 --- a/vreapis/tests/emulated-frontend/dat/extract.json +++ b/vreapis/tests/emulated-frontend/dat/extract.json @@ -1,52 +1,50 @@ -[ - { - "save": false, - "kernel": "ipython", - "cell_index": 0, - "notebook": { - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" +{ + "save": false, + "kernel": "ipython", + "cell_index": 0, + "notebook": { + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.9" + } + }, + "nbformat_minor": 5, + "nbformat": 4, + "cells": [ + { + "cell_type": "code", + "source": "# input names\nnames = [\"Alice\", \"Bob\"]", + "metadata": { + "trusted": true }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.9" - } + "execution_count": null, + "outputs": [], + "id": "87f87efd-50e8-41cd-b4f6-2ba14d442cfb" }, - "nbformat_minor": 5, - "nbformat": 4, - "cells": [ - { - "cell_type": "code", - "source": "# input names\nnames = [\"Alice\", \"Bob\"]", - "metadata": { - "trusted": true - }, - "execution_count": null, - "outputs": [], - "id": "87f87efd-50e8-41cd-b4f6-2ba14d442cfb" + { + "cell_type": "code", + "source": "# print names\nfor name in names:\n print(f\"Hello, {name}!\")", + "metadata": { + "trusted": true }, - { - "cell_type": "code", - "source": "# print names\nfor name in names:\n print(f\"Hello, {name}!\")", - "metadata": { - "trusted": true - }, - "execution_count": null, - "outputs": [], - "id": "848c006a-bf2b-439d-8fec-ed16c5eae66d" - } - ] - } + "execution_count": null, + "outputs": [], + "id": "848c006a-bf2b-439d-8fec-ed16c5eae66d" + } + ] } -] \ No newline at end of file +} From ab385acc0af346263722c20d95e2bd9f824c786d Mon Sep 17 00:00:00 2001 From: AndyBRoswell Date: Thu, 13 Jun 2024 00:29:32 +0200 Subject: [PATCH 030/149] [bugfix] BaseImageHandler, Logger --- vreapis/containerizer/views.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/vreapis/containerizer/views.py b/vreapis/containerizer/views.py index 3132eaff..48204c50 100644 --- a/vreapis/containerizer/views.py +++ b/vreapis/containerizer/views.py @@ -38,8 +38,8 @@ import common -def return_error(err_msg: str, e: Optional[Exception], stat: status = status.HTTP_400_BAD_REQUEST) -> Response: - common.logger.error(err_msg, e) +def return_error(err_msg: str = 'Unknown ERROR', e: Optional[Exception] = None, stat: status = status.HTTP_400_BAD_REQUEST) -> Response: + common.logger.error(err_msg, exc_info=e) return Response(err_msg, status=stat) @@ -169,6 +169,7 @@ def post(self, request: Request): p_type = payload['type'] cell = Catalog.editor_buffer cell.types[port] = p_type + return Response({}) # must return a Response, or 500 occurs class BaseImageHandler(APIView, Catalog): @@ -181,6 +182,7 @@ def post(self, request: Request): base_image = payload['image'] cell = Catalog.editor_buffer cell.base_image = base_image + return Response({}) class CellsHandler(APIView, Catalog): From 36fac0fddb2896d743e1a8d8c095d7d8f1249249 Mon Sep 17 00:00:00 2001 From: AndyBRoswell Date: Thu, 13 Jun 2024 23:42:35 +0200 Subject: [PATCH 031/149] [bugfix] BaseImageHandler, Logger --- vreapis/common.py | 13 ++++++++-- vreapis/containerizer/views.py | 17 ++++++++---- vreapis/db/catalog.py | 47 ++++++++++++++++++++++++++++++++-- 3 files changed, 68 insertions(+), 9 deletions(-) diff --git a/vreapis/common.py b/vreapis/common.py index d965ac3c..01e3360f 100644 --- a/vreapis/common.py +++ b/vreapis/common.py @@ -3,13 +3,22 @@ import os import requests.adapters -project_root = os.path.dirname(os.path.abspath(__file__)) +max_retry_count: int = 10 +initial_retry_delay: int | float = 0.1 +max_retry_delay: int | float = 5 + +project_root: str = os.path.dirname(os.path.abspath(__file__)) # customized requests.Session [w/ auto retry] session = requests.Session() -retry_adapter = requests.adapters.HTTPAdapter(max_retries=urllib3.Retry(total=10, backoff_factor=0.1, backoff_max=2, status_forcelist=[500, 502, 503, 504])) +retry_adapter = requests.adapters.HTTPAdapter(max_retries=urllib3.Retry(total=max_retry_count, backoff_factor=initial_retry_delay, backoff_max=max_retry_delay, status_forcelist=[500, 502, 503, 504])) session.mount('http://', retry_adapter) session.mount('https://', retry_adapter) # global logger logger = logging.getLogger(__name__) + + +def retry_delay(cumulated_retry_count: int, initial_delay: int | float = initial_retry_delay, max_delay: int | float = max_retry_delay) -> int | float: + """delay is in seconds""" + return min(initial_delay ** cumulated_retry_count, max_delay) diff --git a/vreapis/containerizer/views.py b/vreapis/containerizer/views.py index 48204c50..44d7d288 100644 --- a/vreapis/containerizer/views.py +++ b/vreapis/containerizer/views.py @@ -4,6 +4,7 @@ import re import sys import copy +import time import hashlib import uuid from typing import Optional @@ -370,7 +371,7 @@ def query_registry_for_image(self, image_repo, image_name): else: return None - def dispatch_github_workflow(owner, repository_name, task_name, files_info, repository_token, image, wf_id=None, image_version=None): + def dispatch_github_workflow(self, owner, repository_name, task_name, files_info, repository_token, image, wf_id=None, image_version=None): url = CellsHandler.github_url_repos + '/' + owner + '/' + repository_name + '/actions/workflows/' + CellsHandler.github_workflow_file_name + '/dispatches' resp = common.session.post( url=url, @@ -467,10 +468,16 @@ def post(self, request: Request): repository_name = url_repos.split('https://github.com/')[1].split('/')[1] if '.git' in repository_name: repository_name = repository_name.split('.git')[0] - try: - gh_repository = gh_token.get_repo(owner + '/' + repository_name) - except Exception as ex: - return return_error(f'Error getting repository', ex) + retry_count: int = 0 + while True: + try: + gh_repository = gh_token.get_repo(owner + '/' + repository_name) + break + except Exception as ex: + time.sleep(common.retry_delay(retry_count)) + retry_count += 1 + if retry_count >= common.max_retry_count: + return return_error(f'Error getting repository', ex) do_dispatch_github_workflow, image_version = self.create_or_update_cell_in_repository(task_name=current_cell.task_name, repository=gh_repository, files_info=files_info, ) wf_id = str(uuid.uuid4()) diff --git a/vreapis/db/catalog.py b/vreapis/db/catalog.py index 746dff35..3aa13e32 100644 --- a/vreapis/db/catalog.py +++ b/vreapis/db/catalog.py @@ -1,5 +1,6 @@ import logging import os +import re from pathlib import Path from tinydb import TinyDB, where @@ -38,7 +39,6 @@ def add_search_entry(cls, query: dict): def update_cell(cls, cell: Cell): cls.cells.update(cell.__dict__, where('node_id') == cell.node_id) - @classmethod def get_search_entries(cls): return cls.search_entry.all() @@ -72,7 +72,7 @@ def get_registry_credentials(cls): @classmethod def get_repository_credentials(cls): - credentials = cls.repository.all() + credentials = cls.repositories.all() return credentials @classmethod @@ -181,3 +181,46 @@ def cast_document_to_cell(cls, cell_document): kernel=cell_document.get('kernel', ''), notebook_dict=cell_document.get('notebook_dict', {}) ) + + @classmethod + def get_registry_url(cls, registry_url, github_url): + """ Convert registry URL + + https://hub.docker.com/u/my_username/ -> docker.io/my_username + oci://ghcr.io/my_username/my_repo/ -> ghcr.io/my_username/my_repo + oci://my_domain/my/custom/path/ -> my_domain/my/custom/path + None -> ghcr.io url, derived from github_url + + Resulting urls can be converted to pullable, e.g.: + + docker pull {url}/{image_name}:{tag} + + where image_name doesn't contain any path information (e.g. my-cell-name) + + """ + if registry_url: + m = re.match(r'^https://hub\.docker\.com/u/(\w+)/?$', registry_url) + if m: + return f"docker.io/{m.group(1)}" + m = re.match(r'^oci://([\w\./-]+?)/?$', registry_url) + if m: + return m.group(1) + raise ValueError(f"Could not parse registry url: {registry_url}") + else: + m = re.match(r'^https://github.com/([\w-]+/[\w-]+)(?:\.git)?', github_url) + if m: + return f"ghcr.io/{m.group(1).lower()}" + + +github_url: str = os.getenv('CELL_GITHUB') +github_token: str = os.getenv('CELL_GITHUB_TOKEN') +registry_url: str = os.getenv('REGISTRY_URL') +# force: bool = True if os.getenv('FORCED_CREDENTIALS_REPLACEMENT', 'false').lower() == 'true' else 'false' +registry_url = Catalog.get_registry_url(registry_url, github_url) + +input_repository_credentials = Repository(github_url.split('https://github.com/')[1], github_url, github_token) +Catalog.add_gh_credentials(input_repository_credentials) +Catalog.add_repository_credentials(input_repository_credentials) + +input_registry_credentials = Repository(registry_url, registry_url, None) +Catalog.add_registry_credentials(input_registry_credentials) From 2716ebcae3483e04126d94395483f85543e6a3e6 Mon Sep 17 00:00:00 2001 From: AndyBRoswell Date: Fri, 14 Jun 2024 11:18:28 +0200 Subject: [PATCH 032/149] [bugfix] CellsHandler --- vreapis/common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vreapis/common.py b/vreapis/common.py index 01e3360f..c432ec8a 100644 --- a/vreapis/common.py +++ b/vreapis/common.py @@ -21,4 +21,4 @@ def retry_delay(cumulated_retry_count: int, initial_delay: int | float = initial_retry_delay, max_delay: int | float = max_retry_delay) -> int | float: """delay is in seconds""" - return min(initial_delay ** cumulated_retry_count, max_delay) + return min(initial_delay * 2 ** cumulated_retry_count, max_delay) From 7f8b8429f8a585d76593e3e4b07ded7de2ebf523 Mon Sep 17 00:00:00 2001 From: AndyBRoswell Date: Fri, 14 Jun 2024 18:28:29 +0200 Subject: [PATCH 033/149] del obsolete cells app --- vreapis/cells/__init__.py | 0 vreapis/cells/admin.py | 6 ---- vreapis/cells/apps.py | 6 ---- vreapis/cells/fixtures/catalogs.yaml | 7 ----- vreapis/cells/migrations/0001_initial.py | 38 ------------------------ vreapis/cells/migrations/__init__.py | 0 vreapis/cells/models.py | 27 ----------------- vreapis/cells/serializers.py | 11 ------- vreapis/cells/tests.py | 3 -- vreapis/cells/views.py | 30 ------------------- vreapis/vreapis/settings/base.py | 1 - vreapis/vreapis/urls.py | 2 -- 12 files changed, 131 deletions(-) delete mode 100644 vreapis/cells/__init__.py delete mode 100644 vreapis/cells/admin.py delete mode 100644 vreapis/cells/apps.py delete mode 100644 vreapis/cells/fixtures/catalogs.yaml delete mode 100644 vreapis/cells/migrations/0001_initial.py delete mode 100644 vreapis/cells/migrations/__init__.py delete mode 100644 vreapis/cells/models.py delete mode 100644 vreapis/cells/serializers.py delete mode 100644 vreapis/cells/tests.py delete mode 100644 vreapis/cells/views.py diff --git a/vreapis/cells/__init__.py b/vreapis/cells/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/vreapis/cells/admin.py b/vreapis/cells/admin.py deleted file mode 100644 index 21c068ba..00000000 --- a/vreapis/cells/admin.py +++ /dev/null @@ -1,6 +0,0 @@ -from django.contrib import admin - -from cells.models import Catalog, Cell - -admin.site.register(Catalog) -admin.site.register(Cell) diff --git a/vreapis/cells/apps.py b/vreapis/cells/apps.py deleted file mode 100644 index 9833cff9..00000000 --- a/vreapis/cells/apps.py +++ /dev/null @@ -1,6 +0,0 @@ -from django.apps import AppConfig - - -class CellsConfig(AppConfig): - default_auto_field = 'django.db.models.BigAutoField' - name = 'cells' diff --git a/vreapis/cells/fixtures/catalogs.yaml b/vreapis/cells/fixtures/catalogs.yaml deleted file mode 100644 index 7c3dac5a..00000000 --- a/vreapis/cells/fixtures/catalogs.yaml +++ /dev/null @@ -1,7 +0,0 @@ -- model: cells.Catalog - pk: 1 - fields: - name: "Summer School Catalog" - description: "Summer School Catalog" - vlab: 1 - diff --git a/vreapis/cells/migrations/0001_initial.py b/vreapis/cells/migrations/0001_initial.py deleted file mode 100644 index ceb260f2..00000000 --- a/vreapis/cells/migrations/0001_initial.py +++ /dev/null @@ -1,38 +0,0 @@ -# Generated by Django 4.0.5 on 2022-06-29 20:00 - -from django.db import migrations, models -import django.db.models.deletion - - -class Migration(migrations.Migration): - - initial = True - - dependencies = [ - ('virtual_labs', '0004_remove_virtuallab_profiles_and_more'), - ] - - operations = [ - migrations.CreateModel( - name='Catalog', - fields=[ - ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('name', models.CharField(max_length=100, null=True)), - ('description', models.TextField(null=True)), - ('vlab', models.ForeignKey(null=True, on_delete=django.db.models.deletion.DO_NOTHING, to='virtual_labs.virtuallab')), - ], - ), - migrations.CreateModel( - name='Cell', - fields=[ - ('uuid', models.UUIDField(primary_key=True, serialize=False)), - ('name', models.CharField(max_length=100, null=True)), - ('description', models.TextField(null=True)), - ('container_image', models.CharField(max_length=100, null=True)), - ('registry_url', models.URLField(null=True)), - ('repository_url', models.URLField(null=True)), - ('cell_metadata', models.JSONField(null=True)), - ('catalog', models.ForeignKey(null=True, on_delete=django.db.models.deletion.DO_NOTHING, to='cells.catalog')), - ], - ), - ] diff --git a/vreapis/cells/migrations/__init__.py b/vreapis/cells/migrations/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/vreapis/cells/models.py b/vreapis/cells/models.py deleted file mode 100644 index 63f885ce..00000000 --- a/vreapis/cells/models.py +++ /dev/null @@ -1,27 +0,0 @@ -from django.db import models -from virtual_labs.models import VirtualLab - - -class Catalog(models.Model): - - name = models.CharField(max_length=100, null=True) - description = models.TextField(null=True) - vlab = models.ForeignKey(VirtualLab, on_delete=models.DO_NOTHING, null=True) - - def __str__(self): - return self.name - - -class Cell(models.Model): - - uuid = models.UUIDField(primary_key=True) - name = models.CharField(max_length=100, null=True) - description = models.TextField(null=True) - container_image = models.CharField(max_length=100, null=True) - registry_url = models.URLField(null=True) - repository_url = models.URLField(null=True) - cell_metadata = models.JSONField(null=True) - catalog = models.ForeignKey(Catalog, on_delete=models.DO_NOTHING, null=True) - - def __str__(self) -> str: - return self.name \ No newline at end of file diff --git a/vreapis/cells/serializers.py b/vreapis/cells/serializers.py deleted file mode 100644 index a27b942e..00000000 --- a/vreapis/cells/serializers.py +++ /dev/null @@ -1,11 +0,0 @@ -from dataclasses import fields -from rest_framework import serializers -from . import models - - -class CellSerializer(serializers.ModelSerializer): - - class Meta: - - model = models.Cell - fields = "__all__" \ No newline at end of file diff --git a/vreapis/cells/tests.py b/vreapis/cells/tests.py deleted file mode 100644 index 7ce503c2..00000000 --- a/vreapis/cells/tests.py +++ /dev/null @@ -1,3 +0,0 @@ -from django.test import TestCase - -# Create your tests here. diff --git a/vreapis/cells/views.py b/vreapis/cells/views.py deleted file mode 100644 index 42398932..00000000 --- a/vreapis/cells/views.py +++ /dev/null @@ -1,30 +0,0 @@ -from rest_framework import mixins, viewsets -from rest_framework.permissions import IsAuthenticated - -from vreapis.views import GetSerializerMixin - -from . import models, serializers - - -class CellsViewSet(GetSerializerMixin, - mixins.RetrieveModelMixin, - mixins.UpdateModelMixin, - mixins.ListModelMixin, - mixins.CreateModelMixin, - viewsets.GenericViewSet): - queryset = models.Cell.objects.all() - serializer_class = serializers.CellSerializer - serializer_action_classes = { - 'list': serializers.CellSerializer - } - - def get_permissions(self): - if self.action in ['list', 'retrieve']: - permission_classes = [] - else: - permission_classes = [IsAuthenticated] - return [permission() for permission in permission_classes] - - def create(self, request, *args, **kwargs): - print(request.data) - return super().create(request) diff --git a/vreapis/vreapis/settings/base.py b/vreapis/vreapis/settings/base.py index 075aa5ee..4aa83234 100644 --- a/vreapis/vreapis/settings/base.py +++ b/vreapis/vreapis/settings/base.py @@ -35,7 +35,6 @@ # Application definition INSTALLED_APPS = [ - 'cells', 'workflows', 'virtual_labs', 'data_products', diff --git a/vreapis/vreapis/urls.py b/vreapis/vreapis/urls.py index fdb019e8..d01cd096 100644 --- a/vreapis/vreapis/urls.py +++ b/vreapis/vreapis/urls.py @@ -17,7 +17,6 @@ from django.urls import path, include from rest_framework import routers from virtual_labs.views import VirtualLabViewSet, VirtualLabInstanceViewSet -from cells.views import CellsViewSet from workflows.views import WorkflowViewSet from data_products.views import DataProductsViewSet, GeoDataProductsViewSet from paas_configuration.views import PaasConfigurationViewSet @@ -31,7 +30,6 @@ router.register(r'vlabs', VirtualLabViewSet) router.register(r'vlab_instances', VirtualLabInstanceViewSet) router.register(r'workflows', WorkflowViewSet) -router.register(r'cells', CellsViewSet) router.register(r'dataprods', DataProductsViewSet) router.register(r'geodataprods', GeoDataProductsViewSet) router.register(r'paas_configuration', PaasConfigurationViewSet) From 8251c3eb25937335f224d01657bbd939bd29012f Mon Sep 17 00:00:00 2001 From: AndyBRoswell Date: Sun, 16 Jun 2024 21:38:59 +0200 Subject: [PATCH 034/149] [tbc] psql adaptation --- vreapis/catalog/__init__.py | 0 vreapis/catalog/admin.py | 3 + vreapis/catalog/apps.py | 6 ++ vreapis/catalog/models.py | 133 +++++++++++++++++++++++++++++++ vreapis/catalog/serializers.py | 8 ++ vreapis/catalog/tests.py | 3 + vreapis/catalog/views.py | 3 + vreapis/common.py | 2 + vreapis/containerizer/models.py | 4 +- vreapis/containerizer/urls.py | 6 +- vreapis/containerizer/views.py | 93 ++++++++++----------- vreapis/db/catalog.py | 2 - vreapis/db/cell.py | 29 +------ vreapis/vreapis/settings/base.py | 1 + 14 files changed, 210 insertions(+), 83 deletions(-) create mode 100644 vreapis/catalog/__init__.py create mode 100644 vreapis/catalog/admin.py create mode 100644 vreapis/catalog/apps.py create mode 100644 vreapis/catalog/models.py create mode 100644 vreapis/catalog/serializers.py create mode 100644 vreapis/catalog/tests.py create mode 100644 vreapis/catalog/views.py diff --git a/vreapis/catalog/__init__.py b/vreapis/catalog/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/vreapis/catalog/admin.py b/vreapis/catalog/admin.py new file mode 100644 index 00000000..8c38f3f3 --- /dev/null +++ b/vreapis/catalog/admin.py @@ -0,0 +1,3 @@ +from django.contrib import admin + +# Register your models here. diff --git a/vreapis/catalog/apps.py b/vreapis/catalog/apps.py new file mode 100644 index 00000000..a5993c68 --- /dev/null +++ b/vreapis/catalog/apps.py @@ -0,0 +1,6 @@ +from django.apps import AppConfig + + +class CatalogConfig(AppConfig): + default_auto_field = 'django.db.models.BigAutoField' + name = 'catalog' diff --git a/vreapis/catalog/models.py b/vreapis/catalog/models.py new file mode 100644 index 00000000..5d925fc9 --- /dev/null +++ b/vreapis/catalog/models.py @@ -0,0 +1,133 @@ +import json +import re + +from slugify import slugify +from django.db import models + +import common + + +class Cell(models.Model): + title = models.CharField(max_length=common.default_varchar_length) + task_name = models.CharField(max_length=common.default_varchar_length) + original_source = models.CharField(max_length=common.default_varchar_length) + types = models.JSONField(blank=True, null=True, default=dict) + base_image = models.JSONField(blank=True, null=True) + inputs = models.JSONField(blank=True, null=True, default=list) + outputs = models.JSONField(blank=True, null=True, default=list) + params = models.JSONField(blank=True, null=True, default=list) + param_values = models.JSONField(blank=True, null=True) + confs = models.JSONField(blank=True, null=True) + dependencies = models.JSONField(blank=True, null=True) + chart_obj = models.JSONField(blank=True, null=True) + node_id = models.CharField(max_length=common.default_varchar_length, primary_key=True) + container_source = models.CharField(max_length=common.default_varchar_length) + global_conf = models.JSONField(blank=True, null=True) + kernel = models.CharField(max_length=common.default_varchar_length) + notebook_dict = models.JSONField(blank=True, null=True) + image_version = models.CharField(max_length=common.default_varchar_length) + + def _extract_types(self, vars_dict): + """ Extract types to self.types and return list of var names + + :param vars_dict: {'var1': {'name: 'var1', 'type': 'str'}, 'var2': ...} + :return: ['var1', 'var2', ...] + """ + names = [] + for var_props in vars_dict.values(): + var_type = var_props['type'] + var_name = var_props['name'] + self.types[var_name] = var_type + names.append(var_name) + return names + + def add_inputs(self, inputs): + if isinstance(inputs, dict): + inputs = self._extract_types(inputs) + self.inputs = inputs + + def add_outputs(self, outputs): + if isinstance(outputs, dict): + outputs = self._extract_types(outputs) + self.outputs = outputs + + def add_params(self, params): + if isinstance(params, dict): + params = self._extract_types(params) + self.params = params + + def set_image_version(self, image_version): + self.image_version = image_version + + def add_param_values(self, params): + self.param_values = {} + if isinstance(params, dict): + for param_props in params.values(): + if 'value' in param_props: + self.param_values[param_props['name']] = param_props['value'] + + def concatenate_all_inputs(self): + self.all_inputs = list(self.inputs) + list(self.params) + + def clean_code(self): + indices_to_remove = [] + lines = self.original_source.splitlines() + self.original_source = "" + + for line_i in range(0, len(lines)): + line = lines[line_i] + # Do not remove line that startswith param_ if not in the self.params + if line.startswith('param_'): + # clean param name + pattern = r"\b(param_\w+)\b" + param_name = re.findall(pattern, line)[0] + if param_name in self.params: + indices_to_remove.append(line_i) + if re.match('^\s*(#|import|from)', line): + indices_to_remove.append(line_i) + + for ir in sorted(indices_to_remove, reverse=True): + lines.pop(ir) + + self.original_source = "\n".join(lines) + + def clean_task_name(self): + self.task_name = slugify(self.task_name) + + def clean_title(self): + self.title = slugify(self.title) + + def integrate_configuration(self): + lines = self.original_source.splitlines() + self.original_source = "" + for idx, conf in enumerate(self.generate_configuration()): + lines.insert(idx, conf) + self.original_source = "\n".join(lines) + + def generate_dependencies(self): + resolves = [] + for d in self.dependencies: + resolve_to = "import %s" % d['name'] + if d['module']: + resolve_to = "from %s %s" % (d['module'], resolve_to) + if d['asname']: + resolve_to += " as %s" % d['asname'] + resolves.append(resolve_to) + return resolves + + def generate_configuration(self): + resolves = [] + for c in self.confs: + resolves.append(self.confs[c]) + return resolves + + def generate_configuration_dict(self): + resolves = [] + for c in self.confs: + assignment = self.confs[c].split('=')[1].replace('=', '').strip() + conf = {c: assignment} + resolves.append(conf) + return resolves + + def toJSON(self): + return json.dumps(self, default=lambda o: o.__dict__, sort_keys=True, indent=4) diff --git a/vreapis/catalog/serializers.py b/vreapis/catalog/serializers.py new file mode 100644 index 00000000..f8d7c378 --- /dev/null +++ b/vreapis/catalog/serializers.py @@ -0,0 +1,8 @@ +from rest_framework.serializers import ModelSerializer +from . import models + + +class CellSerializer(ModelSerializer): + class Meta: + model = models.Cell + fields = '__all__' diff --git a/vreapis/catalog/tests.py b/vreapis/catalog/tests.py new file mode 100644 index 00000000..7ce503c2 --- /dev/null +++ b/vreapis/catalog/tests.py @@ -0,0 +1,3 @@ +from django.test import TestCase + +# Create your tests here. diff --git a/vreapis/catalog/views.py b/vreapis/catalog/views.py new file mode 100644 index 00000000..91ea44a2 --- /dev/null +++ b/vreapis/catalog/views.py @@ -0,0 +1,3 @@ +from django.shortcuts import render + +# Create your views here. diff --git a/vreapis/common.py b/vreapis/common.py index c432ec8a..5f4ff5bb 100644 --- a/vreapis/common.py +++ b/vreapis/common.py @@ -7,6 +7,8 @@ initial_retry_delay: int | float = 0.1 max_retry_delay: int | float = 5 +default_varchar_length: int = 255 + project_root: str = os.path.dirname(os.path.abspath(__file__)) # customized requests.Session [w/ auto retry] diff --git a/vreapis/containerizer/models.py b/vreapis/containerizer/models.py index 71a83623..b28b04f6 100644 --- a/vreapis/containerizer/models.py +++ b/vreapis/containerizer/models.py @@ -1,3 +1,3 @@ -from django.db import models -# Create your models here. + + diff --git a/vreapis/containerizer/urls.py b/vreapis/containerizer/urls.py index c092c654..43bb6528 100644 --- a/vreapis/containerizer/urls.py +++ b/vreapis/containerizer/urls.py @@ -4,7 +4,7 @@ urlpatterns = [ path('baseimagetags', views.get_base_images), path('extract', views.ExtractorHandler.as_view()), - path('types', views.TypesHandler.as_view()), - path('baseimage', views.BaseImageHandler.as_view()), - path('addcell', views.CellsHandler.as_view()), + path('addcell', views.CellsHandler.as_view({ + 'post': 'create', + })), ] diff --git a/vreapis/containerizer/views.py b/vreapis/containerizer/views.py index 44d7d288..c73c4945 100644 --- a/vreapis/containerizer/views.py +++ b/vreapis/containerizer/views.py @@ -13,19 +13,25 @@ import autopep8 from distro import distro import jinja2 -from rest_framework.permissions import IsAuthenticated +from django.db.models import QuerySet +from rest_framework.authentication import BaseAuthentication +from rest_framework.permissions import IsAuthenticated, BasePermission from rest_framework.response import Response from rest_framework.request import Request from rest_framework.views import APIView from rest_framework import status from rest_framework.decorators import api_view, authentication_classes, permission_classes +from rest_framework import viewsets +from rest_framework.request import Request import requests import nbformat import jsonschema from slugify import slugify from github import Github, UnknownObjectException +from catalog.serializers import CellSerializer from containerizer.RContainerizer import RContainerizer +from db.repository import Repository from services.extractor.extractor import DummyExtractor from services.extractor.headerextractor import HeaderExtractor from services.extractor.pyextractor import PyExtractor @@ -34,7 +40,7 @@ import utils.cors from auth.simple import StaticTokenAuthentication from db.catalog import Catalog -from db.cell import Cell +from catalog.models import Cell import common @@ -155,43 +161,21 @@ def post(self, request: Request): chart = {'offset': {'x': 0, 'y': 0, }, 'scale': 1, 'nodes': {node_id: node}, 'links': {}, 'selected': {}, 'hovered': {}, } cell.chart_obj = chart - Catalog.editor_buffer = copy.deepcopy(cell) return Response(cell.toJSON()) -class TypesHandler(APIView, Catalog): - authentication_classes = [StaticTokenAuthentication] - permission_classes = [IsAuthenticated] - - def post(self, request: Request): - payload = request.data - common.logger.debug('TypesHandler. payload: ' + str(payload)) - port = payload['port'] - p_type = payload['type'] - cell = Catalog.editor_buffer - cell.types[port] = p_type - return Response({}) # must return a Response, or 500 occurs - - -class BaseImageHandler(APIView, Catalog): - authentication_classes = [StaticTokenAuthentication] - permission_classes = [IsAuthenticated] - - def post(self, request: Request): - payload = request.data - common.logger.debug('BaseImageHandler. payload: ' + str(payload)) - base_image = payload['image'] - cell = Catalog.editor_buffer - cell.base_image = base_image - return Response({}) - - -class CellsHandler(APIView, Catalog): - authentication_classes = [StaticTokenAuthentication] - permission_classes = [IsAuthenticated] - cells_path = os.path.join(str(Path.home()), 'NaaVRE', 'cells') - github_url_repos = 'https://api.github.com/repos' - github_workflow_file_name = 'build-push-docker.yml' +class CellsHandler(viewsets.ModelViewSet): + queryset: QuerySet = Cell.objects.all() + serializer_class = CellSerializer + authentication_classes: list[BaseAuthentication] = [StaticTokenAuthentication] + permission_classes: list[BasePermission] = [IsAuthenticated] + cells_path: str = os.path.join(str(Path.home()), 'NaaVRE', 'cells') + github_url_repos: str = 'https://api.github.com/repos' + github_workflow_file_name: str = 'build-push-docker.yml' + github_url: str = os.getenv('CELL_GITHUB') + github_token: str = os.getenv('CELL_GITHUB_TOKEN') + registry_url: str = os.getenv('REGISTRY_URL') + repo_name: str = github_url.split('https://github.com/')[1] def write_cell_to_file(self, current_cell): Path('/tmp/workflow_cells/cells').mkdir(parents=True, exist_ok=True) @@ -391,12 +375,15 @@ def dispatch_github_workflow(self, owner, repository_name, task_name, files_info ) return resp - def get(self, request: Request): - return return_error('Operation not supported.') + def get_registry_credentials(self) -> list[dict[str, str]]: + return [Repository(CellsHandler.registry_url, CellsHandler.registry_url, None).__dict__] - def post(self, request: Request): + def get_repositories(self) -> list[dict[str, str]]: + return [Repository(CellsHandler.repo_name, CellsHandler.github_url, CellsHandler.github_token).__dict__] + + def create(self, request: Request, *args, **kwargs): try: - current_cell = Catalog.editor_buffer + current_cell = Cell(**request.data) current_cell.clean_code() current_cell.clean_title() current_cell.clean_task_name() @@ -405,6 +392,7 @@ def post(self, request: Request): common.logger.debug('current_cell: ' + current_cell.toJSON()) all_vars = current_cell.params + current_cell.inputs + current_cell.outputs + # all_vars = json.loads(current_cell.params) + json.loads(current_cell.inputs) + json.loads(current_cell.outputs) for param_name in all_vars: if param_name not in current_cell.types: return return_error(f'{param_name} not in types') @@ -412,11 +400,14 @@ def post(self, request: Request): if not hasattr(current_cell, 'base_image'): return return_error(f'{current_cell.task_name} has not selected base image') try: - doc_cell = Catalog.get_cell_from_og_node_id(current_cell.node_id) - if doc_cell: - Catalog.update_cell(current_cell) - else: - Catalog.add_cell(current_cell) + # doc_cell = Catalog.get_cell_from_og_node_id(current_cell.node_id) + # if doc_cell: + # Catalog.update_cell(current_cell) + # else: + # Catalog.add_cell(current_cell) + serializer: CellSerializer = self.get_serializer(data=request.data) + serializer.is_valid(raise_exception=True) + instance, created = Cell.objects.update_or_create(defaults=serializer.validated_data) except Exception as ex: return return_error('Error adding or updating cell in catalog', ex) @@ -436,7 +427,7 @@ def post(self, request: Request): else: os.mkdir(cell_path) - registry_credentials = Catalog.get_registry_credentials() + registry_credentials = self.get_registry_credentials() if not registry_credentials or len(registry_credentials) <= 0: return return_error('Registry credentials not found') image_repo = registry_credentials[0]['url'] @@ -453,7 +444,7 @@ def post(self, request: Request): return return_error(f'Kernel {current_cell.kernel} not supported') # upload to GIT - cat_repositories = Catalog.get_repositories() + cat_repositories = self.get_repositories() repo_token = cat_repositories[0]['token'] if not repo_token: @@ -494,7 +485,11 @@ def post(self, request: Request): if resp.status_code != 201 and resp.status_code != 200 and resp.status_code != 204: return return_error(resp.text) current_cell.set_image_version(image_version) - Catalog.delete_cell_from_task_name(current_cell.task_name) - Catalog.add_cell(current_cell) + # Catalog.delete_cell_from_task_name(current_cell.task_name) + # Catalog.add_cell(current_cell) + Cell.objects.filter(task_name=current_cell.task_name).delete() + serializer = self.get_serializer(data=current_cell) + serializer.is_valid(raise_exception=True) + instance = serializer.save() return Response({'wf_id': wf_id, 'dispatched_github_workflow': do_dispatch_github_workflow, 'image_version': image_version}) diff --git a/vreapis/db/catalog.py b/vreapis/db/catalog.py index 3aa13e32..19542863 100644 --- a/vreapis/db/catalog.py +++ b/vreapis/db/catalog.py @@ -29,8 +29,6 @@ class Catalog: registry_credentials = db.table('registry_credentials') search_entry = db.table('search_entries') - editor_buffer: Cell - @classmethod def add_search_entry(cls, query: dict): cls.search_entry.insert(query) diff --git a/vreapis/db/cell.py b/vreapis/db/cell.py index 784ae1d1..378fb7b5 100644 --- a/vreapis/db/cell.py +++ b/vreapis/db/cell.py @@ -1,17 +1,10 @@ import json -import logging import re from slugify import slugify -logger = logging.getLogger(__name__) -logger.setLevel(logging.DEBUG) - - -class Cell: - logger = logging.getLogger(__name__) - logger.setLevel(logging.DEBUG) +class Cell(): title: str task_name: str original_source: str @@ -30,24 +23,7 @@ class Cell: notebook_json: dict image_version: str - def __init__( - self, - title, - task_name, - original_source, - inputs, - outputs, - params, - confs, - dependencies, - container_source, - chart_obj=None, - node_id='', - kernel='', - notebook_dict=None, - image_version=None - ) -> None: - + def __init__(self, title, task_name, original_source, inputs, outputs, params, confs, dependencies, container_source, chart_obj=None, node_id='', kernel='', notebook_dict=None, image_version=None) -> None: self.title = slugify(title.strip()) self.task_name = slugify(task_name) self.original_source = original_source @@ -97,7 +73,6 @@ def add_params(self, params): def set_image_version(self, image_version): self.image_version = image_version - def add_param_values(self, params): self.param_values = {} if isinstance(params, dict): diff --git a/vreapis/vreapis/settings/base.py b/vreapis/vreapis/settings/base.py index 4aa83234..026df70c 100644 --- a/vreapis/vreapis/settings/base.py +++ b/vreapis/vreapis/settings/base.py @@ -35,6 +35,7 @@ # Application definition INSTALLED_APPS = [ + 'catalog', 'workflows', 'virtual_labs', 'data_products', From ed9dcbb95eeb354844ef2415f8dcc4e2928d3963 Mon Sep 17 00:00:00 2001 From: AndyBRoswell Date: Mon, 17 Jun 2024 20:35:01 +0200 Subject: [PATCH 035/149] corrected CellsHandler unit test case --- vreapis/containerizer/tests.py | 29 +++++++++++++---------------- vreapis/containerizer/views.py | 1 - 2 files changed, 13 insertions(+), 17 deletions(-) diff --git a/vreapis/containerizer/tests.py b/vreapis/containerizer/tests.py index e3674de6..f4510756 100644 --- a/vreapis/containerizer/tests.py +++ b/vreapis/containerizer/tests.py @@ -180,26 +180,23 @@ class CellsHandlerTestCase(TestCase): def setUp(self): self.client = Client() - def create_cell_and_add_to_cat(self, cell_path=None): + def create_cell_and_add_to_cat(self, cell_path=None) -> (dict, dict): print('Creating cell from: ', cell_path) with open(cell_path, 'r') as file: cell = json.load(file) - file.close() - notebook_dict = {} - if 'notebook_dict' in cell: - notebook_dict = cell['notebook_dict'] - test_cell = Cell(cell['title'], cell['task_name'], cell['original_source'], cell['inputs'], - cell['outputs'], - cell['params'], cell['confs'], cell['dependencies'], cell['container_source'], - cell['chart_obj'], cell['node_id'], cell['kernel'], notebook_dict) - test_cell.types = cell['types'] - test_cell.base_image = cell['base_image'] - Catalog.editor_buffer = test_cell + # notebook_dict = {} + # if 'notebook_dict' in cell: + # notebook_dict = cell['notebook_dict'] + # test_cell = Cell(cell['title'], cell['task_name'], cell['original_source'], cell['inputs'], + # cell['outputs'], + # cell['params'], cell['confs'], cell['dependencies'], cell['container_source'], + # cell['chart_obj'], cell['node_id'], cell['kernel'], notebook_dict) + # test_cell.types = cell['types'] + # test_cell.base_image = cell['base_image'] + selected_keys: list[str] = ['title', 'task_name', 'original_source', 'inputs', 'outputs', 'params', 'confs', 'dependencies', 'container_source', 'chart_obj', 'node_id', 'kernel', 'types', 'base_image', 'notebook_dict'] + test_cell = {k: cell[k] for k in selected_keys if k in cell} return test_cell, cell - def call_cell_handler(self): - return self.client.post('/api/containerizer/addcell', content_type='application/json', headers=get_auth_header()) - def delete_text(self, file_path, text_to_delete): # Read the file with open(file_path, 'r') as file: @@ -341,7 +338,7 @@ def test(self): for cell_file in cells_files: cell_path = os.path.join(cells_json_path, cell_file) test_cell, cell = self.create_cell_and_add_to_cat(cell_path=cell_path) - response = self.call_cell_handler() + response = self.client.post('/api/containerizer/addcell', content_type='application/json', headers=get_auth_header(), data=test_cell) self.assertEqual(200, response.status_code) wf_id = response.data['wf_id'] wf_creation_utc = datetime.datetime.now(tz=datetime.timezone.utc) diff --git a/vreapis/containerizer/views.py b/vreapis/containerizer/views.py index c73c4945..d69e3511 100644 --- a/vreapis/containerizer/views.py +++ b/vreapis/containerizer/views.py @@ -392,7 +392,6 @@ def create(self, request: Request, *args, **kwargs): common.logger.debug('current_cell: ' + current_cell.toJSON()) all_vars = current_cell.params + current_cell.inputs + current_cell.outputs - # all_vars = json.loads(current_cell.params) + json.loads(current_cell.inputs) + json.loads(current_cell.outputs) for param_name in all_vars: if param_name not in current_cell.types: return return_error(f'{param_name} not in types') From 7effc3ef5c94a4f127cb7ae5bc60aa6d96671fb7 Mon Sep 17 00:00:00 2001 From: AndyBRoswell Date: Mon, 17 Jun 2024 21:05:13 +0200 Subject: [PATCH 036/149] [bugfix] class Cell(models.Model) --- vreapis/catalog/models.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vreapis/catalog/models.py b/vreapis/catalog/models.py index 5d925fc9..ab58066a 100644 --- a/vreapis/catalog/models.py +++ b/vreapis/catalog/models.py @@ -21,11 +21,11 @@ class Cell(models.Model): dependencies = models.JSONField(blank=True, null=True) chart_obj = models.JSONField(blank=True, null=True) node_id = models.CharField(max_length=common.default_varchar_length, primary_key=True) - container_source = models.CharField(max_length=common.default_varchar_length) + container_source = models.CharField(max_length=common.default_varchar_length, blank=True, null=True) global_conf = models.JSONField(blank=True, null=True) kernel = models.CharField(max_length=common.default_varchar_length) notebook_dict = models.JSONField(blank=True, null=True) - image_version = models.CharField(max_length=common.default_varchar_length) + image_version = models.CharField(max_length=common.default_varchar_length, blank=True, null=True) def _extract_types(self, vars_dict): """ Extract types to self.types and return list of var names From c8621ff77da1b72f0c5886fba3d65e9d173f1eda Mon Sep 17 00:00:00 2001 From: AndyBRoswell Date: Mon, 17 Jun 2024 22:51:57 +0200 Subject: [PATCH 037/149] [bugfix] CellsHandler, CellsHandlerTestCase --- vreapis/containerizer/tests.py | 12 ++++++------ vreapis/containerizer/views.py | 34 +++++++++++++++++++++++++++++++++- 2 files changed, 39 insertions(+), 7 deletions(-) diff --git a/vreapis/containerizer/tests.py b/vreapis/containerizer/tests.py index f4510756..08cf807b 100644 --- a/vreapis/containerizer/tests.py +++ b/vreapis/containerizer/tests.py @@ -24,7 +24,7 @@ from services.extractor.pyextractor import PyExtractor from services.extractor.rextractor import RExtractor from services.converter import ConverterReactFlowChart -from db.cell import Cell +from catalog.models import Cell base_path = '' if os.path.exists('resources'): @@ -350,8 +350,8 @@ def test(self): }) if 'skip_exec' in cell and cell['skip_exec']: continue - if 'python' in test_cell.kernel and 'skip_exec': - cell_path = os.path.join(CellsHandlerTestCase.cells_path, test_cell.task_name, 'task.py') + if 'python' in test_cell['kernel'] and 'skip_exec': + cell_path = os.path.join(CellsHandlerTestCase.cells_path, test_cell['task_name'], 'task.py') print('---------------------------------------------------') print('Executing cell: ', cell_path) if 'example_inputs' in cell: @@ -367,9 +367,9 @@ def test(self): print("return code:", cell_exec.returncode) print('---------------------------------------------------') self.assertEqual(0, cell_exec.returncode, 'Cell execution failed: ' + cell_file) - elif test_cell.kernel == 'IRkernel' and 'skip_exec': - cell_path = os.path.join(CellsHandlerTestCase.cells_path, test_cell.task_name, 'task.R') - run_local_cell_path = os.path.join(CellsHandlerTestCase.cells_path, test_cell.task_name, 'run_local.R') + elif test_cell['kernel'] == 'IRkernel' and 'skip_exec': + cell_path = os.path.join(CellsHandlerTestCase.cells_path, test_cell['task_name'], 'task.R') + run_local_cell_path = os.path.join(CellsHandlerTestCase.cells_path, test_cell['task_name'], 'run_local.R') shutil.copy(cell_path, run_local_cell_path) self.delete_text(run_local_cell_path, 'setwd(\'/app\')') example_inputs = '' diff --git a/vreapis/containerizer/views.py b/vreapis/containerizer/views.py index d69e3511..bb9af1c9 100644 --- a/vreapis/containerizer/views.py +++ b/vreapis/containerizer/views.py @@ -375,6 +375,35 @@ def dispatch_github_workflow(self, owner, repository_name, task_name, files_info ) return resp + @classmethod + def get_registry_url(cls, registry_url, github_url): + """ Convert registry URL + + https://hub.docker.com/u/my_username/ -> docker.io/my_username + oci://ghcr.io/my_username/my_repo/ -> ghcr.io/my_username/my_repo + oci://my_domain/my/custom/path/ -> my_domain/my/custom/path + None -> ghcr.io url, derived from github_url + + Resulting urls can be converted to pullable, e.g.: + + docker pull {url}/{image_name}:{tag} + + where image_name doesn't contain any path information (e.g. my-cell-name) + + """ + if registry_url: + m = re.match(r'^https://hub\.docker\.com/u/(\w+)/?$', registry_url) + if m: + return f"docker.io/{m.group(1)}" + m = re.match(r'^oci://([\w\./-]+?)/?$', registry_url) + if m: + return m.group(1) + raise ValueError(f"Could not parse registry url: {registry_url}") + else: + m = re.match(r'^https://github.com/([\w-]+/[\w-]+)(?:\.git)?', github_url) + if m: + return f"ghcr.io/{m.group(1).lower()}" + def get_registry_credentials(self) -> list[dict[str, str]]: return [Repository(CellsHandler.registry_url, CellsHandler.registry_url, None).__dict__] @@ -431,7 +460,7 @@ def create(self, request: Request, *args, **kwargs): return return_error('Registry credentials not found') image_repo = registry_credentials[0]['url'] if not image_repo: - return return_error('Registry not found') + return return_error(f'Registry not found. Registry credentials:{os.linesep}{registry_credentials}') if current_cell.kernel == "IRkernel": files_info = RContainerizer.get_files_info(cell=current_cell, cells_path=CellsHandler.cells_path) @@ -492,3 +521,6 @@ def create(self, request: Request, *args, **kwargs): instance = serializer.save() return Response({'wf_id': wf_id, 'dispatched_github_workflow': do_dispatch_github_workflow, 'image_version': image_version}) + + +CellsHandler.registry_url = CellsHandler.get_registry_url(CellsHandler.registry_url, CellsHandler.github_url) From 2dfa1831b24fc6f98e9f62151177b614a4f04ca5 Mon Sep 17 00:00:00 2001 From: AndyBRoswell Date: Mon, 17 Jun 2024 22:57:06 +0200 Subject: [PATCH 038/149] [bugfix] CellsHandler --- vreapis/containerizer/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vreapis/containerizer/views.py b/vreapis/containerizer/views.py index bb9af1c9..1d3761b9 100644 --- a/vreapis/containerizer/views.py +++ b/vreapis/containerizer/views.py @@ -435,7 +435,7 @@ def create(self, request: Request, *args, **kwargs): # Catalog.add_cell(current_cell) serializer: CellSerializer = self.get_serializer(data=request.data) serializer.is_valid(raise_exception=True) - instance, created = Cell.objects.update_or_create(defaults=serializer.validated_data) + instance, created = Cell.objects.update_or_create(node_id=serializer.validated_data['node_id'], defaults=serializer.validated_data) except Exception as ex: return return_error('Error adding or updating cell in catalog', ex) From 9f325a112bbeb2c77da16a007e5eec38f9b61339 Mon Sep 17 00:00:00 2001 From: AndyBRoswell Date: Mon, 17 Jun 2024 23:02:21 +0200 Subject: [PATCH 039/149] [bugfix] CellsHandler --- vreapis/common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vreapis/common.py b/vreapis/common.py index 5f4ff5bb..79a3bbf1 100644 --- a/vreapis/common.py +++ b/vreapis/common.py @@ -7,7 +7,7 @@ initial_retry_delay: int | float = 0.1 max_retry_delay: int | float = 5 -default_varchar_length: int = 255 +default_varchar_length: int = 4000 project_root: str = os.path.dirname(os.path.abspath(__file__)) From 1a29ad659122aa417efe5f8ec33e1bb52bdeb015 Mon Sep 17 00:00:00 2001 From: AndyBRoswell Date: Tue, 18 Jun 2024 11:45:16 +0200 Subject: [PATCH 040/149] [bugfix] CellsHandlerTestCase --- vreapis/containerizer/tests.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/vreapis/containerizer/tests.py b/vreapis/containerizer/tests.py index 08cf807b..b23f55fe 100644 --- a/vreapis/containerizer/tests.py +++ b/vreapis/containerizer/tests.py @@ -332,6 +332,13 @@ def wait_for_job(self, sleep(5) def test(self): + test_dir: str = '/tmp/data' + if not os.path.exists(test_dir): + os.makedirs(test_dir) + test_file: str = f'{test_dir}/hello.txt' + if not os.path.exists(test_file): + with open(test_file, 'w'): + pass cells_json_path = os.path.join(base_path, 'cells') cells_files = os.listdir(cells_json_path) test_cells = [] From 8a7996bb7285e8fdbc253aea920f55894ee26ddd Mon Sep 17 00:00:00 2001 From: AndyBRoswell Date: Tue, 18 Jun 2024 11:53:24 +0200 Subject: [PATCH 041/149] removed obsolete catalog.py --- vreapis/containerizer/tests.py | 6 +++--- vreapis/containerizer/views.py | 13 +++---------- 2 files changed, 6 insertions(+), 13 deletions(-) diff --git a/vreapis/containerizer/tests.py b/vreapis/containerizer/tests.py index b23f55fe..afa785ca 100644 --- a/vreapis/containerizer/tests.py +++ b/vreapis/containerizer/tests.py @@ -20,7 +20,7 @@ from tornado.gen import sleep import common -from db.catalog import Catalog +from containerizer.views import CellsHandler from services.extractor.pyextractor import PyExtractor from services.extractor.rextractor import RExtractor from services.converter import ConverterReactFlowChart @@ -213,7 +213,7 @@ def delete_text(self, file_path, text_to_delete): file.writelines(updated_lines) def wait_for_github_api_resources(self): - github = Github(Catalog.get_repositories()[0]['token']) + github = Github(CellsHandler.get_repositories()[0]['token']) rate_limit = github.get_rate_limit() while rate_limit.core.remaining <= 0: reset = rate_limit.core.reset @@ -392,7 +392,7 @@ def test(self): result = subprocess.run(shlex.split(command), capture_output=True, text=True) self.assertEqual(0, result.returncode, result.stderr) - cat_repositories = Catalog.get_repositories() + cat_repositories = CellsHandler.get_repositories() repo = cat_repositories[0] repo_token = repo['token'] owner, repository_name = repo['url'].removeprefix('https://github.com/').split('/') diff --git a/vreapis/containerizer/views.py b/vreapis/containerizer/views.py index 1d3761b9..dca48771 100644 --- a/vreapis/containerizer/views.py +++ b/vreapis/containerizer/views.py @@ -39,7 +39,6 @@ from services.converter import ConverterReactFlowChart import utils.cors from auth.simple import StaticTokenAuthentication -from db.catalog import Catalog from catalog.models import Cell import common @@ -65,7 +64,7 @@ def get_base_images(request): return Response(dat, headers=utils.cors.get_CORS_headers(request)) -class ExtractorHandler(APIView, Catalog): +class ExtractorHandler(APIView): authentication_classes = [StaticTokenAuthentication] permission_classes = [IsAuthenticated] @@ -407,7 +406,8 @@ def get_registry_url(cls, registry_url, github_url): def get_registry_credentials(self) -> list[dict[str, str]]: return [Repository(CellsHandler.registry_url, CellsHandler.registry_url, None).__dict__] - def get_repositories(self) -> list[dict[str, str]]: + @classmethod + def get_repositories(cls) -> list[dict[str, str]]: return [Repository(CellsHandler.repo_name, CellsHandler.github_url, CellsHandler.github_token).__dict__] def create(self, request: Request, *args, **kwargs): @@ -428,11 +428,6 @@ def create(self, request: Request, *args, **kwargs): if not hasattr(current_cell, 'base_image'): return return_error(f'{current_cell.task_name} has not selected base image') try: - # doc_cell = Catalog.get_cell_from_og_node_id(current_cell.node_id) - # if doc_cell: - # Catalog.update_cell(current_cell) - # else: - # Catalog.add_cell(current_cell) serializer: CellSerializer = self.get_serializer(data=request.data) serializer.is_valid(raise_exception=True) instance, created = Cell.objects.update_or_create(node_id=serializer.validated_data['node_id'], defaults=serializer.validated_data) @@ -513,8 +508,6 @@ def create(self, request: Request, *args, **kwargs): if resp.status_code != 201 and resp.status_code != 200 and resp.status_code != 204: return return_error(resp.text) current_cell.set_image_version(image_version) - # Catalog.delete_cell_from_task_name(current_cell.task_name) - # Catalog.add_cell(current_cell) Cell.objects.filter(task_name=current_cell.task_name).delete() serializer = self.get_serializer(data=current_cell) serializer.is_valid(raise_exception=True) From 66d38ccd3192a6b401b567f5db002fbccdf36f5f Mon Sep 17 00:00:00 2001 From: AndyBRoswell Date: Tue, 18 Jun 2024 11:55:44 +0200 Subject: [PATCH 042/149] removed obsolete files: catalog.py, cells.py --- vreapis/db/catalog.py | 224 ------------------------------------------ vreapis/db/cell.py | 147 --------------------------- 2 files changed, 371 deletions(-) delete mode 100644 vreapis/db/catalog.py delete mode 100644 vreapis/db/cell.py diff --git a/vreapis/db/catalog.py b/vreapis/db/catalog.py deleted file mode 100644 index 19542863..00000000 --- a/vreapis/db/catalog.py +++ /dev/null @@ -1,224 +0,0 @@ -import logging -import os -import re -from pathlib import Path - -from tinydb import TinyDB, where - -from .cell import Cell -from .repository import Repository -from .sdia_credentials import SDIACredentials - -logger = logging.getLogger(__name__) -logger.setLevel(logging.DEBUG) - - -class Catalog: - naa_vre_path = os.path.join(str(Path.home()), 'NaaVRE') - - if not os.path.exists(naa_vre_path): - os.mkdir(naa_vre_path) - - db_path = os.path.join(naa_vre_path, 'NaaVRE_db.json') - db = TinyDB(db_path) - - cells = db.table('cells') - workflows = db.table('workflows') - repositories = db.table('repositories') - gh_credentials = db.table('gh_credentials') - registry_credentials = db.table('registry_credentials') - search_entry = db.table('search_entries') - - @classmethod - def add_search_entry(cls, query: dict): - cls.search_entry.insert(query) - - @classmethod - def update_cell(cls, cell: Cell): - cls.cells.update(cell.__dict__, where('node_id') == cell.node_id) - - @classmethod - def get_search_entries(cls): - return cls.search_entry.all() - - @classmethod - def delete_all_search_entries(cls): - return cls.search_entry.truncate() - - @classmethod - def add_cell(cls, cell: Cell): - cls.cells.insert(cell.__dict__) - - @classmethod - def delete_cell_from_task_name(cls, task_name: str): - cell = cls.cells.search(where('task_name') == task_name) - cls.cells.remove(where('task_name') == task_name) - - @classmethod - def delete_cell_from_title(cls, title: str): - cell = cls.cells.search(where('title') == title) - cls.cells.remove(where('title') == title) - - @classmethod - def get_all_cells(cls): - return cls.cells.all() - - @classmethod - def get_registry_credentials(cls): - credentials = cls.registry_credentials.all() - return credentials - - @classmethod - def get_repository_credentials(cls): - credentials = cls.repositories.all() - return credentials - - @classmethod - def get_registry_credentials_from_name(cls, name: str): - res = cls.registry_credentials.search(where('name') == name) - if res: - return res[0] - - @classmethod - def add_registry_credentials(cls, cred: Repository): - cls.registry_credentials.insert(cred.__dict__) - - @classmethod - def add_repository_credentials(cls, cred: Repository): - cls.repositories.insert(cred.__dict__) - - @classmethod - def get_gh_credentials(cls): - credentials = cls.gh_credentials.all() - if len(credentials) > 0: - return credentials[0] - - @classmethod - def delete_all_gh_credentials(cls): - cls.gh_credentials.truncate() - - @classmethod - def delete_all_cells(cls): - cls.cells.truncate() - - @classmethod - def delete_all_repository_credentials(cls): - cls.repositories.truncate() - credentials = cls.repositories.all() - ids = [] - for credential in credentials: - ids.append(credential.doc_id) - cls.repositories.remove(doc_ids=ids) - cls.repositories.truncate() - - @classmethod - def delete_all_registry_credentials(cls): - # Looks bad but for now I could not find a way to remove all - credentials = cls.registry_credentials.all() - ids = [] - for credential in credentials: - ids.append(credential.doc_id) - cls.registry_credentials.remove(doc_ids=ids) - cls.registry_credentials.truncate() - - @classmethod - def add_gh_credentials(cls, cred: Repository): - cls.repositories.insert(cred.__dict__) - cls.gh_credentials.insert(cred.__dict__) - - @classmethod - def delete_gh_credentials(cls, url: str): - cls.gh_credentials.remove(where('url') == url) - - @classmethod - def get_credentials_from_username(cls, cred_username) -> SDIACredentials: - res = cls.sdia_credentials.search(where('username') == cred_username) - if res: - return res[0] - - @classmethod - def get_sdia_credentials(cls): - return cls.sdia_credentials.all() - - @classmethod - def get_cell_from_og_node_id(cls, og_node_id) -> Cell: - res = cls.cells.search(where('node_id') == og_node_id) - if res: - return res[0] - else: - logger.warning('Cell not found for og_node_id: ' + og_node_id) - - @classmethod - def get_repositories(cls) -> list: - res = cls.repositories.all() - return res - - @classmethod - def get_repository_from_name(cls, name: str) -> Repository: - res = cls.repositories.search(where('name') == name) - if res: - return res[0] - - @classmethod - def cast_document_to_cell(cls, cell_document): - if not cell_document: - return None - - return Cell( - title=cell_document.get('title', ''), - task_name=cell_document.get('task_name', ''), - original_source=cell_document.get('original_source', ''), - inputs=cell_document.get('inputs', []), - outputs=cell_document.get('outputs', []), - params=cell_document.get('params', []), - confs=cell_document.get('confs', {}), - dependencies=cell_document.get('dependencies', []), - container_source=cell_document.get('container_source', ''), - chart_obj=cell_document.get('chart_obj', {}), - node_id=cell_document.get('node_id', ''), - kernel=cell_document.get('kernel', ''), - notebook_dict=cell_document.get('notebook_dict', {}) - ) - - @classmethod - def get_registry_url(cls, registry_url, github_url): - """ Convert registry URL - - https://hub.docker.com/u/my_username/ -> docker.io/my_username - oci://ghcr.io/my_username/my_repo/ -> ghcr.io/my_username/my_repo - oci://my_domain/my/custom/path/ -> my_domain/my/custom/path - None -> ghcr.io url, derived from github_url - - Resulting urls can be converted to pullable, e.g.: - - docker pull {url}/{image_name}:{tag} - - where image_name doesn't contain any path information (e.g. my-cell-name) - - """ - if registry_url: - m = re.match(r'^https://hub\.docker\.com/u/(\w+)/?$', registry_url) - if m: - return f"docker.io/{m.group(1)}" - m = re.match(r'^oci://([\w\./-]+?)/?$', registry_url) - if m: - return m.group(1) - raise ValueError(f"Could not parse registry url: {registry_url}") - else: - m = re.match(r'^https://github.com/([\w-]+/[\w-]+)(?:\.git)?', github_url) - if m: - return f"ghcr.io/{m.group(1).lower()}" - - -github_url: str = os.getenv('CELL_GITHUB') -github_token: str = os.getenv('CELL_GITHUB_TOKEN') -registry_url: str = os.getenv('REGISTRY_URL') -# force: bool = True if os.getenv('FORCED_CREDENTIALS_REPLACEMENT', 'false').lower() == 'true' else 'false' -registry_url = Catalog.get_registry_url(registry_url, github_url) - -input_repository_credentials = Repository(github_url.split('https://github.com/')[1], github_url, github_token) -Catalog.add_gh_credentials(input_repository_credentials) -Catalog.add_repository_credentials(input_repository_credentials) - -input_registry_credentials = Repository(registry_url, registry_url, None) -Catalog.add_registry_credentials(input_registry_credentials) diff --git a/vreapis/db/cell.py b/vreapis/db/cell.py deleted file mode 100644 index 378fb7b5..00000000 --- a/vreapis/db/cell.py +++ /dev/null @@ -1,147 +0,0 @@ -import json -import re - -from slugify import slugify - - -class Cell(): - title: str - task_name: str - original_source: str - base_image: dict - inputs: list - outputs: list - params: list - param_values: dict - confs: dict - dependencies: list - chart_obj: dict - node_id: str - container_source: str - global_conf: dict - kernel: str - notebook_json: dict - image_version: str - - def __init__(self, title, task_name, original_source, inputs, outputs, params, confs, dependencies, container_source, chart_obj=None, node_id='', kernel='', notebook_dict=None, image_version=None) -> None: - self.title = slugify(title.strip()) - self.task_name = slugify(task_name) - self.original_source = original_source - self.types = dict() - self.add_inputs(inputs) - self.add_outputs(outputs) - self.add_params(params) - self.add_param_values(params) - self.confs = confs - self.all_inputs = list(inputs) + list(params) - self.dependencies = list(sorted(dependencies, key=lambda x: x['name'])) - self.chart_obj = chart_obj - self.node_id = node_id - self.container_source = container_source - self.kernel = kernel - self.notebook_dict = notebook_dict - - def _extract_types(self, vars_dict): - """ Extract types to self.types and return list of var names - - :param vars_dict: {'var1': {'name: 'var1', 'type': 'str'}, 'var2': ...} - :return: ['var1', 'var2', ...] - """ - names = [] - for var_props in vars_dict.values(): - var_type = var_props['type'] - var_name = var_props['name'] - self.types[var_name] = var_type - names.append(var_name) - return names - - def add_inputs(self, inputs): - if isinstance(inputs, dict): - inputs = self._extract_types(inputs) - self.inputs = inputs - - def add_outputs(self, outputs): - if isinstance(outputs, dict): - outputs = self._extract_types(outputs) - self.outputs = outputs - - def add_params(self, params): - if isinstance(params, dict): - params = self._extract_types(params) - self.params = params - - def set_image_version(self, image_version): - self.image_version = image_version - - def add_param_values(self, params): - self.param_values = {} - if isinstance(params, dict): - for param_props in params.values(): - if 'value' in param_props: - self.param_values[param_props['name']] = param_props['value'] - - def concatenate_all_inputs(self): - self.all_inputs = list(self.inputs) + list(self.params) - - def clean_code(self): - indices_to_remove = [] - lines = self.original_source.splitlines() - self.original_source = "" - - for line_i in range(0, len(lines)): - line = lines[line_i] - # Do not remove line that startswith param_ if not in the self.params - if line.startswith('param_'): - # clean param name - pattern = r"\b(param_\w+)\b" - param_name = re.findall(pattern, line)[0] - if param_name in self.params: - indices_to_remove.append(line_i) - if re.match('^\s*(#|import|from)', line): - indices_to_remove.append(line_i) - - for ir in sorted(indices_to_remove, reverse=True): - lines.pop(ir) - - self.original_source = "\n".join(lines) - - def clean_task_name(self): - self.task_name = slugify(self.task_name) - - def clean_title(self): - self.title = slugify(self.title) - - def integrate_configuration(self): - lines = self.original_source.splitlines() - self.original_source = "" - for idx, conf in enumerate(self.generate_configuration()): - lines.insert(idx, conf) - self.original_source = "\n".join(lines) - - def generate_dependencies(self): - resolves = [] - for d in self.dependencies: - resolve_to = "import %s" % d['name'] - if d['module']: - resolve_to = "from %s %s" % (d['module'], resolve_to) - if d['asname']: - resolve_to += " as %s" % d['asname'] - resolves.append(resolve_to) - return resolves - - def generate_configuration(self): - resolves = [] - for c in self.confs: - resolves.append(self.confs[c]) - return resolves - - def generate_configuration_dict(self): - resolves = [] - for c in self.confs: - assignment = self.confs[c].split('=')[1].replace('=', '').strip() - conf = {c: assignment} - resolves.append(conf) - return resolves - - def toJSON(self): - return json.dumps(self, default=lambda o: o.__dict__, sort_keys=True, indent=4) From 41734a9f135c6914c0ac1400c3312e8900b823bf Mon Sep 17 00:00:00 2001 From: AndyBRoswell Date: Wed, 19 Jun 2024 14:00:55 +0200 Subject: [PATCH 043/149] [bugfix] Cell.toJSON --- vreapis/catalog/models.py | 9 +++++++-- vreapis/containerizer/urls.py | 7 +++++++ vreapis/requirements.txt | 1 - 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/vreapis/catalog/models.py b/vreapis/catalog/models.py index ab58066a..74a3b8aa 100644 --- a/vreapis/catalog/models.py +++ b/vreapis/catalog/models.py @@ -129,5 +129,10 @@ def generate_configuration_dict(self): resolves.append(conf) return resolves - def toJSON(self): - return json.dumps(self, default=lambda o: o.__dict__, sort_keys=True, indent=4) + def toJSON(self) -> str: + # return json.dumps(self, default=lambda o: o.__dict__, sort_keys=True, indent=4) + array_fields = ['inputs', 'outputs', 'params', ] + result = {k: v for k, v in self.__dict__.items() if k not in array_fields} + for field in array_fields: + result[field] = list(getattr(self, field)) + return json.dumps(result, default=lambda o: o.__dict__, sort_keys=True, indent=4) diff --git a/vreapis/containerizer/urls.py b/vreapis/containerizer/urls.py index 43bb6528..bd1cba8c 100644 --- a/vreapis/containerizer/urls.py +++ b/vreapis/containerizer/urls.py @@ -4,7 +4,14 @@ urlpatterns = [ path('baseimagetags', views.get_base_images), path('extract', views.ExtractorHandler.as_view()), + path('cell', views.CellsHandler.as_view({ + 'get': 'list', + 'post': 'create', + })), path('addcell', views.CellsHandler.as_view({ 'post': 'create', })), + path('catalog/cells/all', views.CellsHandler.as_view({ + 'get': 'list', + })), ] diff --git a/vreapis/requirements.txt b/vreapis/requirements.txt index d0394b7d..8979a7a8 100644 --- a/vreapis/requirements.txt +++ b/vreapis/requirements.txt @@ -19,7 +19,6 @@ colorhash rpy2==3.5.11 pyflakes pytype -tinydb autopep8 distro pygithub From 27bc04da940578ef332e69f5f88b0ace4f4d682f Mon Sep 17 00:00:00 2001 From: AndyBRoswell Date: Fri, 21 Jun 2024 17:06:00 +0200 Subject: [PATCH 044/149] [bugfix] CellsHandler --- vreapis/catalog/admin.py | 4 + vreapis/catalog/migrations/0001_initial.py | 37 ++++++ ...ntainer_source_alter_cell_image_version.py | 23 ++++ ...ource_alter_cell_image_version_and_more.py | 48 ++++++++ vreapis/catalog/migrations/__init__.py | 0 vreapis/catalog/models.py | 2 +- vreapis/containerizer/migrations/__init__.py | 0 vreapis/containerizer/views.py | 5 +- .../tests/emulated-frontend/containerizer.py | 6 +- .../tests/emulated-frontend/dat/addcell.json | 113 ++++++++++++++++++ 10 files changed, 233 insertions(+), 5 deletions(-) create mode 100644 vreapis/catalog/migrations/0001_initial.py create mode 100644 vreapis/catalog/migrations/0002_alter_cell_container_source_alter_cell_image_version.py create mode 100644 vreapis/catalog/migrations/0003_alter_cell_container_source_alter_cell_image_version_and_more.py create mode 100644 vreapis/catalog/migrations/__init__.py create mode 100644 vreapis/containerizer/migrations/__init__.py create mode 100644 vreapis/tests/emulated-frontend/dat/addcell.json diff --git a/vreapis/catalog/admin.py b/vreapis/catalog/admin.py index 8c38f3f3..87770453 100644 --- a/vreapis/catalog/admin.py +++ b/vreapis/catalog/admin.py @@ -1,3 +1,7 @@ from django.contrib import admin +from . import models + # Register your models here. + +admin.site.register(models.Cell) diff --git a/vreapis/catalog/migrations/0001_initial.py b/vreapis/catalog/migrations/0001_initial.py new file mode 100644 index 00000000..2dcfca3d --- /dev/null +++ b/vreapis/catalog/migrations/0001_initial.py @@ -0,0 +1,37 @@ +# Generated by Django 5.0.3 on 2024-06-17 18:52 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ] + + operations = [ + migrations.CreateModel( + name='Cell', + fields=[ + ('title', models.CharField(max_length=255)), + ('task_name', models.CharField(max_length=255)), + ('original_source', models.CharField(max_length=255)), + ('types', models.JSONField(blank=True, default=dict, null=True)), + ('base_image', models.JSONField(blank=True, null=True)), + ('inputs', models.JSONField(blank=True, default=list, null=True)), + ('outputs', models.JSONField(blank=True, default=list, null=True)), + ('params', models.JSONField(blank=True, default=list, null=True)), + ('param_values', models.JSONField(blank=True, null=True)), + ('confs', models.JSONField(blank=True, null=True)), + ('dependencies', models.JSONField(blank=True, null=True)), + ('chart_obj', models.JSONField(blank=True, null=True)), + ('node_id', models.CharField(max_length=255, primary_key=True, serialize=False)), + ('container_source', models.CharField(max_length=255)), + ('global_conf', models.JSONField(blank=True, null=True)), + ('kernel', models.CharField(max_length=255)), + ('notebook_dict', models.JSONField(blank=True, null=True)), + ('image_version', models.CharField(max_length=255)), + ], + ), + ] diff --git a/vreapis/catalog/migrations/0002_alter_cell_container_source_alter_cell_image_version.py b/vreapis/catalog/migrations/0002_alter_cell_container_source_alter_cell_image_version.py new file mode 100644 index 00000000..19f92c28 --- /dev/null +++ b/vreapis/catalog/migrations/0002_alter_cell_container_source_alter_cell_image_version.py @@ -0,0 +1,23 @@ +# Generated by Django 5.0.3 on 2024-06-17 19:03 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('catalog', '0001_initial'), + ] + + operations = [ + migrations.AlterField( + model_name='cell', + name='container_source', + field=models.CharField(blank=True, max_length=255, null=True), + ), + migrations.AlterField( + model_name='cell', + name='image_version', + field=models.CharField(blank=True, max_length=255, null=True), + ), + ] diff --git a/vreapis/catalog/migrations/0003_alter_cell_container_source_alter_cell_image_version_and_more.py b/vreapis/catalog/migrations/0003_alter_cell_container_source_alter_cell_image_version_and_more.py new file mode 100644 index 00000000..33701f10 --- /dev/null +++ b/vreapis/catalog/migrations/0003_alter_cell_container_source_alter_cell_image_version_and_more.py @@ -0,0 +1,48 @@ +# Generated by Django 5.0.3 on 2024-06-17 21:00 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('catalog', '0002_alter_cell_container_source_alter_cell_image_version'), + ] + + operations = [ + migrations.AlterField( + model_name='cell', + name='container_source', + field=models.CharField(blank=True, max_length=4000, null=True), + ), + migrations.AlterField( + model_name='cell', + name='image_version', + field=models.CharField(blank=True, max_length=4000, null=True), + ), + migrations.AlterField( + model_name='cell', + name='kernel', + field=models.CharField(max_length=4000), + ), + migrations.AlterField( + model_name='cell', + name='node_id', + field=models.CharField(max_length=4000, primary_key=True, serialize=False), + ), + migrations.AlterField( + model_name='cell', + name='original_source', + field=models.CharField(max_length=4000), + ), + migrations.AlterField( + model_name='cell', + name='task_name', + field=models.CharField(max_length=4000), + ), + migrations.AlterField( + model_name='cell', + name='title', + field=models.CharField(max_length=4000), + ), + ] diff --git a/vreapis/catalog/migrations/__init__.py b/vreapis/catalog/migrations/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/vreapis/catalog/models.py b/vreapis/catalog/models.py index 74a3b8aa..5fa6b22c 100644 --- a/vreapis/catalog/models.py +++ b/vreapis/catalog/models.py @@ -24,7 +24,7 @@ class Cell(models.Model): container_source = models.CharField(max_length=common.default_varchar_length, blank=True, null=True) global_conf = models.JSONField(blank=True, null=True) kernel = models.CharField(max_length=common.default_varchar_length) - notebook_dict = models.JSONField(blank=True, null=True) + notebook_dict = models.JSONField(blank=True, null=True, default=dict) image_version = models.CharField(max_length=common.default_varchar_length, blank=True, null=True) def _extract_types(self, vars_dict): diff --git a/vreapis/containerizer/migrations/__init__.py b/vreapis/containerizer/migrations/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/vreapis/containerizer/views.py b/vreapis/containerizer/views.py index dca48771..27d95867 100644 --- a/vreapis/containerizer/views.py +++ b/vreapis/containerizer/views.py @@ -412,7 +412,10 @@ def get_repositories(cls) -> list[dict[str, str]]: def create(self, request: Request, *args, **kwargs): try: - current_cell = Cell(**request.data) + Cell_field_names = [f.name for f in Cell._meta.get_fields()] + filtered_incoming_body = {k: request.data[k] for k in Cell_field_names if k in request.data} + current_cell = Cell(**filtered_incoming_body) + # current_cell = Cell(**request.data) current_cell.clean_code() current_cell.clean_title() current_cell.clean_task_name() diff --git a/vreapis/tests/emulated-frontend/containerizer.py b/vreapis/tests/emulated-frontend/containerizer.py index 83977320..6bc34e35 100644 --- a/vreapis/tests/emulated-frontend/containerizer.py +++ b/vreapis/tests/emulated-frontend/containerizer.py @@ -4,7 +4,7 @@ import sys script_path: str = os.path.dirname(os.path.realpath(__file__)) -endpoint: str = f"{os.getenv('API_ENDPOINT')}/api/containerizer/extract" +endpoint: str = f"{os.getenv('API_ENDPOINT')}/api/containerizer" headers: dict = { "Content-Type": "application/json", "Authorization": f"Token {os.getenv('NAAVRE_API_TOKEN')}", @@ -14,5 +14,5 @@ for i in range(1, len(sys.argv)): with open(f'{script_path}/dat/{sys.argv[i]}.json') as f: body: dict[str, any] = json.load(f) - response = session.post(endpoint, json.dumps(body), headers=headers, verify=False) - print(response.text) \ No newline at end of file + response = session.post(f'{endpoint}/{sys.argv[i]}', json.dumps(body), headers=headers, verify=False) + print(response.text) diff --git a/vreapis/tests/emulated-frontend/dat/addcell.json b/vreapis/tests/emulated-frontend/dat/addcell.json new file mode 100644 index 00000000..4aaf0c4b --- /dev/null +++ b/vreapis/tests/emulated-frontend/dat/addcell.json @@ -0,0 +1,113 @@ +{ + "_state": {}, + "base_image": { + "build": "ghcr.io/qcdis/naavre/naavre-cell-build-python:v0.14", + "runtime": "ghcr.io/qcdis/naavre/naavre-cell-runtime-python:v0.14" + }, + "chart_obj": { + "hovered": {}, + "links": {}, + "nodes": { + "603faa7": { + "id": "603faa7", + "ports": { + "names": { + "id": "names", + "properties": { + "color": "#ac5397" + }, + "type": "left", + "position": { + "x": 22.5, + "y": 74.5 + } + } + }, + "position": { + "x": 35, + "y": 15 + }, + "properties": { + "inputs": [ + "names" + ], + "og_node_id": "603faa7", + "outputs": [], + "params": [], + "title": "print-names0-user", + "vars": [ + { + "color": "#ac5397", + "direction": "input", + "name": "names", + "type": "datatype" + } + ] + }, + "type": "input-output", + "size": { + "width": 250, + "height": 150 + } + } + }, + "offset": { + "x": 0, + "y": 0 + }, + "scale": 1, + "selected": {} + }, + "confs": {}, + "container_source": "", + "dependencies": [], + "global_conf": null, + "image_version": null, + "inputs": [ + "names" + ], + "kernel": "ipython", + "node_id": "603faa7", + "notebook_dict": { + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "848c006a-bf2b-439d-8fec-ed16c5eae66d", + "metadata": {}, + "outputs": [], + "source": "# print names0\nfor name in names:\n print(f\"Hello, {name}!\")" + } + ], + "metadata": { + "kernelspec": { + "display_name": "python3", + "language": "python3", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.9" + } + }, + "nbformat": 4, + "nbformat_minor": 5 + }, + "original_source": "# print names0\nfor name in names:\n print(f\"Hello, {name}!\")", + "outputs": [], + "param_values": {}, + "params": [], + "task_name": "print-names0-user", + "title": "print-names0-user", + "types": { + "names": "list" + } +} \ No newline at end of file From 3d9fb64ffbc32c089fe2f0b00fa9d7a8e1475043 Mon Sep 17 00:00:00 2001 From: AndyBRoswell Date: Fri, 21 Jun 2024 20:02:09 +0200 Subject: [PATCH 045/149] [bugfix] CellsHandler --- vreapis/containerizer/views.py | 4 ++-- vreapis/tests/emulated-frontend/containerizer.py | 4 ++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/vreapis/containerizer/views.py b/vreapis/containerizer/views.py index 27d95867..ce4cd33d 100644 --- a/vreapis/containerizer/views.py +++ b/vreapis/containerizer/views.py @@ -441,7 +441,7 @@ def create(self, request: Request, *args, **kwargs): self.write_cell_to_file(current_cell) if not os.path.exists(self.cells_path): - os.mkdir(self.cells_path) + os.makedirs(self.cells_path, exist_ok=True) cell_path = os.path.join(self.cells_path, current_cell.task_name) @@ -451,7 +451,7 @@ def create(self, request: Request, *args, **kwargs): if os.path.isfile(path): os.remove(path) else: - os.mkdir(cell_path) + os.makedirs(cell_path, exist_ok=True) registry_credentials = self.get_registry_credentials() if not registry_credentials or len(registry_credentials) <= 0: diff --git a/vreapis/tests/emulated-frontend/containerizer.py b/vreapis/tests/emulated-frontend/containerizer.py index 6bc34e35..deccc380 100644 --- a/vreapis/tests/emulated-frontend/containerizer.py +++ b/vreapis/tests/emulated-frontend/containerizer.py @@ -2,6 +2,7 @@ import json import os import sys +import time script_path: str = os.path.dirname(os.path.realpath(__file__)) endpoint: str = f"{os.getenv('API_ENDPOINT')}/api/containerizer" @@ -14,5 +15,8 @@ for i in range(1, len(sys.argv)): with open(f'{script_path}/dat/{sys.argv[i]}.json') as f: body: dict[str, any] = json.load(f) + match sys.argv[1]: + case 'addcell': + body['node_id'] = str(hex(time.time_ns())[len('0x'):]) response = session.post(f'{endpoint}/{sys.argv[i]}', json.dumps(body), headers=headers, verify=False) print(response.text) From c48f0edf83a6e04a83986c80aa75d14108c5a6a4 Mon Sep 17 00:00:00 2001 From: AndyBRoswell Date: Fri, 21 Jun 2024 22:40:52 +0200 Subject: [PATCH 046/149] [bugfix] urlconf emulated frontend test ExtractorHandler: use time.time_ns as node_id to avoid conflicts --- vreapis/containerizer/urls.py | 3 --- vreapis/containerizer/views.py | 7 ++++--- vreapis/tests/emulated-frontend/containerizer.py | 2 +- vreapis/vreapis/urls.py | 5 +++++ 4 files changed, 10 insertions(+), 7 deletions(-) diff --git a/vreapis/containerizer/urls.py b/vreapis/containerizer/urls.py index bd1cba8c..bd4d66d4 100644 --- a/vreapis/containerizer/urls.py +++ b/vreapis/containerizer/urls.py @@ -11,7 +11,4 @@ path('addcell', views.CellsHandler.as_view({ 'post': 'create', })), - path('catalog/cells/all', views.CellsHandler.as_view({ - 'get': 'list', - })), ] diff --git a/vreapis/containerizer/views.py b/vreapis/containerizer/views.py index ce4cd33d..d5bd464a 100644 --- a/vreapis/containerizer/views.py +++ b/vreapis/containerizer/views.py @@ -126,9 +126,10 @@ def post(self, request: Request): title += '-' + slugify(os.environ['JUPYTERHUB_USER']) # If any of these change, we create a new cell in the catalog. This matches the cell properties saved in workflows. - cell_identity_dict = {'title': title, 'params': extractor.params, 'inputs': extractor.ins, 'outputs': extractor.outs, } - cell_identity_str = json.dumps(cell_identity_dict, sort_keys=True) - node_id = hashlib.sha1(cell_identity_str.encode()).hexdigest()[:7] + # cell_identity_dict = {'title': title, 'params': extractor.params, 'inputs': extractor.ins, 'outputs': extractor.outs, } + # cell_identity_str = json.dumps(cell_identity_dict, sort_keys=True) + # node_id = hashlib.sha1(cell_identity_str.encode()).hexdigest()[:7] + node_id = str(time.time_ns())[len('0x'):] cell = Cell( node_id=node_id, diff --git a/vreapis/tests/emulated-frontend/containerizer.py b/vreapis/tests/emulated-frontend/containerizer.py index deccc380..fc9bd1d3 100644 --- a/vreapis/tests/emulated-frontend/containerizer.py +++ b/vreapis/tests/emulated-frontend/containerizer.py @@ -15,7 +15,7 @@ for i in range(1, len(sys.argv)): with open(f'{script_path}/dat/{sys.argv[i]}.json') as f: body: dict[str, any] = json.load(f) - match sys.argv[1]: + match sys.argv[i]: case 'addcell': body['node_id'] = str(hex(time.time_ns())[len('0x'):]) response = session.post(f'{endpoint}/{sys.argv[i]}', json.dumps(body), headers=headers, verify=False) diff --git a/vreapis/vreapis/urls.py b/vreapis/vreapis/urls.py index d01cd096..2639f1df 100644 --- a/vreapis/vreapis/urls.py +++ b/vreapis/vreapis/urls.py @@ -16,6 +16,8 @@ from django.contrib import admin from django.urls import path, include from rest_framework import routers + +import containerizer.views from virtual_labs.views import VirtualLabViewSet, VirtualLabInstanceViewSet from workflows.views import WorkflowViewSet from data_products.views import DataProductsViewSet, GeoDataProductsViewSet @@ -38,6 +40,9 @@ path('admin/', admin.site.urls), path('api/', include(router.urls)), path('api/containerizer/', include('containerizer.urls')), + path('api/catalog/cells/all', containerizer.views.CellsHandler.as_view({ + 'get': 'list', + })), ] if BASE_PATH: From 96ee0565dc57547b13bc2125c729808b365b86de Mon Sep 17 00:00:00 2001 From: AndyBRoswell Date: Sun, 23 Jun 2024 00:44:18 +0200 Subject: [PATCH 047/149] refined emulated frontend test --- .../tests/emulated-frontend/containerizer.py | 41 ++++++++++++++----- 1 file changed, 31 insertions(+), 10 deletions(-) diff --git a/vreapis/tests/emulated-frontend/containerizer.py b/vreapis/tests/emulated-frontend/containerizer.py index fc9bd1d3..5544414c 100644 --- a/vreapis/tests/emulated-frontend/containerizer.py +++ b/vreapis/tests/emulated-frontend/containerizer.py @@ -1,22 +1,43 @@ import requests import json import os -import sys import time +import argparse +import re + +arg_parser = argparse.ArgumentParser() +arg_parser.add_argument('in_def', nargs='+') +arg_parser.add_argument('--in-no', action='append', nargs='+') +args = arg_parser.parse_args() script_path: str = os.path.dirname(os.path.realpath(__file__)) -endpoint: str = f"{os.getenv('API_ENDPOINT')}/api/containerizer" +API_ENDPOINT: str = f"{os.getenv('API_ENDPOINT')}/api/containerizer" headers: dict = { "Content-Type": "application/json", "Authorization": f"Token {os.getenv('NAAVRE_API_TOKEN')}", } session = requests.Session() -for i in range(1, len(sys.argv)): - with open(f'{script_path}/dat/{sys.argv[i]}.json') as f: - body: dict[str, any] = json.load(f) - match sys.argv[i]: - case 'addcell': - body['node_id'] = str(hex(time.time_ns())[len('0x'):]) - response = session.post(f'{endpoint}/{sys.argv[i]}', json.dumps(body), headers=headers, verify=False) - print(response.text) + +def test(endpoint: str, files: list[str]): + for file in files: + with open(f'{script_path}/dat/{file}') as f: + body: dict[str, any] = json.load(f) + match endpoint: + case 'addcell': + body['node_id'] = str(hex(time.time_ns())[len('0x'):]) + response = session.post(f'{API_ENDPOINT}/{endpoint}', json.dumps(body), headers=headers, verify=False) + print(response.text) + + +for endpoint in args.in_def: + file_pattern = re.compile(fr'{endpoint}(\..+)?.json') + files: list[str] = os.listdir(f'{script_path}/dat') + request_body_files: list[str] = [file for file in files if file_pattern.match(file)] + test(endpoint, request_body_files) + +if args.in_no is not None: + for test_info in args.in_no: + endpoint: str = test_info[0] + files: list[str] = [f'{endpoint}.{no}.json' for no in test_info[1:]] + test(endpoint, files) From 17e9abea01d7a8dc742e6ffeaacefd620449201b Mon Sep 17 00:00:00 2001 From: AndyBRoswell Date: Sun, 23 Jun 2024 00:54:59 +0200 Subject: [PATCH 048/149] refined emulated frontend test --- .../tests/emulated-frontend/containerizer.py | 8 +- .../emulated-frontend/dat/addcell.1.json | 113 ++++++++++++++++++ .../dat/{addcell.json => addcell.2.json} | 10 +- 3 files changed, 122 insertions(+), 9 deletions(-) create mode 100644 vreapis/tests/emulated-frontend/dat/addcell.1.json rename vreapis/tests/emulated-frontend/dat/{addcell.json => addcell.2.json} (90%) diff --git a/vreapis/tests/emulated-frontend/containerizer.py b/vreapis/tests/emulated-frontend/containerizer.py index 5544414c..8a9fe3c3 100644 --- a/vreapis/tests/emulated-frontend/containerizer.py +++ b/vreapis/tests/emulated-frontend/containerizer.py @@ -7,7 +7,7 @@ arg_parser = argparse.ArgumentParser() arg_parser.add_argument('in_def', nargs='+') -arg_parser.add_argument('--in-no', action='append', nargs='+') +arg_parser.add_argument('--in-id', action='append', nargs='+') args = arg_parser.parse_args() script_path: str = os.path.dirname(os.path.realpath(__file__)) @@ -36,8 +36,8 @@ def test(endpoint: str, files: list[str]): request_body_files: list[str] = [file for file in files if file_pattern.match(file)] test(endpoint, request_body_files) -if args.in_no is not None: - for test_info in args.in_no: +if args.in_id is not None: + for test_info in args.in_id: endpoint: str = test_info[0] - files: list[str] = [f'{endpoint}.{no}.json' for no in test_info[1:]] + files: list[str] = [f'{endpoint}.{id}.json' for id in test_info[1:]] test(endpoint, files) diff --git a/vreapis/tests/emulated-frontend/dat/addcell.1.json b/vreapis/tests/emulated-frontend/dat/addcell.1.json new file mode 100644 index 00000000..e8b2e835 --- /dev/null +++ b/vreapis/tests/emulated-frontend/dat/addcell.1.json @@ -0,0 +1,113 @@ +{ + "_state": {}, + "base_image": { + "build": "ghcr.io/qcdis/naavre/naavre-cell-build-python:v0.14", + "runtime": "ghcr.io/qcdis/naavre/naavre-cell-runtime-python:v0.14" + }, + "chart_obj": { + "hovered": {}, + "links": {}, + "nodes": { + "19096734787431788": { + "id": "19096734787431788", + "ports": { + "names": { + "id": "names", + "properties": { + "color": "#ac5397" + }, + "type": "right", + "position": { + "x": 225.5, + "y": 74.5 + } + } + }, + "position": { + "x": 35, + "y": 15 + }, + "properties": { + "inputs": [], + "og_node_id": "19096734787431788", + "outputs": [ + "names" + ], + "params": [], + "title": "input-names-user", + "vars": [ + { + "color": "#ac5397", + "direction": "output", + "name": "names", + "type": "datatype" + } + ] + }, + "type": "input-output", + "size": { + "width": 250, + "height": 150 + } + } + }, + "offset": { + "x": 0, + "y": 0 + }, + "scale": 1, + "selected": {} + }, + "confs": {}, + "container_source": "", + "dependencies": [], + "global_conf": null, + "image_version": null, + "inputs": [], + "kernel": "ipython", + "node_id": "19096734787431788", + "notebook_dict": { + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "87f87efd-50e8-41cd-b4f6-2ba14d442cfb", + "metadata": {}, + "outputs": [], + "source": "# input names\nnames = [\"Alice\", \"Bob\"]" + } + ], + "metadata": { + "kernelspec": { + "display_name": "python3", + "language": "python3", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.9" + } + }, + "nbformat": 4, + "nbformat_minor": 5 + }, + "original_source": "# input names\nnames = [\"Alice\", \"Bob\"]", + "outputs": [ + "names" + ], + "param_values": {}, + "params": [], + "task_name": "input-names-user", + "title": "input-names-user", + "types": { + "names": "list" + } +} \ No newline at end of file diff --git a/vreapis/tests/emulated-frontend/dat/addcell.json b/vreapis/tests/emulated-frontend/dat/addcell.2.json similarity index 90% rename from vreapis/tests/emulated-frontend/dat/addcell.json rename to vreapis/tests/emulated-frontend/dat/addcell.2.json index 4aaf0c4b..12a23701 100644 --- a/vreapis/tests/emulated-frontend/dat/addcell.json +++ b/vreapis/tests/emulated-frontend/dat/addcell.2.json @@ -34,7 +34,7 @@ "og_node_id": "603faa7", "outputs": [], "params": [], - "title": "print-names0-user", + "title": "print-names-user", "vars": [ { "color": "#ac5397", @@ -76,7 +76,7 @@ "id": "848c006a-bf2b-439d-8fec-ed16c5eae66d", "metadata": {}, "outputs": [], - "source": "# print names0\nfor name in names:\n print(f\"Hello, {name}!\")" + "source": "# print names\nfor name in names:\n print(f\"Hello, {name}!\")" } ], "metadata": { @@ -101,12 +101,12 @@ "nbformat": 4, "nbformat_minor": 5 }, - "original_source": "# print names0\nfor name in names:\n print(f\"Hello, {name}!\")", + "original_source": "# print names\nfor name in names:\n print(f\"Hello, {name}!\")", "outputs": [], "param_values": {}, "params": [], - "task_name": "print-names0-user", - "title": "print-names0-user", + "task_name": "print-names-user", + "title": "print-names-user", "types": { "names": "list" } From eac72b8b30fb304ed9e77f72b0fbe089c8d98468 Mon Sep 17 00:00:00 2001 From: AndyBRoswell Date: Sun, 23 Jun 2024 01:31:05 +0200 Subject: [PATCH 049/149] refined emulated frontend test --- vreapis/tests/emulated-frontend/containerizer.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/vreapis/tests/emulated-frontend/containerizer.py b/vreapis/tests/emulated-frontend/containerizer.py index 8a9fe3c3..f672b60c 100644 --- a/vreapis/tests/emulated-frontend/containerizer.py +++ b/vreapis/tests/emulated-frontend/containerizer.py @@ -5,9 +5,18 @@ import argparse import re -arg_parser = argparse.ArgumentParser() -arg_parser.add_argument('in_def', nargs='+') -arg_parser.add_argument('--in-id', action='append', nargs='+') +arg_parser = argparse.ArgumentParser(description=''' +These tests emulate a frontend to test the endpoints of VRE PaaS API. +At this moment, folder `dat` contains all the test input data. Each file represents an entire request body. +Some files contain only 1 segment of extension `.json`. This indicates that the file is the only test case. +Some files contain 2 segments of extension (ex. `.1.json`). The substring between 2 dots is the ID of the test case (`1` in this example). +'''.strip()) +arg_parser.add_argument('in_def', nargs='+', help=''' +Default: Test the designated endpoint using all the existing request bodies. +'''.strip()) +arg_parser.add_argument('--in-id', action='append', nargs='+', help=''' +Specify test cases by ID: Test the designated endpoint using the request bodies with the specified IDs. +'''.strip()) args = arg_parser.parse_args() script_path: str = os.path.dirname(os.path.realpath(__file__)) From b7550c0d0bf0cc71a8f467ac8c17f5c32d383a14 Mon Sep 17 00:00:00 2001 From: Andy Bayer Roswell Date: Tue, 23 Jul 2024 23:56:47 +0200 Subject: [PATCH 050/149] adding rmd support --- vreapis/containerizer/views.py | 9 ++++++++- vreapis/requirements.txt | 1 + 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/vreapis/containerizer/views.py b/vreapis/containerizer/views.py index d5bd464a..22efce68 100644 --- a/vreapis/containerizer/views.py +++ b/vreapis/containerizer/views.py @@ -28,6 +28,7 @@ import jsonschema from slugify import slugify from github import Github, UnknownObjectException +import jupytext from catalog.serializers import CellSerializer from containerizer.RContainerizer import RContainerizer @@ -88,9 +89,15 @@ def get(self, request: Request): def post(self, request: Request): payload = request.data common.logger.debug('ExtractorHandler. payload: ' + json.dumps(payload, indent=4)) + if 'rmarkdown' in payload: + Rmd = jupytext.reads(payload['rmarkdown'], fmt='Rmd') + ipynb = jupytext.writes(Rmd, fmt='ipynb') + payload['notebook'] = ipynb kernel = payload['kernel'] cell_index = payload['cell_index'] - notebook = nbformat.reads(json.dumps(payload['notebook']), nbformat.NO_CONVERT) + if isinstance(payload['notebook'], dict): + payload['notebook'] = json.dumps(payload['notebook']) + notebook = nbformat.reads(payload['notebook'], nbformat.NO_CONVERT) source = notebook.cells[cell_index].source diff --git a/vreapis/requirements.txt b/vreapis/requirements.txt index 8979a7a8..848d4334 100644 --- a/vreapis/requirements.txt +++ b/vreapis/requirements.txt @@ -22,3 +22,4 @@ pytype autopep8 distro pygithub +jupytext From b1f95882c08b200fcf34df1f3e8b9059f6c483c5 Mon Sep 17 00:00:00 2001 From: Andy Bayer Roswell Date: Wed, 24 Jul 2024 00:03:59 +0200 Subject: [PATCH 051/149] adding rmd support: type hints --- vreapis/containerizer/views.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/vreapis/containerizer/views.py b/vreapis/containerizer/views.py index 22efce68..9c3c6f92 100644 --- a/vreapis/containerizer/views.py +++ b/vreapis/containerizer/views.py @@ -69,14 +69,14 @@ class ExtractorHandler(APIView): authentication_classes = [StaticTokenAuthentication] permission_classes = [IsAuthenticated] - def extract_cell_by_index(self, notebook, cell_index): + def extract_cell_by_index(self, notebook: nbformat.notebooknode, cell_index: int) -> nbformat.NotebookNode: new_nb = copy.deepcopy(notebook) if cell_index < len(notebook.cells): new_nb.cells = [notebook.cells[cell_index]] return new_nb - def set_notebook_kernel(self, notebook, kernel): - new_nb = copy.deepcopy(notebook) + def set_notebook_kernel(self, notebook: nbformat.NotebookNode, kernel: str) -> nbformat.NotebookNode: + new_nb: nbformat.NotebookNode = copy.deepcopy(notebook) # Replace kernel name in the notebook metadata new_nb.metadata.kernelspec.name = kernel new_nb.metadata.kernelspec.display_name = kernel From e5e13b99844fccbc6c4d75c487e64dcbea620d78 Mon Sep 17 00:00:00 2001 From: Andy Bayer Roswell Date: Wed, 24 Jul 2024 00:09:43 +0200 Subject: [PATCH 052/149] adding rmd support --- vreapis/containerizer/views.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/vreapis/containerizer/views.py b/vreapis/containerizer/views.py index 9c3c6f92..4d940f5e 100644 --- a/vreapis/containerizer/views.py +++ b/vreapis/containerizer/views.py @@ -78,9 +78,11 @@ def extract_cell_by_index(self, notebook: nbformat.notebooknode, cell_index: int def set_notebook_kernel(self, notebook: nbformat.NotebookNode, kernel: str) -> nbformat.NotebookNode: new_nb: nbformat.NotebookNode = copy.deepcopy(notebook) # Replace kernel name in the notebook metadata - new_nb.metadata.kernelspec.name = kernel - new_nb.metadata.kernelspec.display_name = kernel - new_nb.metadata.kernelspec.language = kernel + new_nb.metadata['kernelspec'] = { + 'name': kernel, + 'display_name': kernel, + 'language': kernel, + } return new_nb def get(self, request: Request): From 89d9e9fa90533d537e35378091dbc19bb3f5fe32 Mon Sep 17 00:00:00 2001 From: Andy Bayer Roswell Date: Thu, 25 Jul 2024 00:29:16 +0200 Subject: [PATCH 053/149] adding rmd support --- vreapis/containerizer/jupytext.toml | 2 ++ vreapis/containerizer/views.py | 14 ++++++++++---- 2 files changed, 12 insertions(+), 4 deletions(-) create mode 100644 vreapis/containerizer/jupytext.toml diff --git a/vreapis/containerizer/jupytext.toml b/vreapis/containerizer/jupytext.toml new file mode 100644 index 00000000..9250219a --- /dev/null +++ b/vreapis/containerizer/jupytext.toml @@ -0,0 +1,2 @@ +[tool.jupytext] +split_at_heading = true diff --git a/vreapis/containerizer/views.py b/vreapis/containerizer/views.py index 4d940f5e..0d700d85 100644 --- a/vreapis/containerizer/views.py +++ b/vreapis/containerizer/views.py @@ -9,6 +9,7 @@ import uuid from typing import Optional from pathlib import Path +import subprocess import autopep8 from distro import distro @@ -28,7 +29,6 @@ import jsonschema from slugify import slugify from github import Github, UnknownObjectException -import jupytext from catalog.serializers import CellSerializer from containerizer.RContainerizer import RContainerizer @@ -92,14 +92,20 @@ def post(self, request: Request): payload = request.data common.logger.debug('ExtractorHandler. payload: ' + json.dumps(payload, indent=4)) if 'rmarkdown' in payload: - Rmd = jupytext.reads(payload['rmarkdown'], fmt='Rmd') - ipynb = jupytext.writes(Rmd, fmt='ipynb') - payload['notebook'] = ipynb + # Directly setting `NotebookNode.metadata['jupytext'] = {'split_at_heading': True, }` is no use. I don't know why. So we don't use lib jupytext here. + venv_activator = '/opt/venv/bin/activate' + command_jupytext = f'source {venv_activator}; jupytext --from Rmd --to ipynb --opt split_at_heading=true -o -' + process_jupytext = subprocess.Popen(command_jupytext, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True, executable='/bin/bash') + stdout, stderr = process_jupytext.communicate(input=payload['rmarkdown'].encode()) + process_jupytext.stdin.close() + process_jupytext.wait() + payload['notebook'] = stdout.decode() kernel = payload['kernel'] cell_index = payload['cell_index'] if isinstance(payload['notebook'], dict): payload['notebook'] = json.dumps(payload['notebook']) notebook = nbformat.reads(payload['notebook'], nbformat.NO_CONVERT) + common.logger.debug(len(notebook.cells)) source = notebook.cells[cell_index].source From 9be46cd14e298c70eee5773478cfe43718c241f3 Mon Sep 17 00:00:00 2001 From: Andy Bayer Roswell Date: Thu, 25 Jul 2024 12:17:58 +0200 Subject: [PATCH 054/149] adding rmd support: correcting cell_index --- vreapis/containerizer/views.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/vreapis/containerizer/views.py b/vreapis/containerizer/views.py index 0d700d85..b3a7b717 100644 --- a/vreapis/containerizer/views.py +++ b/vreapis/containerizer/views.py @@ -1,3 +1,4 @@ +import bisect import importlib import os import json @@ -100,12 +101,14 @@ def post(self, request: Request): process_jupytext.stdin.close() process_jupytext.wait() payload['notebook'] = stdout.decode() + cell_index = payload['cell_index'] - bisect.bisect_right(payload['rmarkdown_heading_indices'], payload['cell_index']) - 1 + else: # if 'notebook' in payload + cell_index = payload['cell_index'] kernel = payload['kernel'] - cell_index = payload['cell_index'] if isinstance(payload['notebook'], dict): payload['notebook'] = json.dumps(payload['notebook']) notebook = nbformat.reads(payload['notebook'], nbformat.NO_CONVERT) - common.logger.debug(len(notebook.cells)) + common.logger.debug(cell_index) source = notebook.cells[cell_index].source From ae3f3c27e728a5de2c9075f16e29c64af0b842cf Mon Sep 17 00:00:00 2001 From: Andy Bayer Roswell Date: Thu, 25 Jul 2024 16:09:43 +0200 Subject: [PATCH 055/149] adding rmd support: correcting cell_index fixing 'Conversion rules for `rpy2.robjects` appear to be missing' --- vreapis/services/extractor/rextractor.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/vreapis/services/extractor/rextractor.py b/vreapis/services/extractor/rextractor.py index 9a4959c9..6e3fe46e 100644 --- a/vreapis/services/extractor/rextractor.py +++ b/vreapis/services/extractor/rextractor.py @@ -7,6 +7,7 @@ import rpy2.robjects.packages as rpackages from rpy2.robjects.packages import importr from rpy2.rinterface_lib.callbacks import logger as rpy2_logger +from rpy2.robjects import conversion, default_converter import logging rpy2_logger.setLevel(logging.WARNING) @@ -163,9 +164,10 @@ def __extract_imports(self, sources): ''' Approach 2: Static analysis using 'renv' package. this approach is more safe as it covers more cases and checks comments ''' - with tempfile.NamedTemporaryFile(delete=False, suffix='.R') as tmp_file: + with tempfile.NamedTemporaryFile(delete=False, suffix='.R') as tmp_file, conversion.localconverter(default_converter) as converter: tmp_file.write(s.encode()) tmp_file.flush() + robjects.conversion.set_conversion(converter) renv = rpackages.importr('renv') function_list = renv.dependencies(tmp_file.name) From 56f4fe96b2196df37047997edc759b333b8091c9 Mon Sep 17 00:00:00 2001 From: Andy Bayer Roswell Date: Thu, 25 Jul 2024 16:43:38 +0200 Subject: [PATCH 056/149] adding rmd support: correcting cell_index fixing 'Conversion rules for `rpy2.robjects` appear to be missing' --- vreapis/services/extractor/rextractor.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/vreapis/services/extractor/rextractor.py b/vreapis/services/extractor/rextractor.py index 6e3fe46e..4202e6f9 100644 --- a/vreapis/services/extractor/rextractor.py +++ b/vreapis/services/extractor/rextractor.py @@ -17,6 +17,9 @@ # Create an R environment r_env = robjects.globalenv +robject_converter = default_converter # + conversion.localconverter(default_converter) +robjects.conversion.set_conversion(robject_converter) + # install R packages robjects.r(''' install_package_with_retry <- function(package_name, max_attempts = 5) { @@ -164,10 +167,10 @@ def __extract_imports(self, sources): ''' Approach 2: Static analysis using 'renv' package. this approach is more safe as it covers more cases and checks comments ''' - with tempfile.NamedTemporaryFile(delete=False, suffix='.R') as tmp_file, conversion.localconverter(default_converter) as converter: + with tempfile.NamedTemporaryFile(delete=False, suffix='.R') as tmp_file: tmp_file.write(s.encode()) tmp_file.flush() - robjects.conversion.set_conversion(converter) + robjects.conversion.set_conversion(robject_converter) renv = rpackages.importr('renv') function_list = renv.dependencies(tmp_file.name) From 7d872dcde1a9d0970b7deb3dba2a354b9ae15784 Mon Sep 17 00:00:00 2001 From: Andy Bayer Roswell Date: Thu, 25 Jul 2024 19:49:08 +0200 Subject: [PATCH 057/149] adding rmd support: correcting cell_index fixing 'Conversion rules for `rpy2.robjects` appear to be missing' --- vreapis/services/extractor/rextractor.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/vreapis/services/extractor/rextractor.py b/vreapis/services/extractor/rextractor.py index 4202e6f9..a901221f 100644 --- a/vreapis/services/extractor/rextractor.py +++ b/vreapis/services/extractor/rextractor.py @@ -17,8 +17,8 @@ # Create an R environment r_env = robjects.globalenv -robject_converter = default_converter # + conversion.localconverter(default_converter) -robjects.conversion.set_conversion(robject_converter) +robject_converter = default_converter # + conversion.localconverter(default_converter) +# robjects.conversion.set_conversion(robject_converter) # install R packages robjects.r(''' @@ -40,7 +40,7 @@ return(FALSE) } ''') -packnames = ('rlang', 'lobstr', 'purrr','renv',) +packnames = ('rlang', 'lobstr', 'purrr', 'renv',) for p in packnames: if not rpackages.isinstalled(p): robjects.r(f'install_package_with_retry("{p}")') From 6498131ca4aad6c76a1d81306d4e1a4e568d7721 Mon Sep 17 00:00:00 2001 From: Andy Bayer Roswell Date: Thu, 25 Jul 2024 21:48:11 +0200 Subject: [PATCH 058/149] adding rmd support: corrected cell_index fixed 'Conversion rules for `rpy2.robjects` appear to be missing' eliminated redundant package renv imports --- vreapis/containerizer/views.py | 4 ++-- vreapis/services/extractor/rextractor.py | 7 ++++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/vreapis/containerizer/views.py b/vreapis/containerizer/views.py index b3a7b717..ddc421f8 100644 --- a/vreapis/containerizer/views.py +++ b/vreapis/containerizer/views.py @@ -91,7 +91,7 @@ def get(self, request: Request): def post(self, request: Request): payload = request.data - common.logger.debug('ExtractorHandler. payload: ' + json.dumps(payload, indent=4)) + # common.logger.debug('ExtractorHandler. payload: ' + json.dumps(payload, indent=4)) if 'rmarkdown' in payload: # Directly setting `NotebookNode.metadata['jupytext'] = {'split_at_heading': True, }` is no use. I don't know why. So we don't use lib jupytext here. venv_activator = '/opt/venv/bin/activate' @@ -108,7 +108,7 @@ def post(self, request: Request): if isinstance(payload['notebook'], dict): payload['notebook'] = json.dumps(payload['notebook']) notebook = nbformat.reads(payload['notebook'], nbformat.NO_CONVERT) - common.logger.debug(cell_index) + # common.logger.debug(cell_index) source = notebook.cells[cell_index].source diff --git a/vreapis/services/extractor/rextractor.py b/vreapis/services/extractor/rextractor.py index a901221f..8a2a19a7 100644 --- a/vreapis/services/extractor/rextractor.py +++ b/vreapis/services/extractor/rextractor.py @@ -155,6 +155,9 @@ def __init__(self, notebook, cell_source): super().__init__(notebook, cell_source) def __extract_imports(self, sources): + robjects.conversion.set_conversion(robject_converter) + renv = rpackages.importr('renv') + imports = {} for s in sources: packages = [] @@ -170,8 +173,6 @@ def __extract_imports(self, sources): with tempfile.NamedTemporaryFile(delete=False, suffix='.R') as tmp_file: tmp_file.write(s.encode()) tmp_file.flush() - robjects.conversion.set_conversion(robject_converter) - renv = rpackages.importr('renv') function_list = renv.dependencies(tmp_file.name) # transpose renv dependencies to readable dependencies @@ -183,7 +184,7 @@ def __extract_imports(self, sources): # format the packages for package in packages: imports[package] = { - # asname and module are specific to Python packages. So you can probably leave them out here + # as name and module are specific to Python packages. So you can probably leave them out here 'name': package, 'asname': '', 'module': '' From 604a5aefeffa128842cb570fcf6f3f1c32593396 Mon Sep 17 00:00:00 2001 From: Andy Bayer Roswell Date: Fri, 26 Jul 2024 10:13:01 +0200 Subject: [PATCH 059/149] adding rmd support: corrected cell_index fixed 'Conversion rules for `rpy2.robjects` appear to be missing' eliminated redundant package renv imports --- vreapis/containerizer/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vreapis/containerizer/views.py b/vreapis/containerizer/views.py index ddc421f8..e81825b4 100644 --- a/vreapis/containerizer/views.py +++ b/vreapis/containerizer/views.py @@ -101,7 +101,7 @@ def post(self, request: Request): process_jupytext.stdin.close() process_jupytext.wait() payload['notebook'] = stdout.decode() - cell_index = payload['cell_index'] - bisect.bisect_right(payload['rmarkdown_heading_indices'], payload['cell_index']) - 1 + cell_index = payload['cell_index'] - bisect.bisect_right(payload['rmarkdown_offset_indices'], payload['cell_index']) - 1 else: # if 'notebook' in payload cell_index = payload['cell_index'] kernel = payload['kernel'] From f1da57ce0c98c3370ebc3eac55c8aabce39a05be Mon Sep 17 00:00:00 2001 From: Andy Bayer Roswell Date: Tue, 6 Aug 2024 02:26:54 +0200 Subject: [PATCH 060/149] adding rmd support: corrected cell_index filter blank md cells to keep consistency with parsermd --- vreapis/containerizer/views.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/vreapis/containerizer/views.py b/vreapis/containerizer/views.py index e81825b4..2822ebc0 100644 --- a/vreapis/containerizer/views.py +++ b/vreapis/containerizer/views.py @@ -102,13 +102,20 @@ def post(self, request: Request): process_jupytext.wait() payload['notebook'] = stdout.decode() cell_index = payload['cell_index'] - bisect.bisect_right(payload['rmarkdown_offset_indices'], payload['cell_index']) - 1 - else: # if 'notebook' in payload + else: # if 'notebook' in payload cell_index = payload['cell_index'] kernel = payload['kernel'] if isinstance(payload['notebook'], dict): payload['notebook'] = json.dumps(payload['notebook']) notebook = nbformat.reads(payload['notebook'], nbformat.NO_CONVERT) - # common.logger.debug(cell_index) + if 'rmarkdown' in payload: + filtered_cells = [notebook.cells[0]] + for i in range(1, len(notebook.cells)): + cell = notebook.cells[i] + if not (cell['cell_type'] == 'markdown' and all(line == os.linesep for line in cell['source']) and notebook.cells[i - 1]['cell_type'] == 'code'): + filtered_cells.append(cell) + notebook.cells = filtered_cells + common.logger.debug(cell_index) source = notebook.cells[cell_index].source From f6cf9b5f1a4f6f5f55cdf906e6527ba49b62473d Mon Sep 17 00:00:00 2001 From: Andy Bayer Roswell Date: Wed, 7 Aug 2024 00:42:03 +0200 Subject: [PATCH 061/149] comments on rmd support --- vreapis/containerizer/views.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vreapis/containerizer/views.py b/vreapis/containerizer/views.py index 2822ebc0..4d74259a 100644 --- a/vreapis/containerizer/views.py +++ b/vreapis/containerizer/views.py @@ -93,7 +93,7 @@ def post(self, request: Request): payload = request.data # common.logger.debug('ExtractorHandler. payload: ' + json.dumps(payload, indent=4)) if 'rmarkdown' in payload: - # Directly setting `NotebookNode.metadata['jupytext'] = {'split_at_heading': True, }` is no use. I don't know why. So we don't use lib jupytext here. + # Directly setting `NotebookNode.metadata['jupytext'] = {'split_at_heading': True, }` has no use. I don't know why. So we don't use lib jupytext here. venv_activator = '/opt/venv/bin/activate' command_jupytext = f'source {venv_activator}; jupytext --from Rmd --to ipynb --opt split_at_heading=true -o -' process_jupytext = subprocess.Popen(command_jupytext, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True, executable='/bin/bash') @@ -108,7 +108,7 @@ def post(self, request: Request): if isinstance(payload['notebook'], dict): payload['notebook'] = json.dumps(payload['notebook']) notebook = nbformat.reads(payload['notebook'], nbformat.NO_CONVERT) - if 'rmarkdown' in payload: + if 'rmarkdown' in payload: # parsermd ignores md cells with \n only. We need to remove them here to keep consistency filtered_cells = [notebook.cells[0]] for i in range(1, len(notebook.cells)): cell = notebook.cells[i] From 175eb05cb67f90ee1d2c62a0b7343e97d597aa1e Mon Sep 17 00:00:00 2001 From: Andy Bayer Roswell Date: Thu, 8 Aug 2024 19:07:20 +0200 Subject: [PATCH 062/149] slight modifications --- vreapis/containerizer/views.py | 4 ++-- vreapis/vreapis/settings/base.py | 4 +++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/vreapis/containerizer/views.py b/vreapis/containerizer/views.py index 4d74259a..6432af6a 100644 --- a/vreapis/containerizer/views.py +++ b/vreapis/containerizer/views.py @@ -30,6 +30,7 @@ import jsonschema from slugify import slugify from github import Github, UnknownObjectException +from django.conf import settings from catalog.serializers import CellSerializer from containerizer.RContainerizer import RContainerizer @@ -94,8 +95,7 @@ def post(self, request: Request): # common.logger.debug('ExtractorHandler. payload: ' + json.dumps(payload, indent=4)) if 'rmarkdown' in payload: # Directly setting `NotebookNode.metadata['jupytext'] = {'split_at_heading': True, }` has no use. I don't know why. So we don't use lib jupytext here. - venv_activator = '/opt/venv/bin/activate' - command_jupytext = f'source {venv_activator}; jupytext --from Rmd --to ipynb --opt split_at_heading=true -o -' + command_jupytext = f'source {settings.VENV_ACTIVATOR}; jupytext --from Rmd --to ipynb --opt split_at_heading=true -o -' process_jupytext = subprocess.Popen(command_jupytext, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True, executable='/bin/bash') stdout, stderr = process_jupytext.communicate(input=payload['rmarkdown'].encode()) process_jupytext.stdin.close() diff --git a/vreapis/vreapis/settings/base.py b/vreapis/vreapis/settings/base.py index 026df70c..7299ea7a 100644 --- a/vreapis/vreapis/settings/base.py +++ b/vreapis/vreapis/settings/base.py @@ -188,7 +188,7 @@ STATIC_URL = f'{BASE_PATH}/static/' STATIC_ROOT = BASE_DIR / "staticfiles" -# CUSTOM VARS +# CUSTOM VARS # todo. delete default values API_ENDPOINT: str = os.getenv('API_ENDPOINT', "https://naavre-dev.minikube.test/vre-api-test") @@ -199,3 +199,5 @@ KEYCLOAK_CLIENT_ID: str = os.getenv('KEYCLOAK_CLIENT_ID', 'myclient') NAAVRE_API_TOKEN: str = os.getenv('NAAVRE_API_TOKEN') + +VENV_ACTIVATOR: str = os.getenv('VENV_ACTIVATOR', '/opt/venv/bin/activate') From 3b0f71847b2378ed312601d376e7c4cc2959aec1 Mon Sep 17 00:00:00 2001 From: Andy Bayer Roswell Date: Mon, 12 Aug 2024 17:09:22 +0200 Subject: [PATCH 063/149] [bugfix] if do_dispatch_github_workflow --- vreapis/containerizer/views.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/vreapis/containerizer/views.py b/vreapis/containerizer/views.py index 6432af6a..823f2099 100644 --- a/vreapis/containerizer/views.py +++ b/vreapis/containerizer/views.py @@ -528,6 +528,7 @@ def create(self, request: Request, *args, **kwargs): do_dispatch_github_workflow = True else: image_info = self.query_registry_for_image(image_repo=image_repo, image_name=current_cell.task_name, ) + common.logger.debug(f'image_info: {image_info}') if not image_info: do_dispatch_github_workflow = True @@ -538,9 +539,7 @@ def create(self, request: Request, *args, **kwargs): return return_error(resp.text) current_cell.set_image_version(image_version) Cell.objects.filter(task_name=current_cell.task_name).delete() - serializer = self.get_serializer(data=current_cell) - serializer.is_valid(raise_exception=True) - instance = serializer.save() + current_cell.save() return Response({'wf_id': wf_id, 'dispatched_github_workflow': do_dispatch_github_workflow, 'image_version': image_version}) From 71240cf147c0d009f215cab8311f6a13708536df Mon Sep 17 00:00:00 2001 From: Andy Bayer Roswell Date: Mon, 26 Aug 2024 15:49:52 +0200 Subject: [PATCH 064/149] [bugfix] RContainerizer.py --- vreapis/containerizer/RContainerizer.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/vreapis/containerizer/RContainerizer.py b/vreapis/containerizer/RContainerizer.py index cb1de17e..39287183 100644 --- a/vreapis/containerizer/RContainerizer.py +++ b/vreapis/containerizer/RContainerizer.py @@ -4,6 +4,7 @@ import jinja2 import common +from catalog.models import Cell handler = logging.StreamHandler() handler.setLevel(logging.DEBUG) @@ -57,9 +58,11 @@ def map_dependencies(dependencies, module_name_mapping): return set_conda_deps, set_pip_deps @staticmethod - def build_templates(cell=None, files_info=None, module_name_mapping=None): + def build_templates(cell: Cell=None, files_info=None, module_name_mapping=None): # we also want to always add the id to the input parameters inputs = cell.inputs + if isinstance(cell.types, list): + cell.types = {} # In R, builtin list is used to represent both arrays and dicts. From R side of the frontend, an empty list is serialized into [] rather than {} and it seems difficult to handle this in R. So we correct it here. types = cell.types inputs.append('id') cell.concatenate_all_inputs() From a5ba124286d913ffbd77579da2e596fb21182556 Mon Sep 17 00:00:00 2001 From: Andy Bayer Roswell Date: Mon, 26 Aug 2024 19:33:36 +0200 Subject: [PATCH 065/149] fixing `Error: rpy2 in API mode cannot be built without R in the PATH or R_HOME defined. Correct this or force ABI mode-only by defining the environment variable RPY2_CFFI_MODE=ABI` --- vreapis/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vreapis/Dockerfile b/vreapis/Dockerfile index 9c730e6e..0ff8ad86 100644 --- a/vreapis/Dockerfile +++ b/vreapis/Dockerfile @@ -3,7 +3,7 @@ FROM python:3.12.2-slim WORKDIR /app RUN apt update && \ - apt install -y libpq-dev gcc gdal-bin curl net-tools iputils-ping + apt install -y libpq-dev gcc gdal-bin curl net-tools iputils-ping r-base RUN python3 -m venv /opt/venv RUN /opt/venv/bin/pip install pip --upgrade From 7100c6911d5b292b5cee0571fe4530b0d95de74d Mon Sep 17 00:00:00 2001 From: Andy Bayer Roswell <46857536+AndyBRoswell@users.noreply.github.com> Date: Mon, 26 Aug 2024 19:42:49 +0200 Subject: [PATCH 066/149] Update ci-pipeline.yml: timeout 300 for waiting for healthy app --- .github/workflows/ci-pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index 96c6f31d..1cb732b5 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -46,7 +46,7 @@ jobs: uses: raschmitt/wait-for-healthy-container/@v1 with: container-name: app - timeout: 120 + timeout: 300 - name: Login to github Registry if: ${{ inputs.push }} From 70952b14f1e884b7e9c1c20570ccee40883242e1 Mon Sep 17 00:00:00 2001 From: Andy Bayer Roswell <46857536+AndyBRoswell@users.noreply.github.com> Date: Tue, 27 Aug 2024 00:36:50 +0200 Subject: [PATCH 067/149] Update ci-pipeline.yml: try to capture docker logs --- .github/workflows/ci-pipeline.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index 1cb732b5..d1974f20 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -46,7 +46,13 @@ jobs: uses: raschmitt/wait-for-healthy-container/@v1 with: container-name: app - timeout: 300 + timeout: 120 + + - name: Capture Docker Logs + run: | + echo 'Capturing Docker logs...' + docker logs app + shell: /usr/bin/bash --noprofile --norc -e -o pipefail - name: Login to github Registry if: ${{ inputs.push }} From 2345cb36a47e4ba2967ad50d623b00b9f69d7571 Mon Sep 17 00:00:00 2001 From: Andy Bayer Roswell <46857536+AndyBRoswell@users.noreply.github.com> Date: Tue, 27 Aug 2024 00:42:19 +0200 Subject: [PATCH 068/149] Update ci-pipeline.yml: try to capture docker logs --- .github/workflows/ci-pipeline.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index d1974f20..59897854 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -52,7 +52,6 @@ jobs: run: | echo 'Capturing Docker logs...' docker logs app - shell: /usr/bin/bash --noprofile --norc -e -o pipefail - name: Login to github Registry if: ${{ inputs.push }} From f2d7406b592d1db43c6309c374764c7e181fbc35 Mon Sep 17 00:00:00 2001 From: Andy Bayer Roswell <46857536+AndyBRoswell@users.noreply.github.com> Date: Tue, 27 Aug 2024 00:48:38 +0200 Subject: [PATCH 069/149] Update ci-pipeline.yml: try to capture docker log --- .github/workflows/ci-pipeline.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index 59897854..ae7a01cc 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -49,6 +49,7 @@ jobs: timeout: 120 - name: Capture Docker Logs + if: always() run: | echo 'Capturing Docker logs...' docker logs app From 7e465e05e85c8fd56df6715a9d95a3e846d2190f Mon Sep 17 00:00:00 2001 From: Andy Bayer Roswell <46857536+AndyBRoswell@users.noreply.github.com> Date: Tue, 27 Aug 2024 13:07:59 +0200 Subject: [PATCH 070/149] Update ci-pipeline.yml: capture logs only if failed; refresh envvar --- .github/workflows/ci-pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index ae7a01cc..a73c71ca 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -49,7 +49,7 @@ jobs: timeout: 120 - name: Capture Docker Logs - if: always() + if: failure() run: | echo 'Capturing Docker logs...' docker logs app From 1d917839785911efd242cb409ecb480dcaec595b Mon Sep 17 00:00:00 2001 From: Andy Bayer Roswell <46857536+AndyBRoswell@users.noreply.github.com> Date: Tue, 27 Aug 2024 13:51:18 +0200 Subject: [PATCH 071/149] Update ci-pipeline.yml: add envvar --- .github/workflows/ci-pipeline.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index a73c71ca..274a05de 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -25,6 +25,8 @@ env: DOCKER_FOLDER: ${{ inputs.docker_folder }} TAG: ${{ inputs.tag }} DOCKERHUB_IMAGE_ID: "qcdis/" + CELL_GITHUB: ${{ env.CELL_GITHUB }} + CELL_GITHUB_TOKEN: ${{ secrets.CELL_GITHUB_TOKEN }} jobs: build: From 568b3613b12f2a472fa2dcce547c28268d8d35a8 Mon Sep 17 00:00:00 2001 From: Andy Bayer Roswell <46857536+AndyBRoswell@users.noreply.github.com> Date: Tue, 27 Aug 2024 13:57:33 +0200 Subject: [PATCH 072/149] Update ci-pipeline.yml: add envvar --- .github/workflows/ci-pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index 274a05de..b4f88f27 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -25,7 +25,7 @@ env: DOCKER_FOLDER: ${{ inputs.docker_folder }} TAG: ${{ inputs.tag }} DOCKERHUB_IMAGE_ID: "qcdis/" - CELL_GITHUB: ${{ env.CELL_GITHUB }} + CELL_GITHUB: "https://github.com/QCDIS/NaaVRE-cells-test-3" CELL_GITHUB_TOKEN: ${{ secrets.CELL_GITHUB_TOKEN }} jobs: From e003def06fa2ec0e1611390e11e830bab0e452a3 Mon Sep 17 00:00:00 2001 From: Andy Bayer Roswell <46857536+AndyBRoswell@users.noreply.github.com> Date: Tue, 27 Aug 2024 15:14:47 +0200 Subject: [PATCH 073/149] Update ci-pipeline.yml: envvar chk --- .github/workflows/ci-pipeline.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index b4f88f27..f1b392a9 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -50,6 +50,9 @@ jobs: container-name: app timeout: 120 + - name: Print all environment variables + run: printenv + - name: Capture Docker Logs if: failure() run: | From 47d47dcef5dbc140f675051e406b01658da49343 Mon Sep 17 00:00:00 2001 From: Andy Bayer Roswell <46857536+AndyBRoswell@users.noreply.github.com> Date: Tue, 27 Aug 2024 15:28:13 +0200 Subject: [PATCH 074/149] Update ci-pipeline.yml: envvar chk --- .github/workflows/ci-pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index f1b392a9..483e6bd8 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -51,7 +51,7 @@ jobs: timeout: 120 - name: Print all environment variables - run: printenv + run: env - name: Capture Docker Logs if: failure() From 91a61b0dcf5f2fa3240f43d9c8f35f4a817396d6 Mon Sep 17 00:00:00 2001 From: Andy Bayer Roswell <46857536+AndyBRoswell@users.noreply.github.com> Date: Tue, 27 Aug 2024 15:42:49 +0200 Subject: [PATCH 075/149] Update ci-pipeline.yml: envvar chk --- .github/workflows/ci-pipeline.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index 483e6bd8..ea0da4b9 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -51,7 +51,9 @@ jobs: timeout: 120 - name: Print all environment variables - run: env + run: | + echo 'envvar:' + env - name: Capture Docker Logs if: failure() From 9b235b4c7db07c0922cdc8dcf7ee56432410ada1 Mon Sep 17 00:00:00 2001 From: Andy Bayer Roswell <46857536+AndyBRoswell@users.noreply.github.com> Date: Tue, 27 Aug 2024 15:59:43 +0200 Subject: [PATCH 076/149] Update ci-pipeline.yml --- .github/workflows/ci-pipeline.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index ea0da4b9..78f43ea3 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -58,7 +58,9 @@ jobs: - name: Capture Docker Logs if: failure() run: | - echo 'Capturing Docker logs...' + echo '-> envvar:' + env + echo '-> Capturing Docker logs ...' docker logs app - name: Login to github Registry From 5fbb1f9409c04d3f6ff85ab11b4514d646c08f08 Mon Sep 17 00:00:00 2001 From: Andy Bayer Roswell Date: Tue, 27 Aug 2024 17:28:38 +0200 Subject: [PATCH 077/149] envvar check for github workflow --- vreapis/containerizer/views.py | 1 + 1 file changed, 1 insertion(+) diff --git a/vreapis/containerizer/views.py b/vreapis/containerizer/views.py index 823f2099..16b861bd 100644 --- a/vreapis/containerizer/views.py +++ b/vreapis/containerizer/views.py @@ -190,6 +190,7 @@ def post(self, request: Request): class CellsHandler(viewsets.ModelViewSet): + common.logger.debug(f"CELL_GITHUB: {os.getenv('CELL_GITHUB')}") queryset: QuerySet = Cell.objects.all() serializer_class = CellSerializer authentication_classes: list[BaseAuthentication] = [StaticTokenAuthentication] From 2604f6e5e8a6d53a610b7d3762f8a20434776a63 Mon Sep 17 00:00:00 2001 From: Andy Bayer Roswell <46857536+AndyBRoswell@users.noreply.github.com> Date: Tue, 27 Aug 2024 17:41:55 +0200 Subject: [PATCH 078/149] Update ci-pipeline.yml: envvar chk --- .github/workflows/ci-pipeline.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index 78f43ea3..4d6a605b 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -62,6 +62,8 @@ jobs: env echo '-> Capturing Docker logs ...' docker logs app + echo '-> envvar:' + env - name: Login to github Registry if: ${{ inputs.push }} From e82f546c63e91141be336454242723943d45190e Mon Sep 17 00:00:00 2001 From: Andy Bayer Roswell <46857536+AndyBRoswell@users.noreply.github.com> Date: Tue, 27 Aug 2024 19:07:50 +0200 Subject: [PATCH 079/149] Update ci-pipeline.yml: envvar chk --- .github/workflows/ci-pipeline.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index 4d6a605b..2380fe43 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -37,6 +37,8 @@ jobs: - name: Build docker run: | + echo '-> envvar:' + env cd $DOCKER_FOLDER && docker build . --file Dockerfile -t $TAG --build-arg "NODE_ENV=${{ inputs.environment }}" - name: Run docker compose From 9fe029a8a7aace45f54b184db70ef0aac7decf5e Mon Sep 17 00:00:00 2001 From: Andy Bayer Roswell <46857536+AndyBRoswell@users.noreply.github.com> Date: Tue, 27 Aug 2024 19:38:08 +0200 Subject: [PATCH 080/149] Update ci-pipeline.yml: envvar chk --- .github/workflows/ci-pipeline.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index 2380fe43..87c98ba9 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -35,6 +35,9 @@ jobs: steps: - uses: actions/checkout@v4 + - name: envvar chk + run: python -c "import os; print(os.getenv('CELL_GITHUB'))" + - name: Build docker run: | echo '-> envvar:' From c697bb2da8fae12021e3caa07c61cd6923faba72 Mon Sep 17 00:00:00 2001 From: Andy Bayer Roswell <46857536+AndyBRoswell@users.noreply.github.com> Date: Tue, 27 Aug 2024 19:50:23 +0200 Subject: [PATCH 081/149] Update ci-pipeline.yml: envvar chk --- .github/workflows/ci-pipeline.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index 87c98ba9..29a3e2a2 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -48,6 +48,8 @@ jobs: uses: isbang/compose-action@v1.5.1 with: compose-file: "${{ inputs.docker_folder }}/docker-compose.yaml" + env: + CELL_GITHUB: "https://github.com/QCDIS/NaaVRE-cells-test-3" - name: Wait for healthy app uses: raschmitt/wait-for-healthy-container/@v1 From 9a16b1caafa30f5bb339d06f678e048a6c3accb7 Mon Sep 17 00:00:00 2001 From: Andy Bayer Roswell <46857536+AndyBRoswell@users.noreply.github.com> Date: Tue, 27 Aug 2024 19:57:46 +0200 Subject: [PATCH 082/149] Update ci-pipeline.yml: envvar --- .github/workflows/ci-pipeline.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index 29a3e2a2..b6954d81 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -45,7 +45,8 @@ jobs: cd $DOCKER_FOLDER && docker build . --file Dockerfile -t $TAG --build-arg "NODE_ENV=${{ inputs.environment }}" - name: Run docker compose - uses: isbang/compose-action@v1.5.1 + # uses: isbang/compose-action@v1.5.1 + uses: hoverkraft-tech/compose-action@v0.0.0 with: compose-file: "${{ inputs.docker_folder }}/docker-compose.yaml" env: From 14e59f05102aa96a7b6e34a5459e2709845cc755 Mon Sep 17 00:00:00 2001 From: Andy Bayer Roswell <46857536+AndyBRoswell@users.noreply.github.com> Date: Tue, 27 Aug 2024 19:58:43 +0200 Subject: [PATCH 083/149] Update ci-pipeline.yml: envvar --- .github/workflows/ci-pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index b6954d81..3a07b98e 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -46,7 +46,7 @@ jobs: - name: Run docker compose # uses: isbang/compose-action@v1.5.1 - uses: hoverkraft-tech/compose-action@v0.0.0 + uses: hoverkraft-tech/compose-action@v2.0.1 with: compose-file: "${{ inputs.docker_folder }}/docker-compose.yaml" env: From 7629bc5e71052c4801fd84330f6e008d25be1f50 Mon Sep 17 00:00:00 2001 From: Andy Bayer Roswell <46857536+AndyBRoswell@users.noreply.github.com> Date: Tue, 27 Aug 2024 22:43:32 +0200 Subject: [PATCH 084/149] Update ci-pipeline.yml: envvar chk --- .github/workflows/ci-pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index 3a07b98e..091f4669 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -42,7 +42,7 @@ jobs: run: | echo '-> envvar:' env - cd $DOCKER_FOLDER && docker build . --file Dockerfile -t $TAG --build-arg "NODE_ENV=${{ inputs.environment }}" + cd $DOCKER_FOLDER && docker build . --file Dockerfile -t $TAG --build-arg "NODE_ENV=${{ inputs.environment }}:CELL_GITHUB=${{ env.CELL_GITHUB }}" - name: Run docker compose # uses: isbang/compose-action@v1.5.1 From e7c022eb3c9ba3be903faa46b53233c305d38f0e Mon Sep 17 00:00:00 2001 From: Andy Bayer Roswell Date: Tue, 27 Aug 2024 22:54:51 +0200 Subject: [PATCH 085/149] envvar check for github workflow --- vreapis/docker-compose.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/vreapis/docker-compose.yaml b/vreapis/docker-compose.yaml index 9420187e..623d47b4 100644 --- a/vreapis/docker-compose.yaml +++ b/vreapis/docker-compose.yaml @@ -37,3 +37,7 @@ services: test: | curl --fail http://localhost:8000/vre-api-test/api/ interval: 5s + + envchk: + image: python:3.12.2-slim + command: python -c "import os; print(os.getenv('CELL_GITHUB'))" From 1b9f793005ea252f6e41b5744fbdc2515b820736 Mon Sep 17 00:00:00 2001 From: Andy Bayer Roswell <46857536+AndyBRoswell@users.noreply.github.com> Date: Tue, 27 Aug 2024 22:56:28 +0200 Subject: [PATCH 086/149] Update ci-pipeline.yml: envchk --- .github/workflows/ci-pipeline.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index 091f4669..72302945 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -52,6 +52,9 @@ jobs: env: CELL_GITHUB: "https://github.com/QCDIS/NaaVRE-cells-test-3" + - name: envchk + run: docker-compose logs envchk + - name: Wait for healthy app uses: raschmitt/wait-for-healthy-container/@v1 with: From 8a9373d5975e85e8a748401206d87f6004fdaabe Mon Sep 17 00:00:00 2001 From: Andy Bayer Roswell <46857536+AndyBRoswell@users.noreply.github.com> Date: Tue, 27 Aug 2024 23:03:45 +0200 Subject: [PATCH 087/149] Update ci-pipeline.yml: envchk --- .github/workflows/ci-pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index 72302945..b19d2eb3 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -53,7 +53,7 @@ jobs: CELL_GITHUB: "https://github.com/QCDIS/NaaVRE-cells-test-3" - name: envchk - run: docker-compose logs envchk + run: cd $DOCKER_FOLDER && docker-compose logs envchk - name: Wait for healthy app uses: raschmitt/wait-for-healthy-container/@v1 From 3164f79041a7e0d90f4f05fe87ff5836d92a8ac4 Mon Sep 17 00:00:00 2001 From: Andy Bayer Roswell Date: Tue, 27 Aug 2024 23:08:40 +0200 Subject: [PATCH 088/149] revert docker-compose.yaml --- vreapis/docker-compose.yaml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/vreapis/docker-compose.yaml b/vreapis/docker-compose.yaml index 623d47b4..9420187e 100644 --- a/vreapis/docker-compose.yaml +++ b/vreapis/docker-compose.yaml @@ -37,7 +37,3 @@ services: test: | curl --fail http://localhost:8000/vre-api-test/api/ interval: 5s - - envchk: - image: python:3.12.2-slim - command: python -c "import os; print(os.getenv('CELL_GITHUB'))" From d06d46997a819e9c8e3aab511d000cc7ec3a6e32 Mon Sep 17 00:00:00 2001 From: Andy Bayer Roswell <46857536+AndyBRoswell@users.noreply.github.com> Date: Tue, 27 Aug 2024 23:14:54 +0200 Subject: [PATCH 089/149] Update ci-pipeline.yml: chkenv --- .github/workflows/ci-pipeline.yml | 20 ++++++-------------- 1 file changed, 6 insertions(+), 14 deletions(-) diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index b19d2eb3..b4f4c792 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -35,25 +35,23 @@ jobs: steps: - uses: actions/checkout@v4 - - name: envvar chk + - name: chkenv run: python -c "import os; print(os.getenv('CELL_GITHUB'))" - name: Build docker run: | echo '-> envvar:' env - cd $DOCKER_FOLDER && docker build . --file Dockerfile -t $TAG --build-arg "NODE_ENV=${{ inputs.environment }}:CELL_GITHUB=${{ env.CELL_GITHUB }}" + cd $DOCKER_FOLDER && docker build . --file Dockerfile -t $TAG --build-arg "NODE_ENV=${{ inputs.environment }}" - name: Run docker compose # uses: isbang/compose-action@v1.5.1 uses: hoverkraft-tech/compose-action@v2.0.1 with: compose-file: "${{ inputs.docker_folder }}/docker-compose.yaml" - env: - CELL_GITHUB: "https://github.com/QCDIS/NaaVRE-cells-test-3" - - name: envchk - run: cd $DOCKER_FOLDER && docker-compose logs envchk + - name: chkenv2 + run: python -c "import os; print(os.getenv('CELL_GITHUB'))" - name: Wait for healthy app uses: raschmitt/wait-for-healthy-container/@v1 @@ -61,20 +59,14 @@ jobs: container-name: app timeout: 120 - - name: Print all environment variables - run: | - echo 'envvar:' - env + - name: chkenv + run: python -c "import os; print(os.getenv('CELL_GITHUB'))" - name: Capture Docker Logs if: failure() run: | - echo '-> envvar:' - env echo '-> Capturing Docker logs ...' docker logs app - echo '-> envvar:' - env - name: Login to github Registry if: ${{ inputs.push }} From 5ee56c16488917b2afbeea2e0bd6d1eee9fe4658 Mon Sep 17 00:00:00 2001 From: Andy Bayer Roswell <46857536+AndyBRoswell@users.noreply.github.com> Date: Tue, 27 Aug 2024 23:20:44 +0200 Subject: [PATCH 090/149] Update ci-pipeline.yml: chkenv --- .github/workflows/ci-pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index b4f4c792..fb4b278c 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -59,7 +59,7 @@ jobs: container-name: app timeout: 120 - - name: chkenv + - name: chkenv3 run: python -c "import os; print(os.getenv('CELL_GITHUB'))" - name: Capture Docker Logs From 5a85210641fa9532dc0beb2c69d082565f93db29 Mon Sep 17 00:00:00 2001 From: Andy Bayer Roswell <46857536+AndyBRoswell@users.noreply.github.com> Date: Tue, 27 Aug 2024 23:28:23 +0200 Subject: [PATCH 091/149] Update ci-pipeline.yml: chkenv --- .github/workflows/ci-pipeline.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index fb4b278c..965ab3c3 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -60,6 +60,7 @@ jobs: timeout: 120 - name: chkenv3 + if: always() run: python -c "import os; print(os.getenv('CELL_GITHUB'))" - name: Capture Docker Logs From 7a6faf3b1737c5429d75aff19ce7e0d17a486fe7 Mon Sep 17 00:00:00 2001 From: Andy Bayer Roswell <46857536+AndyBRoswell@users.noreply.github.com> Date: Thu, 29 Aug 2024 01:49:57 +0200 Subject: [PATCH 092/149] Update ci-pipeline.yml: chkenv --- .github/workflows/ci-pipeline.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index 965ab3c3..0ef8173e 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -36,7 +36,7 @@ jobs: - uses: actions/checkout@v4 - name: chkenv - run: python -c "import os; print(os.getenv('CELL_GITHUB'))" + run: python -c "import os; print(os.getenv('CELL_GITHUB')); print(os.getenv('BASE_PATH'))" - name: Build docker run: | @@ -51,7 +51,7 @@ jobs: compose-file: "${{ inputs.docker_folder }}/docker-compose.yaml" - name: chkenv2 - run: python -c "import os; print(os.getenv('CELL_GITHUB'))" + run: python -c "import os; print(os.getenv('CELL_GITHUB')); print(os.getenv('BASE_PATH'))" - name: Wait for healthy app uses: raschmitt/wait-for-healthy-container/@v1 @@ -61,7 +61,7 @@ jobs: - name: chkenv3 if: always() - run: python -c "import os; print(os.getenv('CELL_GITHUB'))" + run: python -c "import os; print(os.getenv('CELL_GITHUB')); print(os.getenv('BASE_PATH'))" - name: Capture Docker Logs if: failure() From 16cfaea54e4a3f9ec66ae7148c4eb0be4593ae89 Mon Sep 17 00:00:00 2001 From: Andy Bayer Roswell Date: Thu, 29 Aug 2024 01:55:53 +0200 Subject: [PATCH 093/149] chkenv --- vreapis/containerizer/views.py | 1 + 1 file changed, 1 insertion(+) diff --git a/vreapis/containerizer/views.py b/vreapis/containerizer/views.py index 16b861bd..437294f6 100644 --- a/vreapis/containerizer/views.py +++ b/vreapis/containerizer/views.py @@ -191,6 +191,7 @@ def post(self, request: Request): class CellsHandler(viewsets.ModelViewSet): common.logger.debug(f"CELL_GITHUB: {os.getenv('CELL_GITHUB')}") + common.logger.debug(f"BASE_PATH: {os.getenv('BASE_PATH')}") queryset: QuerySet = Cell.objects.all() serializer_class = CellSerializer authentication_classes: list[BaseAuthentication] = [StaticTokenAuthentication] From 58cf6b9934f57b032d13bff53995079212447475 Mon Sep 17 00:00:00 2001 From: Andy Bayer Roswell Date: Wed, 4 Sep 2024 12:24:12 +0200 Subject: [PATCH 094/149] chkenv --- vreapis/docker-compose.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/vreapis/docker-compose.yaml b/vreapis/docker-compose.yaml index 9420187e..71a656f0 100644 --- a/vreapis/docker-compose.yaml +++ b/vreapis/docker-compose.yaml @@ -31,6 +31,7 @@ services: - DJANGO_SUPERUSER_PASSWORD=admin - DJANGO_TOKEN=token - SECRET_KEY=secret + - CELL_GITHUB=https://github.com/QCDIS/NaaVRE-cells-test-3 ports: - '127.0.0.1:8000:8000' healthcheck: From 3c728a61f8ed4f8b3a3c73ef05a8f9c96abbde1e Mon Sep 17 00:00:00 2001 From: Andy Bayer Roswell Date: Wed, 4 Sep 2024 12:37:37 +0200 Subject: [PATCH 095/149] try to add CELL_GITHUB_TOKEN to docker-compose.yaml to read this secret given by CELL_GITHUB --- vreapis/docker-compose.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/vreapis/docker-compose.yaml b/vreapis/docker-compose.yaml index 71a656f0..c98ca438 100644 --- a/vreapis/docker-compose.yaml +++ b/vreapis/docker-compose.yaml @@ -32,6 +32,7 @@ services: - DJANGO_TOKEN=token - SECRET_KEY=secret - CELL_GITHUB=https://github.com/QCDIS/NaaVRE-cells-test-3 + - CELL_GITHUB_TOKEN=${CELL_GITHUB_TOKEN} ports: - '127.0.0.1:8000:8000' healthcheck: From 9ccfed3c059b847200a6723fef653592ac25d5f0 Mon Sep 17 00:00:00 2001 From: Andy Bayer Roswell Date: Wed, 4 Sep 2024 13:13:17 +0200 Subject: [PATCH 096/149] extra chkenv --- vreapis/docker-compose.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vreapis/docker-compose.yaml b/vreapis/docker-compose.yaml index c98ca438..c5b4730a 100644 --- a/vreapis/docker-compose.yaml +++ b/vreapis/docker-compose.yaml @@ -31,8 +31,8 @@ services: - DJANGO_SUPERUSER_PASSWORD=admin - DJANGO_TOKEN=token - SECRET_KEY=secret - - CELL_GITHUB=https://github.com/QCDIS/NaaVRE-cells-test-3 - - CELL_GITHUB_TOKEN=${CELL_GITHUB_TOKEN} +# - CELL_GITHUB=https://github.com/QCDIS/NaaVRE-cells-test-3 +# - CELL_GITHUB_TOKEN=${CELL_GITHUB_TOKEN} ports: - '127.0.0.1:8000:8000' healthcheck: From c3d6aaa4418ba85e5493bce46f2077c929f11819 Mon Sep 17 00:00:00 2001 From: Andy Bayer Roswell Date: Wed, 4 Sep 2024 13:18:47 +0200 Subject: [PATCH 097/149] revert extra chkenv --- vreapis/docker-compose.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vreapis/docker-compose.yaml b/vreapis/docker-compose.yaml index c5b4730a..c98ca438 100644 --- a/vreapis/docker-compose.yaml +++ b/vreapis/docker-compose.yaml @@ -31,8 +31,8 @@ services: - DJANGO_SUPERUSER_PASSWORD=admin - DJANGO_TOKEN=token - SECRET_KEY=secret -# - CELL_GITHUB=https://github.com/QCDIS/NaaVRE-cells-test-3 -# - CELL_GITHUB_TOKEN=${CELL_GITHUB_TOKEN} + - CELL_GITHUB=https://github.com/QCDIS/NaaVRE-cells-test-3 + - CELL_GITHUB_TOKEN=${CELL_GITHUB_TOKEN} ports: - '127.0.0.1:8000:8000' healthcheck: From febf60c3b8007267f388b826d9392dfdb8166e00 Mon Sep 17 00:00:00 2001 From: Andy Bayer Roswell Date: Wed, 4 Sep 2024 13:29:16 +0200 Subject: [PATCH 098/149] slight modification --- vreapis/docker-compose.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vreapis/docker-compose.yaml b/vreapis/docker-compose.yaml index c98ca438..510c9c2d 100644 --- a/vreapis/docker-compose.yaml +++ b/vreapis/docker-compose.yaml @@ -31,7 +31,7 @@ services: - DJANGO_SUPERUSER_PASSWORD=admin - DJANGO_TOKEN=token - SECRET_KEY=secret - - CELL_GITHUB=https://github.com/QCDIS/NaaVRE-cells-test-3 + - CELL_GITHUB=S{CELL_GITHUB} - CELL_GITHUB_TOKEN=${CELL_GITHUB_TOKEN} ports: - '127.0.0.1:8000:8000' From 81d8c633508e40219d8f92b102ecc4956a4c5720 Mon Sep 17 00:00:00 2001 From: Andy Bayer Roswell Date: Wed, 4 Sep 2024 13:35:45 +0200 Subject: [PATCH 099/149] slight modification --- vreapis/docker-compose.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/vreapis/docker-compose.yaml b/vreapis/docker-compose.yaml index 510c9c2d..7e7ed8ba 100644 --- a/vreapis/docker-compose.yaml +++ b/vreapis/docker-compose.yaml @@ -31,7 +31,8 @@ services: - DJANGO_SUPERUSER_PASSWORD=admin - DJANGO_TOKEN=token - SECRET_KEY=secret - - CELL_GITHUB=S{CELL_GITHUB} + # for GitHub action to read envvar + - CELL_GITHUB=${CELL_GITHUB} - CELL_GITHUB_TOKEN=${CELL_GITHUB_TOKEN} ports: - '127.0.0.1:8000:8000' From ef180f859250707b9d325599db8ed43e81e72992 Mon Sep 17 00:00:00 2001 From: Andy Bayer Roswell Date: Thu, 14 Nov 2024 00:47:39 +0100 Subject: [PATCH 100/149] add simple emulated frontend test for endpoint baseimagetags --- .../tests/emulated-frontend/containerizer.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/vreapis/tests/emulated-frontend/containerizer.py b/vreapis/tests/emulated-frontend/containerizer.py index f672b60c..26282a2c 100644 --- a/vreapis/tests/emulated-frontend/containerizer.py +++ b/vreapis/tests/emulated-frontend/containerizer.py @@ -28,7 +28,7 @@ session = requests.Session() -def test(endpoint: str, files: list[str]): +def test_post(endpoint: str, files: list[str]): for file in files: with open(f'{script_path}/dat/{file}') as f: body: dict[str, any] = json.load(f) @@ -40,13 +40,18 @@ def test(endpoint: str, files: list[str]): for endpoint in args.in_def: - file_pattern = re.compile(fr'{endpoint}(\..+)?.json') - files: list[str] = os.listdir(f'{script_path}/dat') - request_body_files: list[str] = [file for file in files if file_pattern.match(file)] - test(endpoint, request_body_files) + match endpoint: + case 'baseimagetags': + response = session.get(f'{API_ENDPOINT}/{endpoint}', headers=headers, verify=False) + print(response.text) + case _: + file_pattern = re.compile(fr'{endpoint}(\..+)?.json') + files: list[str] = os.listdir(f'{script_path}/dat') + request_body_files: list[str] = [file for file in files if file_pattern.match(file)] + test_post(endpoint, request_body_files) if args.in_id is not None: for test_info in args.in_id: endpoint: str = test_info[0] files: list[str] = [f'{endpoint}.{id}.json' for id in test_info[1:]] - test(endpoint, files) + test_post(endpoint, files) From ea37ff6834b690b851375ef1568d2cd71880f7bb Mon Sep 17 00:00:00 2001 From: Andy Bayer Roswell Date: Fri, 22 Nov 2024 15:58:25 +0100 Subject: [PATCH 101/149] handle JUPYTERHUB_USER in task title --- vreapis/containerizer/views.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/vreapis/containerizer/views.py b/vreapis/containerizer/views.py index 437294f6..cc2bcb34 100644 --- a/vreapis/containerizer/views.py +++ b/vreapis/containerizer/views.py @@ -147,8 +147,10 @@ def post(self, request: Request): title = source.partition('\n')[0].strip() title = slugify(title) if title and title[0] == "#" else "Untitled" - if 'JUPYTERHUB_USER' in os.environ: - title += '-' + slugify(os.environ['JUPYTERHUB_USER']) + # if 'JUPYTERHUB_USER' in os.environ: + # title += '-' + slugify(os.environ['JUPYTERHUB_USER']) + if 'JUPYTERHUB_USER' in payload: + title += '-' + slugify(payload['JUPYTERHUB_USER']) # If any of these change, we create a new cell in the catalog. This matches the cell properties saved in workflows. # cell_identity_dict = {'title': title, 'params': extractor.params, 'inputs': extractor.ins, 'outputs': extractor.outs, } From 619258d41278e48ad917326d4830cc61e6533c05 Mon Sep 17 00:00:00 2001 From: Andy Bayer Roswell Date: Sat, 30 Nov 2024 00:36:04 +0100 Subject: [PATCH 102/149] [bugfix] install R dependencies used by rpy2 during image build [bugfix] mistaken exception when node_id exists [should update the existing cell] --- vreapis/Dockerfile | 2 + vreapis/containerizer/views.py | 21 ++++++--- vreapis/services/extractor/rextractor.py | 46 +++++++++---------- .../tests/emulated-frontend/containerizer.py | 6 +-- 4 files changed, 42 insertions(+), 33 deletions(-) diff --git a/vreapis/Dockerfile b/vreapis/Dockerfile index 0ff8ad86..97770920 100644 --- a/vreapis/Dockerfile +++ b/vreapis/Dockerfile @@ -11,6 +11,8 @@ RUN /opt/venv/bin/pip install pip --upgrade COPY ./requirements.txt /app RUN /opt/venv/bin/pip install -r requirements.txt +RUN Rscript -e "install.packages(c('rlang', 'lobstr', 'purrr', 'renv'), repos = 'https://cloud.r-project.org')" + COPY . /app RUN chmod +x entrypoint.sh diff --git a/vreapis/containerizer/views.py b/vreapis/containerizer/views.py index cc2bcb34..ca36cb24 100644 --- a/vreapis/containerizer/views.py +++ b/vreapis/containerizer/views.py @@ -17,6 +17,7 @@ import jinja2 from django.db.models import QuerySet from rest_framework.authentication import BaseAuthentication +from rest_framework.exceptions import ValidationError from rest_framework.permissions import IsAuthenticated, BasePermission from rest_framework.response import Response from rest_framework.request import Request @@ -153,10 +154,10 @@ def post(self, request: Request): title += '-' + slugify(payload['JUPYTERHUB_USER']) # If any of these change, we create a new cell in the catalog. This matches the cell properties saved in workflows. - # cell_identity_dict = {'title': title, 'params': extractor.params, 'inputs': extractor.ins, 'outputs': extractor.outs, } - # cell_identity_str = json.dumps(cell_identity_dict, sort_keys=True) - # node_id = hashlib.sha1(cell_identity_str.encode()).hexdigest()[:7] - node_id = str(time.time_ns())[len('0x'):] + cell_identity_dict = {'title': title, 'params': extractor.params, 'inputs': extractor.ins, 'outputs': extractor.outs, } + cell_identity_str = json.dumps(cell_identity_dict, sort_keys=True) + node_id = hashlib.sha1(cell_identity_str.encode()).hexdigest()[:7] + # node_id = str(time.time_ns())[len('0x'):] cell = Cell( node_id=node_id, @@ -452,7 +453,7 @@ def create(self, request: Request, *args, **kwargs): except Exception as ex: return return_error('Error setting cell', ex) - common.logger.debug('current_cell: ' + current_cell.toJSON()) + # common.logger.debug('current_cell: ' + current_cell.toJSON()) all_vars = current_cell.params + current_cell.inputs + current_cell.outputs for param_name in all_vars: if param_name not in current_cell.types: @@ -462,8 +463,14 @@ def create(self, request: Request, *args, **kwargs): return return_error(f'{current_cell.task_name} has not selected base image') try: serializer: CellSerializer = self.get_serializer(data=request.data) - serializer.is_valid(raise_exception=True) - instance, created = Cell.objects.update_or_create(node_id=serializer.validated_data['node_id'], defaults=serializer.validated_data) + try: + serializer.is_valid(raise_exception=True) + except ValidationError as ex: + if 'node_id' in ex.detail and str(ex.detail) == 'cell with this node id already exists.': + Cell.objects.update(**serializer.validated_data) + else: + Cell.objects.create(**serializer.validated_data) + # instance, created = Cell.objects.update_or_create(node_id=serializer.validated_data['node_id'], defaults=serializer.validated_data) except Exception as ex: return return_error('Error adding or updating cell in catalog', ex) diff --git a/vreapis/services/extractor/rextractor.py b/vreapis/services/extractor/rextractor.py index 8a2a19a7..7ea963c1 100644 --- a/vreapis/services/extractor/rextractor.py +++ b/vreapis/services/extractor/rextractor.py @@ -21,29 +21,29 @@ # robjects.conversion.set_conversion(robject_converter) # install R packages -robjects.r(''' -install_package_with_retry <- function(package_name, max_attempts = 5) { - for(i in 1:max_attempts) { - print(paste("Attempt", i, "to install", package_name)) - tryCatch({ - install.packages(package_name, quiet = TRUE) - print(paste(package_name, "installed successfully.")) - return(TRUE) - }, warning = function(w) { - print(paste("Warning while installing", package_name, ":", w)) - Sys.sleep(2) - }, error = function(e) { - print(paste("Failed to install", package_name, ":", e)) - Sys.sleep(2) - }) - } - return(FALSE) -} -''') -packnames = ('rlang', 'lobstr', 'purrr', 'renv',) -for p in packnames: - if not rpackages.isinstalled(p): - robjects.r(f'install_package_with_retry("{p}")') +# robjects.r(''' +# install_package_with_retry <- function(package_name, max_attempts = 5) { +# for(i in 1:max_attempts) { +# print(paste("Attempt", i, "to install", package_name)) +# tryCatch({ +# install.packages(package_name, quiet = TRUE) +# print(paste(package_name, "installed successfully.")) +# return(TRUE) +# }, warning = function(w) { +# print(paste("Warning while installing", package_name, ":", w)) +# Sys.sleep(2) +# }, error = function(e) { +# print(paste("Failed to install", package_name, ":", e)) +# Sys.sleep(2) +# }) +# } +# return(FALSE) +# } +# ''') +# packnames = ('rlang', 'lobstr', 'purrr', 'renv',) +# for p in packnames: +# if not rpackages.isinstalled(p): +# robjects.r(f'install_package_with_retry("{p}")') # This R code is used to obtain all assignment variables (source https://adv-r.hadley.nz/expressions.html) r_env["result"] = robjects.r(""" diff --git a/vreapis/tests/emulated-frontend/containerizer.py b/vreapis/tests/emulated-frontend/containerizer.py index 26282a2c..f8489da2 100644 --- a/vreapis/tests/emulated-frontend/containerizer.py +++ b/vreapis/tests/emulated-frontend/containerizer.py @@ -32,9 +32,9 @@ def test_post(endpoint: str, files: list[str]): for file in files: with open(f'{script_path}/dat/{file}') as f: body: dict[str, any] = json.load(f) - match endpoint: - case 'addcell': - body['node_id'] = str(hex(time.time_ns())[len('0x'):]) + # match endpoint: + # case 'addcell': + # body['node_id'] = str(hex(time.time_ns())[len('0x'):]) # use a unique node_id response = session.post(f'{API_ENDPOINT}/{endpoint}', json.dumps(body), headers=headers, verify=False) print(response.text) From 4162e64826b8f25e79f7d895466ea18089d260b9 Mon Sep 17 00:00:00 2001 From: Andy Bayer Roswell Date: Sun, 1 Dec 2024 13:24:39 +0100 Subject: [PATCH 103/149] trying to fix action failures caused by non-existing r pkgs --- vreapis/containerizer/RContainerizer.py | 3 ++- vreapis/templates/conda_env_template.jinja2 | 3 +++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/vreapis/containerizer/RContainerizer.py b/vreapis/containerizer/RContainerizer.py index 39287183..1a6daec8 100644 --- a/vreapis/containerizer/RContainerizer.py +++ b/vreapis/containerizer/RContainerizer.py @@ -49,7 +49,8 @@ def get_files_info(cell=None, cells_path=None): @staticmethod def map_dependencies(dependencies, module_name_mapping): - dependencies = map(lambda x: 'r-' + x['name'], dependencies) + # dependencies = map(lambda x: 'r-' + x['name'], dependencies) + dependencies = map(lambda x: x['name'], dependencies) dependencies = map(lambda x: module_name_mapping.get('r', {}).get(x, x), dependencies) set_conda_deps = set(dependencies) set_pip_deps = set() diff --git a/vreapis/templates/conda_env_template.jinja2 b/vreapis/templates/conda_env_template.jinja2 index 6506adaf..9c3fdcd8 100644 --- a/vreapis/templates/conda_env_template.jinja2 +++ b/vreapis/templates/conda_env_template.jinja2 @@ -1,6 +1,9 @@ name: venv channels: - conda-forge + - r + - bioconda + - defaults dependencies: - pip - python>=3.8 From 956801ad40fe5925e7a1a30209788c46e1763bf6 Mon Sep 17 00:00:00 2001 From: Andy Bayer Roswell Date: Sun, 1 Dec 2024 13:53:48 +0100 Subject: [PATCH 104/149] trying to fix action failures caused by non-existing r pkgs --- vreapis/containerizer/RContainerizer.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/vreapis/containerizer/RContainerizer.py b/vreapis/containerizer/RContainerizer.py index 1a6daec8..f517230e 100644 --- a/vreapis/containerizer/RContainerizer.py +++ b/vreapis/containerizer/RContainerizer.py @@ -49,8 +49,7 @@ def get_files_info(cell=None, cells_path=None): @staticmethod def map_dependencies(dependencies, module_name_mapping): - # dependencies = map(lambda x: 'r-' + x['name'], dependencies) - dependencies = map(lambda x: x['name'], dependencies) + dependencies = map(lambda x: 'r-' + x['name'], dependencies) # prefix `r-` is naming convention dependencies = map(lambda x: module_name_mapping.get('r', {}).get(x, x), dependencies) set_conda_deps = set(dependencies) set_pip_deps = set() From 66d570f8c6a2651df6804cbd53d9ade79d6abc1c Mon Sep 17 00:00:00 2001 From: Andy Bayer Roswell Date: Tue, 3 Dec 2024 14:21:24 +0100 Subject: [PATCH 105/149] fixed type inference for py cells --- vreapis/Dockerfile | 3 +- vreapis/containerizer/views.py | 11 +- vreapis/requirements.txt | 18 +- vreapis/services/extractor/pyextractor.py | 3 + .../services/extractor/pyheaderextractor.py | 245 +++++++++++++++++ .../services/extractor/rheaderextractor.py | 259 ++++++++++++++++++ 6 files changed, 527 insertions(+), 12 deletions(-) create mode 100644 vreapis/services/extractor/pyheaderextractor.py create mode 100644 vreapis/services/extractor/rheaderextractor.py diff --git a/vreapis/Dockerfile b/vreapis/Dockerfile index 97770920..9cc4a7d7 100644 --- a/vreapis/Dockerfile +++ b/vreapis/Dockerfile @@ -1,4 +1,5 @@ -FROM python:3.12.2-slim +# FROM python:3.12.2-slim +FROM python:3.11.9-slim WORKDIR /app diff --git a/vreapis/containerizer/views.py b/vreapis/containerizer/views.py index ca36cb24..7610a6a7 100644 --- a/vreapis/containerizer/views.py +++ b/vreapis/containerizer/views.py @@ -37,7 +37,8 @@ from containerizer.RContainerizer import RContainerizer from db.repository import Repository from services.extractor.extractor import DummyExtractor -from services.extractor.headerextractor import HeaderExtractor +from services.extractor.pyheaderextractor import PyHeaderExtractor +from services.extractor.rheaderextractor import RHeaderExtractor from services.extractor.pyextractor import PyExtractor from services.extractor.rextractor import RExtractor from services.converter import ConverterReactFlowChart @@ -126,7 +127,11 @@ def post(self, request: Request): else: # extractor based on the cell header try: - extractor = HeaderExtractor(notebook, source) + # extractor = HeaderExtractor(notebook, source) + if 'python' in kernel.lower(): + extractor = PyHeaderExtractor(notebook, source) + elif 'r' in kernel.lower(): + extractor = RHeaderExtractor(notebook, source) except jsonschema.ValidationError as e: return return_error('Error in cell header', e, stat=status.HTTP_400_BAD_REQUEST) @@ -173,6 +178,8 @@ def post(self, request: Request): kernel=kernel, notebook_dict=extracted_nb.dict() ) + cell.add_inputs(extractor.ins) + cell.add_outputs(extractor.outs) cell.integrate_configuration() extractor.params = extractor.extract_cell_params(cell.original_source) cell.add_params(extractor.params) diff --git a/vreapis/requirements.txt b/vreapis/requirements.txt index 848d4334..b4b813bd 100644 --- a/vreapis/requirements.txt +++ b/vreapis/requirements.txt @@ -13,13 +13,13 @@ psycopg2==2.9.9 django-probes==1.7.0 pyjwt==2.8.0 cryptography==42.0.7 -nbformat -python-slugify -colorhash +nbformat==5.10.4 +python-slugify==8.0.4 +colorhash==2.0.0 rpy2==3.5.11 -pyflakes -pytype -autopep8 -distro -pygithub -jupytext +pyflakes==3.2.0 +pytype==2024.4.11 +autopep8==2.2.0 +distro==1.9.0 +pygithub==2.3.0 +jupytext==1.16.3 diff --git a/vreapis/services/extractor/pyextractor.py b/vreapis/services/extractor/pyextractor.py index e374c4a3..93bc408d 100644 --- a/vreapis/services/extractor/pyextractor.py +++ b/vreapis/services/extractor/pyextractor.py @@ -151,7 +151,9 @@ def __convert_type_annotation(self, type_annotation): :param type_annotation: type annotation obtained by e.g. pytype :return: converted type: 'int', 'float', 'str', 'list', or None """ + # logging.getLogger(__name__).debug(f'type_annotation = {type_annotation}') if type_annotation is None: + # logging.getLogger(__name__).debug(f'type_annotation is None') return None patterns = { @@ -176,6 +178,7 @@ def __convert_type_annotation(self, type_annotation): for type_name, regs in patterns.items(): for reg in regs: if reg.match(type_annotation): + # logging.getLogger(__name__).debug(f'type_name = {type_name}') return type_name logging.getLogger(__name__).debug(f'Unmatched type: {type_annotation}') diff --git a/vreapis/services/extractor/pyheaderextractor.py b/vreapis/services/extractor/pyheaderextractor.py new file mode 100644 index 00000000..dc7e2a56 --- /dev/null +++ b/vreapis/services/extractor/pyheaderextractor.py @@ -0,0 +1,245 @@ +import json +import logging +import os +import re +from typing import Literal, Union + +import jsonschema +import yaml + +from .extractor import Extractor + + +class PyHeaderExtractor(Extractor): + """ Extracts cells using information defined by the user in its header + + Cells should contain a comment with a yaml block defining inputs, outputs, + params and confs. Eg: + + # My cell + # --- + # NaaVRE: + # cell: + # inputs: + # - my_input: + # type: String + # - my_other_input: + # type: Integer + # outputs: + # - my_output: + # type: List + # params: + # - param_something: + # type: String + # default_value: "my default value" + # confs: + # - conf_something_else: + # assignation: "conf_something_else = 'my other value'" + # ... + [cell code] + + The document is validated with the schema `cell_header.schema.json` + + """ + ins: Union[dict, None] + outs: Union[dict, None] + params: Union[dict, None] + confs: Union[list, None] + dependencies: Union[list, None] + + def __init__(self, notebook, cell_source): + self.re_yaml_doc_in_comment = re.compile( + (r"^(?:.*\n)*" + r"\s*#\s*---\s*\n" + r"((?:\s*#.*\n)+?)" + r"\s*#\s*\.\.\.\s*\n" + ), + re.MULTILINE) + self.schema = self._load_schema() + self.cell_header = self._extract_header(cell_source) + self._external_extract_cell_params = None + + super().__init__(notebook, cell_source) + + @staticmethod + def _load_schema(): + filename = os.path.join( + os.path.dirname(__file__), + 'cell_header.schema.json') + with open(filename) as f: + schema = json.load(f) + return schema + + def enabled(self): + return self.cell_header is not None + + def is_complete(self): + return ( + (self.ins is not None) + and (self.outs is not None) + and (self.params is not None) + and (self.confs is not None) + and (self.dependencies is not None) + ) + + def _extract_header(self, cell_source): + # get yaml document from cell comments + m = self.re_yaml_doc_in_comment.match(cell_source) + if not (m and m.groups()): + return None + yaml_doc = m.group(1) + # remove comment symbol + yaml_doc = '\n'.join([ + line.lstrip().lstrip('#') + for line in yaml_doc.splitlines() + ]) + # parse yaml + header = yaml.safe_load(yaml_doc) + # validate schema + try: + jsonschema.validate(header, self.schema) + except jsonschema.ValidationError as e: + logging.getLogger().debug(f"Cell header validation error: {e}") + raise e + return header + + def add_missing_values(self, extractor: Extractor): + """ Add values not specified in the header from another extractor + (e.g. PyExtractor or RExtractor) + """ + if self.ins is None: + self.ins = extractor.ins + if self.outs is None: + self.outs = extractor.outs + if self.params is None: + self.params = extractor.params + # We store a reference to extractor.extract_cell_params because + # self.extract_cell_params is called after self.add_missing_values + # in component_containerizer.handlers.ExtractorHandler.post() + self._external_extract_cell_params = extractor.extract_cell_params + if self.confs is None: + self.confs = extractor.confs + if self.dependencies is None: + self.dependencies = extractor.dependencies + + @staticmethod + def _parse_inputs_outputs_param_items( + item: Union[str, dict], + item_type: Literal['inputs', 'outputs', 'params'], + ) -> dict: + """ Parse inputs, outputs, or params items from the header + + They can have either format + - ElementVarName: 'my_name' + - ElementVarNameType {'my_name': 'my_type'} + - IOElementVarDict {'my_name': {'type': 'my_type'}} + or ParamElementVarDict {'my_name': {'type': 'my_type', + 'default_value': 'my_value'}} + + Returns + - if item_type is 'inputs' or 'outputs': + {'name': 'my_name', 'type': 'my_type'} + - if item_type is 'params': + {'name': 'my_name', 'type': 'my_type', 'value': 'my_value'} + """ + var_dict = {} + + # ElementVarName + if isinstance(item, str): + var_dict = { + 'name': item, + 'type': None, + 'value': None, + } + elif isinstance(item, dict): + if len(item.keys()) != 1: + # this should have been caught by the schema validation + raise ValueError(f"Unexpected item in {item_type}: {item}") + var_name = list(item.keys())[0] + var_props = item[var_name] + # ElementVarNameType + if isinstance(var_props, str): + var_dict = { + 'name': var_name, + 'type': var_props, + 'value': None, + } + # IOElementVarDict or ParamElementVarDict + elif isinstance(var_props, dict): + var_dict = { + 'name': var_name, + 'type': var_props.get('type'), + 'value': var_props.get('default_value'), + } + + # Convert types + types_conversion = { + 'Integer': 'int', + 'Float': 'float', + 'String': 'str', + 'List': 'list', + None: None, + } + var_dict['type'] = types_conversion[var_dict['type']] + + # 'value' should only be kept for params + if item_type not in ['params']: + del var_dict['value'] + + return var_dict + + def _infer_cell_inputs_outputs_params( + self, + header: Union[dict, None], + item_type: Literal['inputs', 'outputs', 'params'], + ) -> Union[dict, None]: + if header is None: + return None + items = header['NaaVRE']['cell'].get(item_type) + if items is None: + return None + items = [self._parse_inputs_outputs_param_items(it, item_type) + for it in items] + return {it['name']: it for it in items} + + def infer_cell_inputs(self): + return self._infer_cell_inputs_outputs_params( + self.cell_header, + 'inputs', + ) + + def infer_cell_outputs(self): + return self._infer_cell_inputs_outputs_params( + self.cell_header, + 'outputs', + ) + + def extract_cell_params(self, source): + if self._external_extract_cell_params is not None: + return self._external_extract_cell_params(source) + return self._infer_cell_inputs_outputs_params( + self._extract_header(source), + 'params', + ) + + def extract_cell_conf_ref(self): + if self.cell_header is None: + return None + items = self.cell_header['NaaVRE']['cell'].get('confs') + if items is None: + return None + return {k: v['assignation'] for it in items for k, v in it.items()} + + def infer_cell_dependencies(self, confs): + if self.cell_header is None: + return None + items = self.cell_header['NaaVRE']['cell'].get('dependencies') + if items is None: + return None + return [ + { + 'name': it.get('name'), + 'asname': it.get('asname', None), + 'module': it.get('module', ''), + } + for it in items] diff --git a/vreapis/services/extractor/rheaderextractor.py b/vreapis/services/extractor/rheaderextractor.py new file mode 100644 index 00000000..113cfb19 --- /dev/null +++ b/vreapis/services/extractor/rheaderextractor.py @@ -0,0 +1,259 @@ +import json +import logging +import os +import re +from typing import Literal, Union + +import jsonschema +import yaml + +from .extractor import Extractor + + +class RHeaderExtractor(Extractor): + """ Extracts cells using information defined by the user in its header + + Cells should contain a comment with a yaml block defining inputs, outputs, + params and confs. Eg: + + # My cell + # --- + # NaaVRE: + # cell: + # inputs: + # - my_input: + # type: String + # - my_other_input: + # type: Integer + # outputs: + # - my_output: + # type: List + # params: + # - param_something: + # type: String + # default_value: "my default value" + # confs: + # - conf_something_else: + # assignation: "conf_something_else = 'my other value'" + # ... + [cell code] + + The document is validated with the schema `cell_header.schema.json` + + """ + ins: Union[dict, None] + outs: Union[dict, None] + params: Union[dict, None] + confs: Union[list, None] + dependencies: Union[list, None] + + def __init__(self, notebook, cell_source): + self.re_yaml_doc_in_comment = re.compile( + (r"^(?:.*\n)*" + r"\s*#\s*---\s*\n" + r"((?:\s*#.*\n)+?)" + r"\s*#\s*\.\.\.\s*\n" + ), + re.MULTILINE) + self.schema = self._load_schema() + self.cell_header = self._extract_header(cell_source) + self._external_extract_cell_params = None + + super().__init__(notebook, cell_source) + + @staticmethod + def _load_schema(): + filename = os.path.join( + os.path.dirname(__file__), + 'cell_header.schema.json') + with open(filename) as f: + schema = json.load(f) + return schema + + def enabled(self): + return self.cell_header is not None + + def is_complete(self): + return ( + (self.ins is not None) + and (self.outs is not None) + and (self.params is not None) + and (self.confs is not None) + and (self.dependencies is not None) + ) + + def _extract_header(self, cell_source): + # get yaml document from cell comments + m = self.re_yaml_doc_in_comment.match(cell_source) + if not (m and m.groups()): + return None + yaml_doc = m.group(1) + # remove comment symbol + yaml_doc = '\n'.join([ + line.lstrip().lstrip('#') + for line in yaml_doc.splitlines() + ]) + # parse yaml + header = yaml.safe_load(yaml_doc) + # validate schema + try: + jsonschema.validate(header, self.schema) + except jsonschema.ValidationError as e: + logging.getLogger().debug(f"Cell header validation error: {e}") + raise e + return header + + def add_missing_values(self, extractor: Extractor): + """ Add values not specified in the header from another extractor + (e.g. PyExtractor or RExtractor) + """ + if self.ins is None: + self.ins = extractor.ins + if self.outs is None: + self.outs = extractor.outs + if self.params is None: + self.params = extractor.params + # We store a reference to extractor.extract_cell_params because + # self.extract_cell_params is called after self.add_missing_values + # in component_containerizer.handlers.ExtractorHandler.post() + self._external_extract_cell_params = extractor.extract_cell_params + if self.confs is None: + self.confs = extractor.confs + if self.dependencies is None: + self.dependencies = extractor.dependencies + + @staticmethod + def _parse_inputs_outputs_param_items( + item: Union[str, dict], + item_type: Literal['inputs', 'outputs', 'params'], + ) -> dict: + """ Parse inputs, outputs, or params items from the header + + They can have either format + - ElementVarName: 'my_name' + - ElementVarNameType {'my_name': 'my_type'} + - IOElementVarDict {'my_name': {'type': 'my_type'}} + or ParamElementVarDict {'my_name': {'type': 'my_type', + 'default_value': 'my_value'}} + + Returns + - if item_type is 'inputs' or 'outputs': + {'name': 'my_name', 'type': 'my_type'} + - if item_type is 'params': + {'name': 'my_name', 'type': 'my_type', 'value': 'my_value'} + """ + var_dict = {} + + # ElementVarName + if isinstance(item, str): + var_dict = { + 'name': item, + 'type': None, + 'value': None, + } + elif isinstance(item, dict): + if len(item.keys()) != 1: + # this should have been caught by the schema validation + raise ValueError(f"Unexpected item in {item_type}: {item}") + var_name = list(item.keys())[0] + var_props = item[var_name] + # ElementVarNameType + if isinstance(var_props, str): + var_dict = { + 'name': var_name, + 'type': var_props, + 'value': None, + } + # IOElementVarDict or ParamElementVarDict + elif isinstance(var_props, dict): + var_type = var_props.get('type') + default_value = var_props.get('default_value') + if var_type == 'List': + # Convert list to string representation + default_value = json.dumps(default_value) + var_dict = { + 'name': var_name, + 'type': var_type, + 'value': default_value, + } + + # Convert types + types_conversion = { + 'Integer': 'int', + 'Float': 'float', + 'String': 'str', + 'List': 'list', + None: None, + } + var_dict['type'] = types_conversion[var_dict['type']] + + # 'value' should only be kept for params + if item_type not in ['params']: + del var_dict['value'] + return var_dict + + def _infer_cell_inputs_outputs_params( + self, + header: Union[dict, None], + item_type: Literal['inputs', 'outputs', 'params'], + ) -> Union[dict, None]: + if header is None: + return None + items = header['NaaVRE']['cell'].get(item_type) + if items is None: + return None + items = [self._parse_inputs_outputs_param_items(it, item_type) + for it in items] + inputs_outputs_params = {it['name']: it for it in items} + return inputs_outputs_params + + def infer_cell_inputs(self): + return self._infer_cell_inputs_outputs_params( + self.cell_header, + 'inputs', + ) + + def infer_cell_outputs(self): + return self._infer_cell_inputs_outputs_params( + self.cell_header, + 'outputs', + ) + + def extract_cell_params(self, source): + if self._external_extract_cell_params is not None: + return self._external_extract_cell_params(source) + return self._infer_cell_inputs_outputs_params( + self._extract_header(source), + 'params', + ) + + def extract_cell_conf_ref(self): + if self.cell_header is None: + return None + items = self.cell_header['NaaVRE']['cell'].get('confs') + if items is None: + return None + for item in items: + for k, v in item.items(): + if 'assignation' in v: + assignation = v.get('assignation') + if '[' in assignation and ']' in assignation: + # Replace to R list format + assignation = assignation.replace('[', 'list(').replace(']', ')') + item[k]['assignation'] = assignation + cell_conf = {k: v['assignation'] for it in items for k, v in it.items()} + return cell_conf + + def infer_cell_dependencies(self, confs): + if self.cell_header is None: + return None + items = self.cell_header['NaaVRE']['cell'].get('dependencies') + if items is None: + return None + return [ + { + 'name': it.get('name'), + 'asname': it.get('asname', None), + 'module': it.get('module', ''), + } + for it in items] From ee76a6692d509441383be3bcd43c599016da1ace Mon Sep 17 00:00:00 2001 From: Andy Bayer Roswell Date: Wed, 4 Dec 2024 19:55:23 +0100 Subject: [PATCH 106/149] adding support for export in legacy db schema [for the old tinydb] --- .../commands/export_in_legacy_db_schema.py | 19 +++++++++++++++++++ vreapis/manage.py | 4 ++-- vreapis/requirements.txt | 1 + 3 files changed, 22 insertions(+), 2 deletions(-) create mode 100644 vreapis/catalog/management/commands/export_in_legacy_db_schema.py diff --git a/vreapis/catalog/management/commands/export_in_legacy_db_schema.py b/vreapis/catalog/management/commands/export_in_legacy_db_schema.py new file mode 100644 index 00000000..ccd6013b --- /dev/null +++ b/vreapis/catalog/management/commands/export_in_legacy_db_schema.py @@ -0,0 +1,19 @@ +from django.core.management import CommandParser +from django.core.management.base import BaseCommand, CommandError +from catalog.models import Cell + +class Command(BaseCommand): + help = 'Export designated tables in legacy db schema' + + + def add_arguments(self, parser: CommandParser): + parser.add_argument('tables', type=str, nargs='+', help='Tables to export') + + def handle(self, *args, **options): + for table in options['tables']: + match table: + case 'Cell': + queryset = Cell.objects.all() + print(queryset) + case _: + raise CommandError(f"Table {table} is not supported") diff --git a/vreapis/manage.py b/vreapis/manage.py index 88ef4fb5..4b3bc637 100755 --- a/vreapis/manage.py +++ b/vreapis/manage.py @@ -2,7 +2,7 @@ """Django's command-line utility for administrative tasks.""" import os import sys -import dotenv +# import dotenv # always complains `No module named 'dotenv'` I don't know why def main(): @@ -20,5 +20,5 @@ def main(): if __name__ == '__main__': - dotenv.read_dotenv() + # dotenv.read_dotenv() main() diff --git a/vreapis/requirements.txt b/vreapis/requirements.txt index b4b813bd..66fd927a 100644 --- a/vreapis/requirements.txt +++ b/vreapis/requirements.txt @@ -7,6 +7,7 @@ gunicorn==21.2.0 pyyaml==6.0.1 requests==2.31.0 django-dotenv==1.4.2 +# python-dotenv==1.0.1 django-cors-headers==4.3.1 django-extensions==3.2.3 psycopg2==2.9.9 From b373e4a01c36491774a17c929273d2425be4a247 Mon Sep 17 00:00:00 2001 From: Andy Bayer Roswell Date: Wed, 4 Dec 2024 20:01:55 +0100 Subject: [PATCH 107/149] adding support for export in legacy db schema [for the old tinydb] --- .../management/commands/export_in_legacy_db_schema.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/vreapis/catalog/management/commands/export_in_legacy_db_schema.py b/vreapis/catalog/management/commands/export_in_legacy_db_schema.py index ccd6013b..67be8c2a 100644 --- a/vreapis/catalog/management/commands/export_in_legacy_db_schema.py +++ b/vreapis/catalog/management/commands/export_in_legacy_db_schema.py @@ -1,6 +1,8 @@ from django.core.management import CommandParser from django.core.management.base import BaseCommand, CommandError from catalog.models import Cell +from catalog.serializers import CellSerializer + class Command(BaseCommand): help = 'Export designated tables in legacy db schema' @@ -14,6 +16,8 @@ def handle(self, *args, **options): match table: case 'Cell': queryset = Cell.objects.all() - print(queryset) + for cell in queryset: + serializer = CellSerializer(cell) + print(serializer.data) case _: raise CommandError(f"Table {table} is not supported") From 4ab0e594188b962edd09a97371988edea9a2768e Mon Sep 17 00:00:00 2001 From: Andy Bayer Roswell Date: Thu, 5 Dec 2024 01:41:41 +0100 Subject: [PATCH 108/149] adding support for export in legacy db schema [for the old tinydb] --- .../commands/export_in_legacy_db_schema.py | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/vreapis/catalog/management/commands/export_in_legacy_db_schema.py b/vreapis/catalog/management/commands/export_in_legacy_db_schema.py index 67be8c2a..885b4fea 100644 --- a/vreapis/catalog/management/commands/export_in_legacy_db_schema.py +++ b/vreapis/catalog/management/commands/export_in_legacy_db_schema.py @@ -1,11 +1,16 @@ +import json +import os.path + from django.core.management import CommandParser from django.core.management.base import BaseCommand, CommandError +from django.db.models import QuerySet + from catalog.models import Cell from catalog.serializers import CellSerializer class Command(BaseCommand): - help = 'Export designated tables in legacy db schema' + help: str = 'Export designated tables in legacy db schema' def add_arguments(self, parser: CommandParser): @@ -15,9 +20,17 @@ def handle(self, *args, **options): for table in options['tables']: match table: case 'Cell': - queryset = Cell.objects.all() + queryset: QuerySet = Cell.objects.all() + no: int = 1 + cells: dict[str, dict] = {} for cell in queryset: serializer = CellSerializer(cell) - print(serializer.data) + cells[str(no)] = serializer.data + db_file: str = os.path.expanduser('~/NaaVRE/NaaVRE_db.json') + with open(db_file) as f: + db = json.load(f) + db['cells'] = cells + with open(db_file, 'w') as f: + json.dump(db, f) case _: raise CommandError(f"Table {table} is not supported") From dfa58345cc082d437913dea49c4e5e353682cf78 Mon Sep 17 00:00:00 2001 From: Andy Bayer Roswell Date: Thu, 5 Dec 2024 01:46:58 +0100 Subject: [PATCH 109/149] adding support for export in legacy db schema [for the old tinydb] --- .../catalog/management/commands/export_in_legacy_db_schema.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vreapis/catalog/management/commands/export_in_legacy_db_schema.py b/vreapis/catalog/management/commands/export_in_legacy_db_schema.py index 885b4fea..63b7b529 100644 --- a/vreapis/catalog/management/commands/export_in_legacy_db_schema.py +++ b/vreapis/catalog/management/commands/export_in_legacy_db_schema.py @@ -1,10 +1,10 @@ import json -import os.path from django.core.management import CommandParser from django.core.management.base import BaseCommand, CommandError from django.db.models import QuerySet +import common from catalog.models import Cell from catalog.serializers import CellSerializer @@ -26,7 +26,7 @@ def handle(self, *args, **options): for cell in queryset: serializer = CellSerializer(cell) cells[str(no)] = serializer.data - db_file: str = os.path.expanduser('~/NaaVRE/NaaVRE_db.json') + db_file: str = f'{common.project_root}/NaaVRE_db.json' with open(db_file) as f: db = json.load(f) db['cells'] = cells From 7c4c53db0baa958753069d9b5a8897d97c850108 Mon Sep 17 00:00:00 2001 From: Andy Bayer Roswell Date: Fri, 6 Dec 2024 01:50:46 +0100 Subject: [PATCH 110/149] fixed authn token issue --- vreapis/auth/simple.py | 2 +- vreapis/containerizer/views.py | 8 ++++---- vreapis/vreapis/settings/base.py | 22 +++++++++++----------- 3 files changed, 16 insertions(+), 16 deletions(-) diff --git a/vreapis/auth/simple.py b/vreapis/auth/simple.py index 5cc7ce6b..6684bf03 100644 --- a/vreapis/auth/simple.py +++ b/vreapis/auth/simple.py @@ -20,5 +20,5 @@ def authenticate(self, request: HttpRequest): access_token: str = request.headers.get('Authorization', '') print(access_token == f'Token {settings.NAAVRE_API_TOKEN}') if access_token != f'Token {settings.NAAVRE_API_TOKEN}': - raise AuthenticationFailed(f'Invalid token') + raise AuthenticationFailed(f'Invalid NaaVRE API token') return StaticTokenAuthentication.dummy_user, None diff --git a/vreapis/containerizer/views.py b/vreapis/containerizer/views.py index 7610a6a7..742022fe 100644 --- a/vreapis/containerizer/views.py +++ b/vreapis/containerizer/views.py @@ -43,7 +43,7 @@ from services.extractor.rextractor import RExtractor from services.converter import ConverterReactFlowChart import utils.cors -from auth.simple import StaticTokenAuthentication +# from auth.simple import StaticTokenAuthentication from catalog.models import Cell import common @@ -55,7 +55,7 @@ def return_error(err_msg: str = 'Unknown ERROR', e: Optional[Exception] = None, @api_view(['GET']) -@authentication_classes([StaticTokenAuthentication]) +# @authentication_classes([StaticTokenAuthentication]) @permission_classes([IsAuthenticated]) def get_base_images(request): url: str = os.getenv('BASE_IMAGE_TAGS_URL', 'https://github.com/QCDIS/NaaVRE-flavors/releases/latest/download/base_image_tags.json') @@ -70,7 +70,7 @@ def get_base_images(request): class ExtractorHandler(APIView): - authentication_classes = [StaticTokenAuthentication] + # authentication_classes = [StaticTokenAuthentication] permission_classes = [IsAuthenticated] def extract_cell_by_index(self, notebook: nbformat.notebooknode, cell_index: int) -> nbformat.NotebookNode: @@ -204,7 +204,7 @@ class CellsHandler(viewsets.ModelViewSet): common.logger.debug(f"BASE_PATH: {os.getenv('BASE_PATH')}") queryset: QuerySet = Cell.objects.all() serializer_class = CellSerializer - authentication_classes: list[BaseAuthentication] = [StaticTokenAuthentication] + # authentication_classes: list[BaseAuthentication] = [StaticTokenAuthentication] permission_classes: list[BasePermission] = [IsAuthenticated] cells_path: str = os.path.join(str(Path.home()), 'NaaVRE', 'cells') github_url_repos: str = 'https://api.github.com/repos' diff --git a/vreapis/vreapis/settings/base.py b/vreapis/vreapis/settings/base.py index 7299ea7a..b97bdd9b 100644 --- a/vreapis/vreapis/settings/base.py +++ b/vreapis/vreapis/settings/base.py @@ -22,8 +22,8 @@ ALLOWED_HOSTS = ['*'] -# CSRF_TRUSTED_ORIGINS = [os.getenv('TRUSTED_ORIGINS')] -CSRF_TRUSTED_ORIGINS = os.getenv('TRUSTED_ORIGINS').split(sep=';') +CSRF_TRUSTED_ORIGINS = [os.getenv('TRUSTED_ORIGINS')] +# CSRF_TRUSTED_ORIGINS = os.getenv('TRUSTED_ORIGINS').split(sep=';') CORS_ALLOWED_ORIGIN_REGEXES = [ r'^http:\/\/localhost:\d+$', @@ -190,14 +190,14 @@ # CUSTOM VARS # todo. delete default values -API_ENDPOINT: str = os.getenv('API_ENDPOINT', "https://naavre-dev.minikube.test/vre-api-test") - -KEYCLOAK_URL: str = os.getenv('KEYCLOAK_URL', 'https://naavre-dev.minikube.test/auth') -KEYCLOAK_REALM: str = os.getenv('KEYCLOAK_REALM', 'vre') -KEYCLOAK_LOGIN_URL: str = f'{KEYCLOAK_URL}/realms/{KEYCLOAK_REALM}/protocol/openid-connect/token' -KEYCLOAK_VERIF_URL: str = f'{KEYCLOAK_URL}/realms/{KEYCLOAK_REALM}/protocol/openid-connect/certs' -KEYCLOAK_CLIENT_ID: str = os.getenv('KEYCLOAK_CLIENT_ID', 'myclient') - -NAAVRE_API_TOKEN: str = os.getenv('NAAVRE_API_TOKEN') +# API_ENDPOINT: str = os.getenv('API_ENDPOINT', "https://naavre-dev.minikube.test/vre-api-test") +# +# KEYCLOAK_URL: str = os.getenv('KEYCLOAK_URL', 'https://naavre-dev.minikube.test/auth') +# KEYCLOAK_REALM: str = os.getenv('KEYCLOAK_REALM', 'vre') +# KEYCLOAK_LOGIN_URL: str = f'{KEYCLOAK_URL}/realms/{KEYCLOAK_REALM}/protocol/openid-connect/token' +# KEYCLOAK_VERIF_URL: str = f'{KEYCLOAK_URL}/realms/{KEYCLOAK_REALM}/protocol/openid-connect/certs' +# KEYCLOAK_CLIENT_ID: str = os.getenv('KEYCLOAK_CLIENT_ID', 'myclient') +# +# NAAVRE_API_TOKEN: str = os.getenv('NAAVRE_API_TOKEN') VENV_ACTIVATOR: str = os.getenv('VENV_ACTIVATOR', '/opt/venv/bin/activate') From 52e2751406fe1b9b015f9c02092c9e3002f2c79c Mon Sep 17 00:00:00 2001 From: Andy Bayer Roswell Date: Fri, 6 Dec 2024 02:20:36 +0100 Subject: [PATCH 111/149] .dockerignore --- vreapis/.dockerignore | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 vreapis/.dockerignore diff --git a/vreapis/.dockerignore b/vreapis/.dockerignore new file mode 100644 index 00000000..d2e350f5 --- /dev/null +++ b/vreapis/.dockerignore @@ -0,0 +1,6 @@ +.env +export_VARS +.idea +.pytest_cache +__pycache__ +NaaVRE_db.json From bfc6f691de2cc447a08b4391a4b405f0725cfb49 Mon Sep 17 00:00:00 2001 From: Andy Bayer Roswell Date: Sun, 8 Dec 2024 02:50:45 +0100 Subject: [PATCH 112/149] added some handy scripts to export cells from psql pod in legacy db schema [used in tinydb] --- ...port-from-psql-pod-in-legacy-db-schema.ps1 | 24 +++++++++++++++++++ ...xport-from-psql-pod-in-legacy-db-schema.sh | 3 +++ 2 files changed, 27 insertions(+) create mode 100644 vreapis/catalog/management/commands/export-from-psql-pod-in-legacy-db-schema.ps1 create mode 100644 vreapis/catalog/management/commands/export-from-psql-pod-in-legacy-db-schema.sh diff --git a/vreapis/catalog/management/commands/export-from-psql-pod-in-legacy-db-schema.ps1 b/vreapis/catalog/management/commands/export-from-psql-pod-in-legacy-db-schema.ps1 new file mode 100644 index 00000000..5898b2f8 --- /dev/null +++ b/vreapis/catalog/management/commands/export-from-psql-pod-in-legacy-db-schema.ps1 @@ -0,0 +1,24 @@ +$project_root = Resolve-Path "$PSScriptRoot/../../.." + +$vreapi_pod_inf = kubectl get pod | sls '^vrepaas-vreapi-\w+-\w+' +$vreapi_pod_name = $vreapi_pod_inf.Matches.Value +$psql_pod_inf = kubectl get pod -owide | sls '^vrepaas-postgresql-0' +$psql_pod_IP = ($psql_pod_inf.Line | sls '\d+\.\d+\.\d+\.\d+').Matches.Value + +cd $project_root +$var = Get-Content export_VARS +$off_DB_HOST = ($var | sls '^export DB_HOST=').LineNumber - 1 +$off_DB_NAME = ($var | sls '^export DB_NAME=').LineNumber - 1 +$off_DB_USER = ($var | sls '^export DB_USER=').LineNumber - 1 +$off_DB_PASSWORD = ($var | sls '^export DB_PASSWORD=').LineNumber - 1 +$var[$off_DB_HOST] = "export DB_HOST=$psql_pod_IP" +$var[$off_DB_NAME] = "export DB_NAME=vrepaas" +$var[$off_DB_USER] = "export DB_USER=vrepaas" +$var[$off_DB_PASSWORD] = "export DB_PASSWORD=vrepaas" + +$var > /tmp/export_VARS +kubectl cp /tmp/export_VARS "${vreapi_pod_name}:/tmp" +kubectl cp "$PSScriptRoot/export-from-psql-pod-in-legacy-db-schema.sh" "${vreapi_pod_name}:/tmp" +kubectl exec -it $vreapi_pod_name -- bash "/tmp/export-from-psql-pod-in-legacy-db-schema.sh" + +kubectl cp "${vreapi_pod_name}:/app/NaaVRE_db.json" $project_root/NaaVRE_db.json diff --git a/vreapis/catalog/management/commands/export-from-psql-pod-in-legacy-db-schema.sh b/vreapis/catalog/management/commands/export-from-psql-pod-in-legacy-db-schema.sh new file mode 100644 index 00000000..1e3d8bf2 --- /dev/null +++ b/vreapis/catalog/management/commands/export-from-psql-pod-in-legacy-db-schema.sh @@ -0,0 +1,3 @@ +source /opt/venv/bin/activate +source /tmp/export_VARS +python manage.py export_in_legacy_db_schema Cell From 83fcf1bb60ac33f40b670cce09dca886cbcb3944 Mon Sep 17 00:00:00 2001 From: Andy Bayer Roswell Date: Sun, 8 Dec 2024 16:07:49 +0100 Subject: [PATCH 113/149] added some handy scripts to export cells from psql pod in legacy db schema [used in tinydb] --- .../commands/export-from-psql-pod-in-legacy-db-schema.ps1 | 8 +++++++- .../management/commands/export_in_legacy_db_schema.py | 3 +++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/vreapis/catalog/management/commands/export-from-psql-pod-in-legacy-db-schema.ps1 b/vreapis/catalog/management/commands/export-from-psql-pod-in-legacy-db-schema.ps1 index 5898b2f8..c73bff49 100644 --- a/vreapis/catalog/management/commands/export-from-psql-pod-in-legacy-db-schema.ps1 +++ b/vreapis/catalog/management/commands/export-from-psql-pod-in-legacy-db-schema.ps1 @@ -1,3 +1,5 @@ +$cwd = $PWD + $project_root = Resolve-Path "$PSScriptRoot/../../.." $vreapi_pod_inf = kubectl get pod | sls '^vrepaas-vreapi-\w+-\w+' @@ -19,6 +21,10 @@ $var[$off_DB_PASSWORD] = "export DB_PASSWORD=vrepaas" $var > /tmp/export_VARS kubectl cp /tmp/export_VARS "${vreapi_pod_name}:/tmp" kubectl cp "$PSScriptRoot/export-from-psql-pod-in-legacy-db-schema.sh" "${vreapi_pod_name}:/tmp" + +kubectl cp ~/NaaVRE/NaaVRE_db.json "${vreapi_pod_name}:/app" kubectl exec -it $vreapi_pod_name -- bash "/tmp/export-from-psql-pod-in-legacy-db-schema.sh" -kubectl cp "${vreapi_pod_name}:/app/NaaVRE_db.json" $project_root/NaaVRE_db.json +kubectl cp "${vreapi_pod_name}:/app/NaaVRE_db.json" ~/NaaVRE/NaaVRE_db.json + +cd $cwd diff --git a/vreapis/catalog/management/commands/export_in_legacy_db_schema.py b/vreapis/catalog/management/commands/export_in_legacy_db_schema.py index 63b7b529..baa98ec7 100644 --- a/vreapis/catalog/management/commands/export_in_legacy_db_schema.py +++ b/vreapis/catalog/management/commands/export_in_legacy_db_schema.py @@ -21,11 +21,14 @@ def handle(self, *args, **options): match table: case 'Cell': queryset: QuerySet = Cell.objects.all() + print(f'Cell count: {queryset.count()}') no: int = 1 cells: dict[str, dict] = {} for cell in queryset: serializer = CellSerializer(cell) + print(f'task_name: {cell.task_name}') cells[str(no)] = serializer.data + no += 1 db_file: str = f'{common.project_root}/NaaVRE_db.json' with open(db_file) as f: db = json.load(f) From ef0b9f37f65993c6ba41fda52dd4b91861281943 Mon Sep 17 00:00:00 2001 From: Andy Bayer Roswell Date: Sun, 15 Dec 2024 15:50:29 +0100 Subject: [PATCH 114/149] added some handy scripts to convert some test data to ipynb/rmd --- vreapis/tests/extract-notebook.ps1 | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 vreapis/tests/extract-notebook.ps1 diff --git a/vreapis/tests/extract-notebook.ps1 b/vreapis/tests/extract-notebook.ps1 new file mode 100644 index 00000000..d9e5fd2c --- /dev/null +++ b/vreapis/tests/extract-notebook.ps1 @@ -0,0 +1,15 @@ +param( + [System.IO.FileInfo[]]$input_files, + [System.IO.FileInfo]$output_dir +) +New-Item -i Directory -f $output_dir +foreach ($input_file in $input_files) { + $root = ConvertFrom-Json (Get-Content -raw $input_file) -d 100 + ConvertTo-Json $root.data.notebook -d 100 > $output_dir/$($input_file.BaseName).ipynb + switch ($root.data.kernel) { + 'ipython' {} + 'IRkernel' { + + } + } +} From 171f0bbb4cc02fff91006e4a864f8fa1581463ad Mon Sep 17 00:00:00 2001 From: Andy Bayer Roswell Date: Sun, 15 Dec 2024 17:59:06 +0100 Subject: [PATCH 115/149] added some handy scripts to convert some test data to ipynb/rmd --- vreapis/tests/extract-notebook.ps1 | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/vreapis/tests/extract-notebook.ps1 b/vreapis/tests/extract-notebook.ps1 index d9e5fd2c..3b0d5568 100644 --- a/vreapis/tests/extract-notebook.ps1 +++ b/vreapis/tests/extract-notebook.ps1 @@ -5,11 +5,12 @@ param( New-Item -i Directory -f $output_dir foreach ($input_file in $input_files) { $root = ConvertFrom-Json (Get-Content -raw $input_file) -d 100 - ConvertTo-Json $root.data.notebook -d 100 > $output_dir/$($input_file.BaseName).ipynb + $serialized_nb = ConvertTo-Json $root.data.notebook -d 100 + $serialized_nb > $output_dir/$($input_file.BaseName).ipynb switch ($root.data.kernel) { 'ipython' {} 'IRkernel' { - + $serialized_nb | jupytext --from ipynb --to Rmd > $output_dir/$($input_file.BaseName).Rmd } } } From 523e5e71e2d8ee90c21031abd27cd676310ea818 Mon Sep 17 00:00:00 2001 From: Andy Bayer Roswell Date: Thu, 19 Dec 2024 23:15:58 +0100 Subject: [PATCH 116/149] handy scripts for database dump --- .../catalog/management/commands/export_in_legacy_db_schema.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vreapis/catalog/management/commands/export_in_legacy_db_schema.py b/vreapis/catalog/management/commands/export_in_legacy_db_schema.py index baa98ec7..77ef1725 100644 --- a/vreapis/catalog/management/commands/export_in_legacy_db_schema.py +++ b/vreapis/catalog/management/commands/export_in_legacy_db_schema.py @@ -20,7 +20,7 @@ def handle(self, *args, **options): for table in options['tables']: match table: case 'Cell': - queryset: QuerySet = Cell.objects.all() + queryset: QuerySet = Cell.objects.all().order_by('task_name') print(f'Cell count: {queryset.count()}') no: int = 1 cells: dict[str, dict] = {} From a6e658eafc15fdfbb117f8815c92416bb3707b14 Mon Sep 17 00:00:00 2001 From: Andy Bayer Roswell Date: Wed, 5 Feb 2025 02:59:07 +0100 Subject: [PATCH 117/149] bugfixes for test; elim CSRF_TRUSTED_ORIGINS --- vreapis/containerizer/tests.py | 25 +++++++++++++------ .../tests/emulated-frontend/containerizer.py | 2 +- vreapis/vreapis/settings/base.py | 2 +- 3 files changed, 20 insertions(+), 9 deletions(-) diff --git a/vreapis/containerizer/tests.py b/vreapis/containerizer/tests.py index afa785ca..28bca5fd 100644 --- a/vreapis/containerizer/tests.py +++ b/vreapis/containerizer/tests.py @@ -12,6 +12,7 @@ from django.test import TestCase, Client from django.conf import settings from django.contrib.auth.models import User +from rest_framework.authtoken.models import Token from urllib.parse import urlencode import requests import nbformat @@ -32,9 +33,22 @@ elif os.path.exists('tests/resources/'): base_path = 'tests/resources/' +dummy_username = f'test' +dummy_password = '0' + +def create_dummy_credentials(): + dummy_username = f'test' + dummy_password = '0' + try: + dummy_user: User = User.objects.get(username=dummy_username) + except User.DoesNotExist: + dummy_user = User.objects.create_user(dummy_username, password=dummy_password) + django_token, created = Token.objects.get_or_create(user=dummy_user, key=os.getenv("DJANGO_TOKEN")) + def get_auth_header() -> dict[str, str]: - return {'Authorization': f'Token {settings.NAAVRE_API_TOKEN}'} + # return {'Authorization': f'Token {django_token}'} + return {'Authorization': f'Token {os.getenv("DJANGO_TOKEN")}'} class GetBaseImagesTestCase(TestCase): @@ -44,12 +58,7 @@ def Keycloak_login() -> dict[str, any]: def test_get_base_images(self): client = Client() - dummy_username = f'test' - dummy_password = '0' - try: - dummy_user: User = User.objects.get(username=dummy_username) - except User.DoesNotExist: - dummy_user = User.objects.create_user(dummy_username, password=dummy_password) + create_dummy_credentials() client.login(username=dummy_username, password=dummy_password) response = client.get('/api/containerizer/baseimagetags', headers=get_auth_header()) @@ -164,6 +173,7 @@ def test(self): notebook = json.load(file) file.close() client = Client() + create_dummy_credentials() response = client.post('/api/containerizer/extract', headers=get_auth_header(), data=notebook, content_type="application/json") self.assertEqual(response.status_code, 200) # get JSON response @@ -179,6 +189,7 @@ class CellsHandlerTestCase(TestCase): def setUp(self): self.client = Client() + create_dummy_credentials() def create_cell_and_add_to_cat(self, cell_path=None) -> (dict, dict): print('Creating cell from: ', cell_path) diff --git a/vreapis/tests/emulated-frontend/containerizer.py b/vreapis/tests/emulated-frontend/containerizer.py index f8489da2..068b1711 100644 --- a/vreapis/tests/emulated-frontend/containerizer.py +++ b/vreapis/tests/emulated-frontend/containerizer.py @@ -23,7 +23,7 @@ API_ENDPOINT: str = f"{os.getenv('API_ENDPOINT')}/api/containerizer" headers: dict = { "Content-Type": "application/json", - "Authorization": f"Token {os.getenv('NAAVRE_API_TOKEN')}", + "Authorization": f"Token {os.getenv('DJANGO_TOKEN')}", } session = requests.Session() diff --git a/vreapis/vreapis/settings/base.py b/vreapis/vreapis/settings/base.py index b97bdd9b..04eb5c7a 100644 --- a/vreapis/vreapis/settings/base.py +++ b/vreapis/vreapis/settings/base.py @@ -22,7 +22,7 @@ ALLOWED_HOSTS = ['*'] -CSRF_TRUSTED_ORIGINS = [os.getenv('TRUSTED_ORIGINS')] +# CSRF_TRUSTED_ORIGINS = [os.getenv('TRUSTED_ORIGINS')] # CSRF_TRUSTED_ORIGINS = os.getenv('TRUSTED_ORIGINS').split(sep=';') CORS_ALLOWED_ORIGIN_REGEXES = [ From 0b626910f6ffaacd6d529390af3c0b181d625ae9 Mon Sep 17 00:00:00 2001 From: Andy Bayer Roswell Date: Tue, 11 Feb 2025 04:42:38 +0100 Subject: [PATCH 118/149] script for pf test --- vreapis/tests/perf/dup-test-files.ps1 | 0 vreapis/tests/perf/record-usage.ps1 | 41 +++++++++++++++++++++++++++ 2 files changed, 41 insertions(+) create mode 100644 vreapis/tests/perf/dup-test-files.ps1 create mode 100644 vreapis/tests/perf/record-usage.ps1 diff --git a/vreapis/tests/perf/dup-test-files.ps1 b/vreapis/tests/perf/dup-test-files.ps1 new file mode 100644 index 00000000..e69de29b diff --git a/vreapis/tests/perf/record-usage.ps1 b/vreapis/tests/perf/record-usage.ps1 new file mode 100644 index 00000000..0385eecf --- /dev/null +++ b/vreapis/tests/perf/record-usage.ps1 @@ -0,0 +1,41 @@ +param( + [string]$pod_name = 'vrepaas-vreapi', + [string]$log_dir = '.log', + [Double]$interval = 1 +) + +if ((Test-Path $log_dir) -eq $false) { + & '/usr/bin/mkdir' -p $log_dir +} + +$pod_name = ((kubectl get pod | Select-String $pod_name).Line -split '\s+')[0] + +$log_file = $log_dir + "/$pod_name.$(Get-Date -Format 'yyyyMMdd-HHmmss').csv" + +Write-Host -NoNewline 'Recording CPU & mem usage of pod ' +Write-Host -NoNewline -ForegroundColor Green $pod_name +Write-Host -NoNewline ' to ' +Write-Host -NoNewline -ForegroundColor Green $log_file +Write-Host -NoNewline ' every ' +Write-Host -NoNewline -ForegroundColor Green $interval +Write-Host ' second(s) ...' + +class Pod_Metric { + [string]$time = (Get-Date -Format 'yyyy-MM-dd HH:mm:ss.fff') + [string]$CPU + [string]$mem + + Pod_Metric([string]$cpu, [string]$mem) { + $this.CPU = $cpu + $this.mem = $mem + } +} + +while ($true) { + $entry = kubectl top pod $pod_name --no-headers | ForEach-Object { + $col = $_ -split '\s+' + [Pod_Metric]::new($col[1], $col[2]) + } + $entry | Export-Csv -Append $log_file + Start-Sleep $interval +} From e0ba654df3e8416c248a6febd776cb02cafe7498 Mon Sep 17 00:00:00 2001 From: Andy Bayer Roswell Date: Wed, 12 Feb 2025 16:54:51 +0100 Subject: [PATCH 119/149] script for pf test --- vreapis/tests/perf/dup-test-files.ps1 | 21 +++++++++++++++++++++ vreapis/tests/perf/record-usage.ps1 | 5 ++++- 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/vreapis/tests/perf/dup-test-files.ps1 b/vreapis/tests/perf/dup-test-files.ps1 index e69de29b..edf37e75 100644 --- a/vreapis/tests/perf/dup-test-files.ps1 +++ b/vreapis/tests/perf/dup-test-files.ps1 @@ -0,0 +1,21 @@ +# duplicate rmd test files for test repetitions + +param( + [Parameter(Mandatory = $true)][System.IO.FileInfo[]]$pathnames, + [int]$copies = 10, + [string]$extension_prefix_prefix = '.', + [string]$extension_prefix_suffix = '', + [string]$magic_prefix = ' # C25B14E054D65738 [rpt ', + [string]$magic_suffix = '] ' +) + +foreach ($pathname in $pathnames) { + $content = Get-Content -Raw $pathname + $re = '((^|[\r\n])```{r[^\r\n]+[\r\n]+(\s#\|[^\r\n]*[\r\n]+)*#[^\r\n]*)' # code chunk header to 1st line comment + # $res = $content | Select-String -AllMatches $re + # $res.Matches | ForEach-Object { Write-Host $_.Value } + for ($i = 0; $i -lt $copies; ++$i) { + $new_content = $content -replace $re, "`$1$magic_prefix$i$magic_suffix" + Set-Content "$($pathname.DirectoryName)/$($pathname.BaseName)$extension_prefix_prefix$i$extension_prefix_suffix$($pathname.Extension)" $new_content + } +} diff --git a/vreapis/tests/perf/record-usage.ps1 b/vreapis/tests/perf/record-usage.ps1 index 0385eecf..53986ba7 100644 --- a/vreapis/tests/perf/record-usage.ps1 +++ b/vreapis/tests/perf/record-usage.ps1 @@ -1,5 +1,8 @@ param( - [string]$pod_name = 'vrepaas-vreapi', + [string]$browser_cmd = $null, + [switch]$jupyterlab_backend = $false, + [switch]$rstudio_backend = $false, + [string]$pod_name = $null, [string]$log_dir = '.log', [Double]$interval = 1 ) From 11c3366b2b587b9b9bcdcd9fa7375c612dc5aa4e Mon Sep 17 00:00:00 2001 From: Andy Bayer Roswell Date: Wed, 12 Feb 2025 17:01:34 +0100 Subject: [PATCH 120/149] script for pf test --- vreapis/tests/perf/dup-test-files.ps1 | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/vreapis/tests/perf/dup-test-files.ps1 b/vreapis/tests/perf/dup-test-files.ps1 index edf37e75..fb7373a9 100644 --- a/vreapis/tests/perf/dup-test-files.ps1 +++ b/vreapis/tests/perf/dup-test-files.ps1 @@ -1,4 +1,5 @@ # duplicate rmd test files for test repetitions +# At present, Cell Containerizer from NaaVRE automatically generates Docker image name for a cell using login username and its 1st line comment. To bring more convenience to test repetitions, this script directly duplicates designated test files with suffix added to the 1st line comment. param( [Parameter(Mandatory = $true)][System.IO.FileInfo[]]$pathnames, @@ -11,7 +12,7 @@ param( foreach ($pathname in $pathnames) { $content = Get-Content -Raw $pathname - $re = '((^|[\r\n])```{r[^\r\n]+[\r\n]+(\s#\|[^\r\n]*[\r\n]+)*#[^\r\n]*)' # code chunk header to 1st line comment + $re = '((^|[\r\n])```{r[^\r\n]+[\r\n]+(\s#\|[^\r\n]*[\r\n]+)*#[^\r\n]*)' # code chunk header to 1st line comment [chunk options w/ prefix #| skipped] # $res = $content | Select-String -AllMatches $re # $res.Matches | ForEach-Object { Write-Host $_.Value } for ($i = 0; $i -lt $copies; ++$i) { From 923adcdf1988c0c10ec8ec1e5be0314d77be7af8 Mon Sep 17 00:00:00 2001 From: Andy Bayer Roswell Date: Fri, 14 Feb 2025 03:10:03 +0100 Subject: [PATCH 121/149] script for pf test --- vreapis/tests/perf/dup-test-files.ps1 | 2 +- vreapis/tests/perf/record-usage.ps1 | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/vreapis/tests/perf/dup-test-files.ps1 b/vreapis/tests/perf/dup-test-files.ps1 index fb7373a9..5906682d 100644 --- a/vreapis/tests/perf/dup-test-files.ps1 +++ b/vreapis/tests/perf/dup-test-files.ps1 @@ -12,7 +12,7 @@ param( foreach ($pathname in $pathnames) { $content = Get-Content -Raw $pathname - $re = '((^|[\r\n])```{r[^\r\n]+[\r\n]+(\s#\|[^\r\n]*[\r\n]+)*#[^\r\n]*)' # code chunk header to 1st line comment [chunk options w/ prefix #| skipped] + $re = '((^|[\r\n])```{r[^\r\n]+[\r\n]+(\s#\|[^\r\n]*[\r\n]+)*#[^\r\n]*)' # code chunk header to 1st line comment [chunk options w/ prefix #| skipped] [see https://yihui.org/knitr/options/?utm_source=chatgpt.com] # $res = $content | Select-String -AllMatches $re # $res.Matches | ForEach-Object { Write-Host $_.Value } for ($i = 0; $i -lt $copies; ++$i) { diff --git a/vreapis/tests/perf/record-usage.ps1 b/vreapis/tests/perf/record-usage.ps1 index 53986ba7..2eb2e03b 100644 --- a/vreapis/tests/perf/record-usage.ps1 +++ b/vreapis/tests/perf/record-usage.ps1 @@ -37,7 +37,7 @@ class Pod_Metric { while ($true) { $entry = kubectl top pod $pod_name --no-headers | ForEach-Object { $col = $_ -split '\s+' - [Pod_Metric]::new($col[1], $col[2]) + [Pod_Metric]::new($col[1], $col[2]) # cols from /usr/bin/ps: USER PID %CPU %MEM VSZ RSS TTY STAT START TIME COMMAND } $entry | Export-Csv -Append $log_file Start-Sleep $interval From 04c08437f6d6e8be35883c43a551d6ad3590899b Mon Sep 17 00:00:00 2001 From: Andy Bayer Roswell Date: Fri, 21 Feb 2025 01:36:37 +0100 Subject: [PATCH 122/149] script for pf test --- vreapis/tests/perf/record-usage.ps1 | 156 ++++++++++++++++++++++++---- 1 file changed, 136 insertions(+), 20 deletions(-) diff --git a/vreapis/tests/perf/record-usage.ps1 b/vreapis/tests/perf/record-usage.ps1 index 2eb2e03b..f8b1eddf 100644 --- a/vreapis/tests/perf/record-usage.ps1 +++ b/vreapis/tests/perf/record-usage.ps1 @@ -1,7 +1,7 @@ param( - [string]$browser_cmd = $null, - [switch]$jupyterlab_backend = $false, - [switch]$rstudio_backend = $false, + [string]$browser_program_name = $null, + [switch]$JupyterLab_backend = $false, + [switch]$RStudio_backend = $false, [string]$pod_name = $null, [string]$log_dir = '.log', [Double]$interval = 1 @@ -11,34 +11,150 @@ if ((Test-Path $log_dir) -eq $false) { & '/usr/bin/mkdir' -p $log_dir } -$pod_name = ((kubectl get pod | Select-String $pod_name).Line -split '\s+')[0] +$date = Get-Date -Format 'yyyyMMdd-HHmmss' +$raw_log_file = $log_dir + "/$date.raw.csv" +$cooked_log_file = $log_dir + "/$date.cooked.csv" -$log_file = $log_dir + "/$pod_name.$(Get-Date -Format 'yyyyMMdd-HHmmss').csv" - -Write-Host -NoNewline 'Recording CPU & mem usage of pod ' -Write-Host -NoNewline -ForegroundColor Green $pod_name -Write-Host -NoNewline ' to ' -Write-Host -NoNewline -ForegroundColor Green $log_file +Write-Host -NoNewline 'Recording CPU & mem usage at ' +Write-Host -NoNewline -ForegroundColor Green $raw_log_file +Write-Host -NoNewline ' and ' +Write-Host -NoNewline -ForegroundColor Green $cooked_log_file Write-Host -NoNewline ' every ' Write-Host -NoNewline -ForegroundColor Green $interval Write-Host ' second(s) ...' -class Pod_Metric { - [string]$time = (Get-Date -Format 'yyyy-MM-dd HH:mm:ss.fff') - [string]$CPU - [string]$mem +class Simplified_ps_Entry { + static [string] $header_row = 'user,pid,%cpu,rss,command' # complete header row: 'USER,PID,%CPU,%MEM,VSZ,RSS,TTY,STAT,START,TIME,COMMAND' + static [string[]] $col_name = [Simplified_ps_Entry]::header_row -replace '%', '' -split ',' + static $col_name_index = @{} + static [void] Init() { + for ([int]$i = 0; $i -lt [Simplified_ps_Entry]::col_name.Count; ++$i) { + [Simplified_ps_Entry]::col_name_index[[Simplified_ps_Entry]::col_name[$i]] = $i + } + } + [string]$time = '' + [string]$USER = '' + [int]$PID = -1 + [double]$CPU = 0.0 + [int]$RSS = 0 # KB + [string]$COMMAND = '' + [void]constructor([string]$user, [int]$process_ID, [double]$cpu, [int]$rss, [string]$command) { + $this.USER = $user + $this.PID = $process_ID + $this.CPU = $cpu + $this.RSS = $rss + $this.COMMAND = $command + } + Simplified_ps_Entry([string]$time, [string]$user, [int]$process_ID, [double]$cpu, [int]$rss, [string]$command) { + $this.time = $time + $this.constructor($user, $process_ID, $cpu, $rss, $command) + } + [void]constructor([string]$raw_entry) { + $cooked_entry = $raw_entry -split '\s+' + $this.USER = $cooked_entry[[Simplified_ps_Entry]::col_name_index['user']] + $this.PID = [int]$cooked_entry[[Simplified_ps_Entry]::col_name_index['pid']] + $this.CPU = [double]$cooked_entry[[Simplified_ps_Entry]::col_name_index['cpu']] + $this.RSS = [int]$cooked_entry[[Simplified_ps_Entry]::col_name_index['rss']] + $this.COMMAND = $cooked_entry[[Simplified_ps_Entry]::col_name_index['command']..($cooked_entry.count - 1)] + } + Simplified_ps_Entry([string]$time, [string]$raw_entry) { + $this.time = $time + $this.constructor($raw_entry) + } +} +[Simplified_ps_Entry]::Init() - Pod_Metric([string]$cpu, [string]$mem) { +class Resource_Metric { + [double]$CPU # % + [double]$mem # Mi + Resource_Metric() { + $this.CPU = 0 + $this.mem = 0 + } + Resource_Metric([double]$cpu, [double]$mem) { $this.CPU = $cpu $this.mem = $mem } } +if (-not (Test-Path $raw_log_file)) { (@('time') + [Simplified_ps_Entry]::col_name) -join ',' | Out-File $raw_log_file } # add csv headers first + +$compound_resource_metric = [PSCustomObject]@{ + "time" = $null +} +$cpu_headers = $mem_headers = @() +if ($browser_program_name -ne $null) { + $cpu_headers += "CPU:browser:$browser_program_name" + $mem_headers += "mem:browser:$browser_program_name" +} +if ($JupyterLab_backend) { + $cpu_headers += "CPU:JupyterLab backend" + $mem_headers += "mem:JupyterLab backend" +} +if ($RStudio_backend) { + $cpu_headers += "CPU:RStudio backend" + $mem_headers += "mem:RStudio backend" +} +if ($pod_name -ne $null) { + $pod_name = ((kubectl get pod | Select-String $pod_name).Line -split '\s+')[0] + $cpu_headers += "CPU:pod:$pod_name" + $mem_headers += "mem:pod:$pod_name" +} +$cpu_headers + $mem_headers | ForEach-Object { $compound_resource_metric | Add-Member -MemberType NoteProperty -Name $_ -Value $null } +if (-not (Test-Path $cooked_log_file)) { $compound_resource_metric | Export-Csv $cooked_log_file } + +$ps_pathname = '/usr/bin/ps' +$time_format = 'yyyy-MM-dd HH:mm:ss.fff' + while ($true) { - $entry = kubectl top pod $pod_name --no-headers | ForEach-Object { - $col = $_ -split '\s+' - [Pod_Metric]::new($col[1], $col[2]) # cols from /usr/bin/ps: USER PID %CPU %MEM VSZ RSS TTY STAT START TIME COMMAND + $time = Get-Date -Format $time_format + $compound_resource_metric.time = $time + $new_raw_ps_entries = New-Object System.Collections.Generic.List[Simplified_ps_Entry] + if ($browser_program_name -ne $null) { + $browser_processes = & $ps_pathname 'exo' $([Simplified_ps_Entry]::header_row) | Select-String -a $browser_program_name + $browser_metric = [Resource_Metric]::new() + foreach ($process in $browser_processes) { + $m = [Simplified_ps_Entry]::new($time, $process) + $new_raw_ps_entries.Add($m) + $browser_metric.CPU += $m.CPU + $browser_metric.mem += $m.RSS * 1000 / 1024.0 + } + $compound_resource_metric."CPU:browser:$browser_program_name" = $browser_metric.CPU + $compound_resource_metric."mem:browser:$browser_program_name" = $browser_metric.mem + } + if ($JupyterLab_backend) { + $JupyterLab_backend = & $ps_pathname 'exo' $([Simplified_ps_Entry]::header_row) | Select-String -a 'jupyter.?lab' + $JupyterLab_backend_metric = [Resource_Metric]::new() + foreach ($process in $JupyterLab_backend) { + $m = [Simplified_ps_Entry]::new($time, $process) + $new_raw_ps_entries.Add($m) + $JupyterLab_backend_metric.CPU += [double]$m.CPU + $JupyterLab_backend_metric.mem += [double]$m.RSS * 1000 / 1024.0 + } + $compound_resource_metric."CPU:JupyterLab backend" = $JupyterLab_backend_metric.CPU + $compound_resource_metric."mem:JupyterLab backend" = $JupyterLab_backend_metric.mem + } + if ($RStudio_backend) { + $RStudio_backend_processes = & $ps_pathname 'axo' $([Simplified_ps_Entry]::header_row) | Select-String -a 'rstudio' # here axo not exo since all process got by the latter are contained in those got by the former + $RStudio_backend_metric = [Resource_Metric]::new() + foreach ($process in $RStudio_backend_processes) { + $m = [Simplified_ps_Entry]::new($time, $process) + $new_raw_ps_entries.Add($m) + $RStudio_backend_metric.CPU += [double]$m.CPU + $RStudio_backend_metric.mem += [double]$m.RSS * 1000 / 1024.0 + } + $compound_resource_metric."CPU:RStudio backend" = $RStudio_backend_metric.CPU + $compound_resource_metric."mem:RStudio backend" = $RStudio_backend_metric.mem + } + if ($pod_name -ne $null) { + $pod_metric = kubectl top pod $pod_name --no-headers | ForEach-Object { + $col = $_ -split '\s+' + [Resource_Metric]::new([double]($col[1].substring(0, $col[1].Length - 'm'.Length)) / 10.0, [double]($col[2]).substring(0, $col[2].Length - 'Mi'.Length)) + } + $compound_resource_metric."CPU:pod:$pod_name" = $pod_metric.CPU + $compound_resource_metric."mem:pod:$pod_name" = $pod_metric.mem } - $entry | Export-Csv -Append $log_file - Start-Sleep $interval + $new_raw_ps_entries | Export-Csv -Append $raw_log_file + $compound_resource_metric | Export-Csv -Append $cooked_log_file + Start-Sleep($interval) } From 6299146ac68e25b985804839df53bf3d2d0555d9 Mon Sep 17 00:00:00 2001 From: Andy Bayer Roswell Date: Fri, 21 Feb 2025 02:18:07 +0100 Subject: [PATCH 123/149] script for pf test --- vreapis/tests/perf/record-usage.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vreapis/tests/perf/record-usage.ps1 b/vreapis/tests/perf/record-usage.ps1 index f8b1eddf..23456ed2 100644 --- a/vreapis/tests/perf/record-usage.ps1 +++ b/vreapis/tests/perf/record-usage.ps1 @@ -101,7 +101,7 @@ if ($pod_name -ne $null) { $mem_headers += "mem:pod:$pod_name" } $cpu_headers + $mem_headers | ForEach-Object { $compound_resource_metric | Add-Member -MemberType NoteProperty -Name $_ -Value $null } -if (-not (Test-Path $cooked_log_file)) { $compound_resource_metric | Export-Csv $cooked_log_file } +if (-not (Test-Path $cooked_log_file)) { $compound_resource_metric | ConvertTo-Csv | Select-Object -First 1 | Out-File $cooked_log_file } # add csv headers first $ps_pathname = '/usr/bin/ps' $time_format = 'yyyy-MM-dd HH:mm:ss.fff' From 4e844da4f03fd407822ec803339973bad5abdd2e Mon Sep 17 00:00:00 2001 From: Andy Bayer Roswell Date: Fri, 21 Feb 2025 02:51:42 +0100 Subject: [PATCH 124/149] script for pf test --- vreapis/tests/perf/record-usage.ps1 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vreapis/tests/perf/record-usage.ps1 b/vreapis/tests/perf/record-usage.ps1 index 23456ed2..aaad2ef0 100644 --- a/vreapis/tests/perf/record-usage.ps1 +++ b/vreapis/tests/perf/record-usage.ps1 @@ -123,9 +123,9 @@ while ($true) { $compound_resource_metric."mem:browser:$browser_program_name" = $browser_metric.mem } if ($JupyterLab_backend) { - $JupyterLab_backend = & $ps_pathname 'exo' $([Simplified_ps_Entry]::header_row) | Select-String -a 'jupyter.?lab' + $JupyterLab_backend_processes = & $ps_pathname 'exo' $([Simplified_ps_Entry]::header_row) | Select-String -a 'jupyter.?lab' $JupyterLab_backend_metric = [Resource_Metric]::new() - foreach ($process in $JupyterLab_backend) { + foreach ($process in $JupyterLab_backend_processes) { $m = [Simplified_ps_Entry]::new($time, $process) $new_raw_ps_entries.Add($m) $JupyterLab_backend_metric.CPU += [double]$m.CPU From 506c73ece8fc8c1178e3296d5fa717c4a3fd58eb Mon Sep 17 00:00:00 2001 From: Andy Bayer Roswell Date: Fri, 21 Feb 2025 03:10:18 +0100 Subject: [PATCH 125/149] script for pf test --- vreapis/tests/perf/record-usage.ps1 | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/vreapis/tests/perf/record-usage.ps1 b/vreapis/tests/perf/record-usage.ps1 index aaad2ef0..9e4fdf7f 100644 --- a/vreapis/tests/perf/record-usage.ps1 +++ b/vreapis/tests/perf/record-usage.ps1 @@ -1,15 +1,13 @@ param( - [string]$browser_program_name = $null, - [switch]$JupyterLab_backend = $false, - [switch]$RStudio_backend = $false, - [string]$pod_name = $null, - [string]$log_dir = '.log', - [Double]$interval = 1 + [string]$browser_program_name = $null, # record resource usage for browser. use this param to specify re pattern for browser pathname + [switch]$JupyterLab_backend = $false, # record resource usage for JupyterLab backend + [switch]$RStudio_backend = $false, # record resource usage for RStudio backend + [string]$pod_name = $null, # record resource usage for pod [common backend]. use this param to specify re pattern for pod name + [string]$log_dir = '.log', # directory to store log files + [Double]$interval = 1 # interval between 2 adjacent resource usage captures [seconds] ) -if ((Test-Path $log_dir) -eq $false) { - & '/usr/bin/mkdir' -p $log_dir -} +if ((Test-Path $log_dir) -eq $false) { & '/usr/bin/mkdir' -p $log_dir } $date = Get-Date -Format 'yyyyMMdd-HHmmss' $raw_log_file = $log_dir + "/$date.raw.csv" @@ -135,7 +133,7 @@ while ($true) { $compound_resource_metric."mem:JupyterLab backend" = $JupyterLab_backend_metric.mem } if ($RStudio_backend) { - $RStudio_backend_processes = & $ps_pathname 'axo' $([Simplified_ps_Entry]::header_row) | Select-String -a 'rstudio' # here axo not exo since all process got by the latter are contained in those got by the former + $RStudio_backend_processes = & $ps_pathname 'axo' $([Simplified_ps_Entry]::header_row) | Select-String -a 'rstudio' # here 'axo' not 'exo' since all process got by the latter are contained in those got by the former $RStudio_backend_metric = [Resource_Metric]::new() foreach ($process in $RStudio_backend_processes) { $m = [Simplified_ps_Entry]::new($time, $process) From 390f42a7505120c15a99866f390161b589f0a9ea Mon Sep 17 00:00:00 2001 From: Andy Bayer Roswell Date: Fri, 21 Feb 2025 19:16:13 +0100 Subject: [PATCH 126/149] script for pf test --- vreapis/tests/perf/dup-test-files.ps1 | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/vreapis/tests/perf/dup-test-files.ps1 b/vreapis/tests/perf/dup-test-files.ps1 index 5906682d..dfd03fd9 100644 --- a/vreapis/tests/perf/dup-test-files.ps1 +++ b/vreapis/tests/perf/dup-test-files.ps1 @@ -1,12 +1,18 @@ # duplicate rmd test files for test repetitions # At present, Cell Containerizer from NaaVRE automatically generates Docker image name for a cell using login username and its 1st line comment. To bring more convenience to test repetitions, this script directly duplicates designated test files with suffix added to the 1st line comment. +# Example: +# ./dup-test-files a.Rmd +# Results: +# 'a.Rmd' -> 'a.0.Rmd', 'a.1.Rmd', 'a.2.Rmd', ..., 'a.9.Rmd' +# Convert to ipynb: +# ls a*.Rmd | % { cat -raw $_ | jupytext --from Rmd --to ipynb --opt split_at_heading=true -o - > "$($_.BaseName).ipynb" } param( - [Parameter(Mandatory = $true)][System.IO.FileInfo[]]$pathnames, - [int]$copies = 10, + [Parameter(Mandatory = $true)][System.IO.FileInfo[]]$pathnames, # files to dup + [int]$copies = 10, # number of copies [string]$extension_prefix_prefix = '.', [string]$extension_prefix_suffix = '', - [string]$magic_prefix = ' # C25B14E054D65738 [rpt ', + [string]$magic_prefix = ' # C25B14E054D65738 [rpt ', # to conveniently recognise copies [string]$magic_suffix = '] ' ) From b46064a814d9406b412b6e90a270e9f5f560008f Mon Sep 17 00:00:00 2001 From: Andy Bayer Roswell Date: Fri, 21 Feb 2025 20:53:00 +0100 Subject: [PATCH 127/149] script for pf test --- vreapis/tests/perf/record-usage.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vreapis/tests/perf/record-usage.ps1 b/vreapis/tests/perf/record-usage.ps1 index 9e4fdf7f..03005011 100644 --- a/vreapis/tests/perf/record-usage.ps1 +++ b/vreapis/tests/perf/record-usage.ps1 @@ -133,7 +133,7 @@ while ($true) { $compound_resource_metric."mem:JupyterLab backend" = $JupyterLab_backend_metric.mem } if ($RStudio_backend) { - $RStudio_backend_processes = & $ps_pathname 'axo' $([Simplified_ps_Entry]::header_row) | Select-String -a 'rstudio' # here 'axo' not 'exo' since all process got by the latter are contained in those got by the former + $RStudio_backend_processes = & $ps_pathname 'axo' $([Simplified_ps_Entry]::header_row) | Select-String -a 'rstudio-server' # here 'axo' not 'exo' since all process got by the latter are contained in those got by the former $RStudio_backend_metric = [Resource_Metric]::new() foreach ($process in $RStudio_backend_processes) { $m = [Simplified_ps_Entry]::new($time, $process) From 3f78f2895f39ba3308e286841bb6747a9bec5e8f Mon Sep 17 00:00:00 2001 From: Andy Bayer Roswell Date: Sun, 23 Feb 2025 04:08:28 +0100 Subject: [PATCH 128/149] script for pf test --- vreapis/tests/perf/record-usage.ps1 | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/vreapis/tests/perf/record-usage.ps1 b/vreapis/tests/perf/record-usage.ps1 index 03005011..06636928 100644 --- a/vreapis/tests/perf/record-usage.ps1 +++ b/vreapis/tests/perf/record-usage.ps1 @@ -4,7 +4,9 @@ param( [switch]$RStudio_backend = $false, # record resource usage for RStudio backend [string]$pod_name = $null, # record resource usage for pod [common backend]. use this param to specify re pattern for pod name [string]$log_dir = '.log', # directory to store log files - [Double]$interval = 1 # interval between 2 adjacent resource usage captures [seconds] + [Double]$interval = 1, # interval between 2 adjacent resource usage captures [seconds] + [long]$number_of_records = 0, # 0 or neg means infinite records, pos value means number of records to capture + [switch]$console = $false # print usage data to console ) if ((Test-Path $log_dir) -eq $false) { & '/usr/bin/mkdir' -p $log_dir } @@ -75,7 +77,7 @@ class Resource_Metric { } } -if (-not (Test-Path $raw_log_file)) { (@('time') + [Simplified_ps_Entry]::col_name) -join ',' | Out-File $raw_log_file } # add csv headers first +if ((-not $console) -and (-not (Test-Path $raw_log_file))) { (@('time') + [Simplified_ps_Entry]::col_name) -join ',' | Out-File $raw_log_file } # add csv headers first $compound_resource_metric = [PSCustomObject]@{ "time" = $null @@ -99,12 +101,12 @@ if ($pod_name -ne $null) { $mem_headers += "mem:pod:$pod_name" } $cpu_headers + $mem_headers | ForEach-Object { $compound_resource_metric | Add-Member -MemberType NoteProperty -Name $_ -Value $null } -if (-not (Test-Path $cooked_log_file)) { $compound_resource_metric | ConvertTo-Csv | Select-Object -First 1 | Out-File $cooked_log_file } # add csv headers first +if ((-not $console) -and (-not (Test-Path $cooked_log_file))) { $compound_resource_metric | ConvertTo-Csv | Select-Object -First 1 | Out-File $cooked_log_file } # add csv headers first $ps_pathname = '/usr/bin/ps' $time_format = 'yyyy-MM-dd HH:mm:ss.fff' -while ($true) { +$loop_body = { $time = Get-Date -Format $time_format $compound_resource_metric.time = $time $new_raw_ps_entries = New-Object System.Collections.Generic.List[Simplified_ps_Entry] @@ -152,7 +154,17 @@ while ($true) { $compound_resource_metric."CPU:pod:$pod_name" = $pod_metric.CPU $compound_resource_metric."mem:pod:$pod_name" = $pod_metric.mem } - $new_raw_ps_entries | Export-Csv -Append $raw_log_file - $compound_resource_metric | Export-Csv -Append $cooked_log_file + if ($console) { + Write-Output $compound_resource_metric + } else { + $new_raw_ps_entries | Export-Csv -Append $raw_log_file + $compound_resource_metric | Export-Csv -Append $cooked_log_file + } Start-Sleep($interval) } + +if ($number_of_records -le 0) { + while ($true) { & $loop_body } +} else { + for ($i = 0; $i -lt $number_of_records; ++$i) { & $loop_body } +} From b544b94c757d92f7ef60ad96437a6ff14509e4ca Mon Sep 17 00:00:00 2001 From: Andy Bayer Roswell Date: Sun, 23 Feb 2025 14:49:53 +0100 Subject: [PATCH 129/149] script for pf test --- vreapis/tests/perf/record-usage.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vreapis/tests/perf/record-usage.ps1 b/vreapis/tests/perf/record-usage.ps1 index 06636928..7682cdd0 100644 --- a/vreapis/tests/perf/record-usage.ps1 +++ b/vreapis/tests/perf/record-usage.ps1 @@ -155,7 +155,7 @@ $loop_body = { $compound_resource_metric."mem:pod:$pod_name" = $pod_metric.mem } if ($console) { - Write-Output $compound_resource_metric + $compound_resource_metric | Format-Table | Write-Output } else { $new_raw_ps_entries | Export-Csv -Append $raw_log_file $compound_resource_metric | Export-Csv -Append $cooked_log_file From d6190cc0b5be9f1f1245148c6ba1bf9b2899681b Mon Sep 17 00:00:00 2001 From: Andy Bayer Roswell Date: Mon, 24 Feb 2025 02:16:42 +0100 Subject: [PATCH 130/149] script for pf test --- vreapis/tests/perf/record-usage.ps1 | 30 ++++++++++++++++++----------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/vreapis/tests/perf/record-usage.ps1 b/vreapis/tests/perf/record-usage.ps1 index 7682cdd0..2a0a701c 100644 --- a/vreapis/tests/perf/record-usage.ps1 +++ b/vreapis/tests/perf/record-usage.ps1 @@ -9,19 +9,27 @@ param( [switch]$console = $false # print usage data to console ) -if ((Test-Path $log_dir) -eq $false) { & '/usr/bin/mkdir' -p $log_dir } +if ($console) { + Write-Host -NoNewline 'Recording CPU & mem usage at ' + Write-Host -NoNewline -ForegroundColor Green 'console' + Write-Host -NoNewline ' every ' + Write-Host -NoNewline -ForegroundColor Green $interval + Write-Host ' second(s) ...' +} else { + if ((Test-Path $log_dir) -eq $false) { & '/usr/bin/mkdir' -p $log_dir } -$date = Get-Date -Format 'yyyyMMdd-HHmmss' -$raw_log_file = $log_dir + "/$date.raw.csv" -$cooked_log_file = $log_dir + "/$date.cooked.csv" + $date = Get-Date -Format 'yyyyMMdd-HHmmss' + $raw_log_file = $log_dir + "/$date.raw.csv" + $cooked_log_file = $log_dir + "/$date.cooked.csv" -Write-Host -NoNewline 'Recording CPU & mem usage at ' -Write-Host -NoNewline -ForegroundColor Green $raw_log_file -Write-Host -NoNewline ' and ' -Write-Host -NoNewline -ForegroundColor Green $cooked_log_file -Write-Host -NoNewline ' every ' -Write-Host -NoNewline -ForegroundColor Green $interval -Write-Host ' second(s) ...' + Write-Host -NoNewline 'Recording CPU & mem usage at ' + Write-Host -NoNewline -ForegroundColor Green $raw_log_file + Write-Host -NoNewline ' and ' + Write-Host -NoNewline -ForegroundColor Green $cooked_log_file + Write-Host -NoNewline ' every ' + Write-Host -NoNewline -ForegroundColor Green $interval + Write-Host ' second(s) ...' +} class Simplified_ps_Entry { static [string] $header_row = 'user,pid,%cpu,rss,command' # complete header row: 'USER,PID,%CPU,%MEM,VSZ,RSS,TTY,STAT,START,TIME,COMMAND' From bc2135ef8b937219d4f14fad5b855c9dfb8b2bb5 Mon Sep 17 00:00:00 2001 From: Andy Bayer Roswell Date: Mon, 24 Feb 2025 02:22:00 +0100 Subject: [PATCH 131/149] script for pf test --- vreapis/tests/perf/record-usage.ps1 | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/vreapis/tests/perf/record-usage.ps1 b/vreapis/tests/perf/record-usage.ps1 index 2a0a701c..241d1a51 100644 --- a/vreapis/tests/perf/record-usage.ps1 +++ b/vreapis/tests/perf/record-usage.ps1 @@ -125,7 +125,7 @@ $loop_body = { $m = [Simplified_ps_Entry]::new($time, $process) $new_raw_ps_entries.Add($m) $browser_metric.CPU += $m.CPU - $browser_metric.mem += $m.RSS * 1000 / 1024.0 + $browser_metric.mem += $m.RSS * 1000 / 1024.0 / 1024 } $compound_resource_metric."CPU:browser:$browser_program_name" = $browser_metric.CPU $compound_resource_metric."mem:browser:$browser_program_name" = $browser_metric.mem @@ -137,7 +137,7 @@ $loop_body = { $m = [Simplified_ps_Entry]::new($time, $process) $new_raw_ps_entries.Add($m) $JupyterLab_backend_metric.CPU += [double]$m.CPU - $JupyterLab_backend_metric.mem += [double]$m.RSS * 1000 / 1024.0 + $JupyterLab_backend_metric.mem += [double]$m.RSS * 1000 / 1024.0 / 1024 } $compound_resource_metric."CPU:JupyterLab backend" = $JupyterLab_backend_metric.CPU $compound_resource_metric."mem:JupyterLab backend" = $JupyterLab_backend_metric.mem @@ -149,7 +149,7 @@ $loop_body = { $m = [Simplified_ps_Entry]::new($time, $process) $new_raw_ps_entries.Add($m) $RStudio_backend_metric.CPU += [double]$m.CPU - $RStudio_backend_metric.mem += [double]$m.RSS * 1000 / 1024.0 + $RStudio_backend_metric.mem += [double]$m.RSS * 1000 / 1024.0 / 1024 } $compound_resource_metric."CPU:RStudio backend" = $RStudio_backend_metric.CPU $compound_resource_metric."mem:RStudio backend" = $RStudio_backend_metric.mem From 4a5e38612067080a517935af9ea0bdb02362114e Mon Sep 17 00:00:00 2001 From: Andy Bayer Roswell Date: Mon, 24 Feb 2025 02:48:11 +0100 Subject: [PATCH 132/149] script for pf test --- vreapis/tests/perf/record-usage.ps1 | 32 +++++++++++++++++++++-------- 1 file changed, 23 insertions(+), 9 deletions(-) diff --git a/vreapis/tests/perf/record-usage.ps1 b/vreapis/tests/perf/record-usage.ps1 index 241d1a51..9e89c72d 100644 --- a/vreapis/tests/perf/record-usage.ps1 +++ b/vreapis/tests/perf/record-usage.ps1 @@ -2,7 +2,8 @@ param( [string]$browser_program_name = $null, # record resource usage for browser. use this param to specify re pattern for browser pathname [switch]$JupyterLab_backend = $false, # record resource usage for JupyterLab backend [switch]$RStudio_backend = $false, # record resource usage for RStudio backend - [string]$pod_name = $null, # record resource usage for pod [common backend]. use this param to specify re pattern for pod name + [string]$vreapi_pod_name = $null, # record resource usage for vreapi pod [common backend]. use this param to specify re pattern for pod name + [string]$database_pod_name = $null, # record resource usage for db pod [common backend]. use this param to specify re pattern for pod name [string]$log_dir = '.log', # directory to store log files [Double]$interval = 1, # interval between 2 adjacent resource usage captures [seconds] [long]$number_of_records = 0, # 0 or neg means infinite records, pos value means number of records to capture @@ -103,10 +104,15 @@ if ($RStudio_backend) { $cpu_headers += "CPU:RStudio backend" $mem_headers += "mem:RStudio backend" } -if ($pod_name -ne $null) { - $pod_name = ((kubectl get pod | Select-String $pod_name).Line -split '\s+')[0] - $cpu_headers += "CPU:pod:$pod_name" - $mem_headers += "mem:pod:$pod_name" +if ($vreapi_pod_name -ne $null) { + $vreapi_pod_name = ((kubectl get pod | Select-String $vreapi_pod_name).Line -split '\s+')[0] + $cpu_headers += "CPU:pod:$vreapi_pod_name" + $mem_headers += "mem:pod:$vreapi_pod_name" +} +if ($database_pod_name -ne $null) { + $database_pod_name = ((kubectl get pod | Select-String $database_pod_name).Line -split '\s+')[0] + $cpu_headers += "CPU:pod:$database_pod_name" + $mem_headers += "mem:pod:$database_pod_name" } $cpu_headers + $mem_headers | ForEach-Object { $compound_resource_metric | Add-Member -MemberType NoteProperty -Name $_ -Value $null } if ((-not $console) -and (-not (Test-Path $cooked_log_file))) { $compound_resource_metric | ConvertTo-Csv | Select-Object -First 1 | Out-File $cooked_log_file } # add csv headers first @@ -154,13 +160,21 @@ $loop_body = { $compound_resource_metric."CPU:RStudio backend" = $RStudio_backend_metric.CPU $compound_resource_metric."mem:RStudio backend" = $RStudio_backend_metric.mem } - if ($pod_name -ne $null) { - $pod_metric = kubectl top pod $pod_name --no-headers | ForEach-Object { + if ($vreapi_pod_name -ne $null) { + $pod_metric = kubectl top pod $vreapi_pod_name --no-headers | ForEach-Object { + $col = $_ -split '\s+' + [Resource_Metric]::new([double]($col[1].substring(0, $col[1].Length - 'm'.Length)) / 10.0, [double]($col[2]).substring(0, $col[2].Length - 'Mi'.Length)) + } + $compound_resource_metric."CPU:pod:$vreapi_pod_name" = $pod_metric.CPU + $compound_resource_metric."mem:pod:$vreapi_pod_name" = $pod_metric.mem + } + if ($database_pod_name -ne $null) { + $pod_metric = kubectl top pod $database_pod_name --no-headers | ForEach-Object { $col = $_ -split '\s+' [Resource_Metric]::new([double]($col[1].substring(0, $col[1].Length - 'm'.Length)) / 10.0, [double]($col[2]).substring(0, $col[2].Length - 'Mi'.Length)) } - $compound_resource_metric."CPU:pod:$pod_name" = $pod_metric.CPU - $compound_resource_metric."mem:pod:$pod_name" = $pod_metric.mem + $compound_resource_metric."CPU:pod:$database_pod_name" = $pod_metric.CPU + $compound_resource_metric."mem:pod:$database_pod_name" = $pod_metric.mem } if ($console) { $compound_resource_metric | Format-Table | Write-Output From 3ffaeeab280b13dff2e5dc18baa61f55ba115c22 Mon Sep 17 00:00:00 2001 From: Andy Bayer Roswell Date: Tue, 25 Feb 2025 02:39:08 +0100 Subject: [PATCH 133/149] script for pf test --- vreapis/tests/perf/cvt-Rmd-to-ipynb.ps1 | 30 +++++++++++++++++++++++++ vreapis/tests/perf/dup-test-files.ps1 | 5 ++++- 2 files changed, 34 insertions(+), 1 deletion(-) create mode 100644 vreapis/tests/perf/cvt-Rmd-to-ipynb.ps1 diff --git a/vreapis/tests/perf/cvt-Rmd-to-ipynb.ps1 b/vreapis/tests/perf/cvt-Rmd-to-ipynb.ps1 new file mode 100644 index 00000000..c941f069 --- /dev/null +++ b/vreapis/tests/perf/cvt-Rmd-to-ipynb.ps1 @@ -0,0 +1,30 @@ +param( + [System.IO.FileInfo[]]$pathnames # files to convert +) + +$kernelspec = @{ + display_name = 'R [conda env:jupyterlab] *' + language = 'R' + name = 'conda-env-jupyterlab-r' +} +$language_info = @{ + codemirror_mode = 'r' + file_extension = '.r' + mimetype = 'text/x-r-source' + name = 'R' + pygments_lexer = 'r' + version = '4.3.3' +} + +foreach ($pathname in $pathnames) { + Write-Host -NoNewline "src: " + Write-Host -ForegroundColor Yellow $pathname + $dst = "$($pathname.DirectoryName)/$($pathname.BaseName).ipynb" + Get-Content -Raw $pathname | jupytext --from Rmd --to ipynb --opt split_at_heading=true -o - > $dst + $generated_ipynb = ConvertFrom-Json (Get-Content $dst -Raw) + $generated_ipynb.metadata | Add-Member -NotePropertyName kernelspec -NotePropertyValue $kernelspec + $generated_ipynb.metadata | Add-Member -NotePropertyName language_info -NotePropertyValue $language_info + $generated_ipynb | ConvertTo-Json -Depth 100 | Set-Content $dst + Write-Host -NoNewline "dst: " + Write-Host -ForegroundColor Green $dst +} diff --git a/vreapis/tests/perf/dup-test-files.ps1 b/vreapis/tests/perf/dup-test-files.ps1 index dfd03fd9..334fecee 100644 --- a/vreapis/tests/perf/dup-test-files.ps1 +++ b/vreapis/tests/perf/dup-test-files.ps1 @@ -23,6 +23,9 @@ foreach ($pathname in $pathnames) { # $res.Matches | ForEach-Object { Write-Host $_.Value } for ($i = 0; $i -lt $copies; ++$i) { $new_content = $content -replace $re, "`$1$magic_prefix$i$magic_suffix" - Set-Content "$($pathname.DirectoryName)/$($pathname.BaseName)$extension_prefix_prefix$i$extension_prefix_suffix$($pathname.Extension)" $new_content + $dst = "$($pathname.DirectoryName)/$($pathname.BaseName)$extension_prefix_prefix$i$extension_prefix_suffix$($pathname.Extension)" + Set-Content $dst $new_content + Write-Host -NoNewline "new replica: " + Write-Host -ForegroundColor Green $dst } } From cf0894725bae461da0e7294cfcfc1dbd43d8b3cd Mon Sep 17 00:00:00 2001 From: Andy Bayer Roswell Date: Thu, 27 Feb 2025 00:06:54 +0100 Subject: [PATCH 134/149] script for pf test --- vreapis/tests/perf/record-usage.ps1 | 98 +++++++++++++++++++---------- 1 file changed, 66 insertions(+), 32 deletions(-) diff --git a/vreapis/tests/perf/record-usage.ps1 b/vreapis/tests/perf/record-usage.ps1 index 9e89c72d..a8f91be2 100644 --- a/vreapis/tests/perf/record-usage.ps1 +++ b/vreapis/tests/perf/record-usage.ps1 @@ -1,13 +1,15 @@ param( - [string]$browser_program_name = $null, # record resource usage for browser. use this param to specify re pattern for browser pathname - [switch]$JupyterLab_backend = $false, # record resource usage for JupyterLab backend - [switch]$RStudio_backend = $false, # record resource usage for RStudio backend - [string]$vreapi_pod_name = $null, # record resource usage for vreapi pod [common backend]. use this param to specify re pattern for pod name - [string]$database_pod_name = $null, # record resource usage for db pod [common backend]. use this param to specify re pattern for pod name - [string]$log_dir = '.log', # directory to store log files - [Double]$interval = 1, # interval between 2 adjacent resource usage captures [seconds] - [long]$number_of_records = 0, # 0 or neg means infinite records, pos value means number of records to capture - [switch]$console = $false # print usage data to console + [string]$browser_process_filter = '', # record resource usage for browser. use this param to specify re pattern to filter browser processes from ps entries + [switch]$JupyterLab_backend = $false, # record resource usage for JupyterLab backend + [switch]$RStudio_backend = $false, # record resource usage for RStudio backend + [alias('v')][string]$vreapi_process_filter = '', # record resource usage for vreapi [common backend]. use this param to specify re pattern to filter vreapi process from ps entries + [alias('d')][string]$database_process_filter = '', # record resource usage for db. use this param to specify re pattern to filter db processes from ps entries + [alias('vp')][string]$vreapi_pod_filter = '', # record resource usage for vreapi pod [common backend]. use this param to specify re pattern to filter vreapi pod from kubectl top pod entries + [alias('dp')][string]$database_pod_filter = '', # record resource usage for db pod [common backend]. use this param to specify re pattern to filter db pod from kubectl top pod entries + [string]$log_dir = '.log', # directory to store log files + [Double]$interval = 1, # interval between 2 adjacent resource usage captures [seconds] + [long]$number_of_records = 0, # 0 or negative means infinite records. positive means number of records to capture + [switch]$console = $false # print usage data to console ) if ($console) { @@ -92,9 +94,9 @@ $compound_resource_metric = [PSCustomObject]@{ "time" = $null } $cpu_headers = $mem_headers = @() -if ($browser_program_name -ne $null) { - $cpu_headers += "CPU:browser:$browser_program_name" - $mem_headers += "mem:browser:$browser_program_name" +if ($browser_process_filter -ne '') { + $cpu_headers += "CPU:browser:$browser_process_filter" + $mem_headers += "mem:browser:$browser_process_filter" } if ($JupyterLab_backend) { $cpu_headers += "CPU:JupyterLab backend" @@ -104,15 +106,23 @@ if ($RStudio_backend) { $cpu_headers += "CPU:RStudio backend" $mem_headers += "mem:RStudio backend" } -if ($vreapi_pod_name -ne $null) { - $vreapi_pod_name = ((kubectl get pod | Select-String $vreapi_pod_name).Line -split '\s+')[0] - $cpu_headers += "CPU:pod:$vreapi_pod_name" - $mem_headers += "mem:pod:$vreapi_pod_name" +if ($vreapi_process_filter -ne '') { + $cpu_headers += "CPU:vreapi:$vreapi_process_filter" + $mem_headers += "mem:vreapi:$vreapi_process_filter" } -if ($database_pod_name -ne $null) { - $database_pod_name = ((kubectl get pod | Select-String $database_pod_name).Line -split '\s+')[0] - $cpu_headers += "CPU:pod:$database_pod_name" - $mem_headers += "mem:pod:$database_pod_name" +if ($database_process_filter -ne '') { + $cpu_headers += "CPU:database:$database_process_filter" + $mem_headers += "mem:database:$database_process_filter" +} +if ($vreapi_pod_filter -ne '') { + $vreapi_pod_filter = ((kubectl get pod | Select-String $vreapi_pod_filter).Line -split '\s+')[0] + $cpu_headers += "CPU:pod:$vreapi_pod_filter" + $mem_headers += "mem:pod:$vreapi_pod_filter" +} +if ($database_pod_filter -ne '') { + $database_pod_filter = ((kubectl get pod | Select-String $database_pod_filter).Line -split '\s+')[0] + $cpu_headers += "CPU:pod:$database_pod_filter" + $mem_headers += "mem:pod:$database_pod_filter" } $cpu_headers + $mem_headers | ForEach-Object { $compound_resource_metric | Add-Member -MemberType NoteProperty -Name $_ -Value $null } if ((-not $console) -and (-not (Test-Path $cooked_log_file))) { $compound_resource_metric | ConvertTo-Csv | Select-Object -First 1 | Out-File $cooked_log_file } # add csv headers first @@ -124,8 +134,8 @@ $loop_body = { $time = Get-Date -Format $time_format $compound_resource_metric.time = $time $new_raw_ps_entries = New-Object System.Collections.Generic.List[Simplified_ps_Entry] - if ($browser_program_name -ne $null) { - $browser_processes = & $ps_pathname 'exo' $([Simplified_ps_Entry]::header_row) | Select-String -a $browser_program_name + if ($browser_process_filter -ne '') { + $browser_processes = & $ps_pathname 'exo' $([Simplified_ps_Entry]::header_row) | Select-String -a $browser_process_filter $browser_metric = [Resource_Metric]::new() foreach ($process in $browser_processes) { $m = [Simplified_ps_Entry]::new($time, $process) @@ -133,8 +143,8 @@ $loop_body = { $browser_metric.CPU += $m.CPU $browser_metric.mem += $m.RSS * 1000 / 1024.0 / 1024 } - $compound_resource_metric."CPU:browser:$browser_program_name" = $browser_metric.CPU - $compound_resource_metric."mem:browser:$browser_program_name" = $browser_metric.mem + $compound_resource_metric."CPU:browser:$browser_process_filter" = $browser_metric.CPU + $compound_resource_metric."mem:browser:$browser_process_filter" = $browser_metric.mem } if ($JupyterLab_backend) { $JupyterLab_backend_processes = & $ps_pathname 'exo' $([Simplified_ps_Entry]::header_row) | Select-String -a 'jupyter.?lab' @@ -160,21 +170,45 @@ $loop_body = { $compound_resource_metric."CPU:RStudio backend" = $RStudio_backend_metric.CPU $compound_resource_metric."mem:RStudio backend" = $RStudio_backend_metric.mem } - if ($vreapi_pod_name -ne $null) { - $pod_metric = kubectl top pod $vreapi_pod_name --no-headers | ForEach-Object { + if ($vreapi_process_filter -ne '') { + $vreapi_processes = & $ps_pathname 'exo' $([Simplified_ps_Entry]::header_row) | Select-String -a $vreapi_process_filter + $vreapi_metric = [Resource_Metric]::new() + foreach ($process in $vreapi_processes) { + $m = [Simplified_ps_Entry]::new($time, $process) + $new_raw_ps_entries.Add($m) + $vreapi_metric.CPU += $m.CPU + $vreapi_metric.mem += $m.RSS * 1000 / 1024.0 / 1024 + } + $compound_resource_metric."CPU:vreapi:$vreapi_process_filter" = $vreapi_metric.CPU + $compound_resource_metric."mem:vreapi:$vreapi_process_filter" = $vreapi_metric.mem + } + if ($database_process_filter -ne '') { + $database_processes = & $ps_pathname 'axo' $([Simplified_ps_Entry]::header_row) | Select-String -a $database_process_filter # axo works fine to filter postgres processes. not sure when it comes to other db systems + $database_metric = [Resource_Metric]::new() + foreach ($process in $database_processes) { + $m = [Simplified_ps_Entry]::new($time, $process) + $new_raw_ps_entries.Add($m) + $database_metric.CPU += $m.CPU + $database_metric.mem += $m.RSS * 1000 / 1024.0 / 1024 + } + $compound_resource_metric."CPU:database:$database_process_filter" = $database_metric.CPU + $compound_resource_metric."mem:database:$database_process_filter" = $database_metric.mem + } + if ($vreapi_pod_filter -ne '') { + $pod_metric = kubectl top pod $vreapi_pod_filter --no-headers | ForEach-Object { $col = $_ -split '\s+' [Resource_Metric]::new([double]($col[1].substring(0, $col[1].Length - 'm'.Length)) / 10.0, [double]($col[2]).substring(0, $col[2].Length - 'Mi'.Length)) } - $compound_resource_metric."CPU:pod:$vreapi_pod_name" = $pod_metric.CPU - $compound_resource_metric."mem:pod:$vreapi_pod_name" = $pod_metric.mem + $compound_resource_metric."CPU:pod:$vreapi_pod_filter" = $pod_metric.CPU + $compound_resource_metric."mem:pod:$vreapi_pod_filter" = $pod_metric.mem } - if ($database_pod_name -ne $null) { - $pod_metric = kubectl top pod $database_pod_name --no-headers | ForEach-Object { + if ($database_pod_filter -ne '') { + $pod_metric = kubectl top pod $database_pod_filter --no-headers | ForEach-Object { $col = $_ -split '\s+' [Resource_Metric]::new([double]($col[1].substring(0, $col[1].Length - 'm'.Length)) / 10.0, [double]($col[2]).substring(0, $col[2].Length - 'Mi'.Length)) } - $compound_resource_metric."CPU:pod:$database_pod_name" = $pod_metric.CPU - $compound_resource_metric."mem:pod:$database_pod_name" = $pod_metric.mem + $compound_resource_metric."CPU:pod:$database_pod_filter" = $pod_metric.CPU + $compound_resource_metric."mem:pod:$database_pod_filter" = $pod_metric.mem } if ($console) { $compound_resource_metric | Format-Table | Write-Output From 72c52159ff98289d2585c5d59ccef3dbfb7c8059 Mon Sep 17 00:00:00 2001 From: Andy Bayer Roswell Date: Fri, 4 Apr 2025 01:12:08 +0200 Subject: [PATCH 135/149] [pf test] DEBUG = False --- vreapis/vreapis/settings/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vreapis/vreapis/settings/base.py b/vreapis/vreapis/settings/base.py index 04eb5c7a..09e72d9c 100644 --- a/vreapis/vreapis/settings/base.py +++ b/vreapis/vreapis/settings/base.py @@ -18,7 +18,7 @@ SECRET_KEY = get_random_secret_key() -DEBUG = True +DEBUG = False ALLOWED_HOSTS = ['*'] From f84153b7bb6fac5c9b9117b9d55f58fb7ce3c893 Mon Sep 17 00:00:00 2001 From: Andy Bayer Roswell Date: Fri, 4 Apr 2025 01:12:55 +0200 Subject: [PATCH 136/149] comment --- vreapis/tests/extract-notebook.ps1 | 1 + 1 file changed, 1 insertion(+) diff --git a/vreapis/tests/extract-notebook.ps1 b/vreapis/tests/extract-notebook.ps1 index 3b0d5568..60b12d3d 100644 --- a/vreapis/tests/extract-notebook.ps1 +++ b/vreapis/tests/extract-notebook.ps1 @@ -1,3 +1,4 @@ +# extract notebook contents from test json files param( [System.IO.FileInfo[]]$input_files, [System.IO.FileInfo]$output_dir From fb9ba2b444468af88210eb394af12743c2f2e650 Mon Sep 17 00:00:00 2001 From: Andy Bayer Roswell Date: Wed, 2 Jul 2025 00:16:20 +0200 Subject: [PATCH 137/149] refined performance recording script --- .../emulated-frontend/dat/addcell.1.json | 6 +- .../emulated-frontend/dat/addcell.2.json | 6 +- vreapis/tests/perf/rec-utils.ps1 | 168 ++++++++++++++++++ vreapis/tests/perf/record-usage.ps1 | 1 + 4 files changed, 175 insertions(+), 6 deletions(-) create mode 100644 vreapis/tests/perf/rec-utils.ps1 diff --git a/vreapis/tests/emulated-frontend/dat/addcell.1.json b/vreapis/tests/emulated-frontend/dat/addcell.1.json index e8b2e835..9bfc8e78 100644 --- a/vreapis/tests/emulated-frontend/dat/addcell.1.json +++ b/vreapis/tests/emulated-frontend/dat/addcell.1.json @@ -34,7 +34,7 @@ "names" ], "params": [], - "title": "input-names-user", + "title": "input-names-user0", "vars": [ { "color": "#ac5397", @@ -105,8 +105,8 @@ ], "param_values": {}, "params": [], - "task_name": "input-names-user", - "title": "input-names-user", + "task_name": "input-names-user0", + "title": "input-names-user0", "types": { "names": "list" } diff --git a/vreapis/tests/emulated-frontend/dat/addcell.2.json b/vreapis/tests/emulated-frontend/dat/addcell.2.json index 12a23701..8223d09b 100644 --- a/vreapis/tests/emulated-frontend/dat/addcell.2.json +++ b/vreapis/tests/emulated-frontend/dat/addcell.2.json @@ -34,7 +34,7 @@ "og_node_id": "603faa7", "outputs": [], "params": [], - "title": "print-names-user", + "title": "print-names-user0", "vars": [ { "color": "#ac5397", @@ -105,8 +105,8 @@ "outputs": [], "param_values": {}, "params": [], - "task_name": "print-names-user", - "title": "print-names-user", + "task_name": "print-names-user0", + "title": "print-names-user0", "types": { "names": "list" } diff --git a/vreapis/tests/perf/rec-utils.ps1 b/vreapis/tests/perf/rec-utils.ps1 new file mode 100644 index 00000000..873fea2d --- /dev/null +++ b/vreapis/tests/perf/rec-utils.ps1 @@ -0,0 +1,168 @@ +param( + [string]$browser_process_filter = '', # record resource usage for browser. use this param to specify re pattern to filter browser processes from ps entries + [switch]$JupyterLab_backend = $false, # record resource usage for JupyterLab backend + [switch]$RStudio_backend = $false, # record resource usage for RStudio backend + [alias('v')][string]$vreapi_process_filter = '', # record resource usage for vreapi [common backend]. use this param to specify re pattern to filter vreapi process from ps entries + [alias('d')][string]$database_process_filter = '', # record resource usage for db. use this param to specify re pattern to filter db processes from ps entries + [alias('vp')][string]$vreapi_pod_filter = '', # record resource usage for vreapi pod [common backend]. use this param to specify re pattern to filter vreapi pod from kubectl top pod entries + [alias('dp')][string]$database_pod_filter = '', # record resource usage for db pod [common backend]. use this param to specify re pattern to filter db pod from kubectl top pod entries + [string]$log_dir = '.log', # directory to store log files + [Double]$interval = 1, # interval between 2 adjacent resource usage captures [seconds] + [long]$number_of_records = 0, # 0 or negative means infinite records. positive means number of records to capture + [switch]$console = $false # print usage data to console +) + +# intro messages before recording +if ($console) { + Write-Host -NoNewline 'Recording CPU & mem usage at ' + Write-Host -NoNewline -ForegroundColor Green 'console' + Write-Host -NoNewline ' every ' + Write-Host -NoNewline -ForegroundColor Green $interval + Write-Host ' second(s) ...' +} else { + if ((Test-Path $log_dir) -eq $false) { & '/usr/bin/mkdir' -p $log_dir } + + $date = Get-Date -Format 'yyyyMMdd-HHmmss' + $raw_log_file = $log_dir + "/$date.raw.csv" + $cooked_log_file = $log_dir + "/$date.cooked.csv" + + Write-Host -NoNewline 'Recording CPU & mem usage at ' + Write-Host -NoNewline -ForegroundColor Green $raw_log_file + Write-Host -NoNewline ' and ' + Write-Host -NoNewline -ForegroundColor Green $cooked_log_file + Write-Host -NoNewline ' every ' + Write-Host -NoNewline -ForegroundColor Green $interval + Write-Host ' second(s) ...' +} + +class Process_CPU_Entry { # pidstat used here. ps shows average CPU util only while top always show units like m, g, t, p for mem util thus less precise + # columns for CPU util: [time], UID, PID, %usr, %system, %guest, %wait, %CPU, CPU, Command + [DateTime]$time + [int]$UID = -1 + [int]$PID = -1 + [double]${%usr} = 0 + [double]${%system} = 0 + [double]${%guest} = 0 + [double]${%wait} = 0 + [double]${%CPU} = 0 + [int]$CPU = 0 + [string]$Command = '' + Process_CPU_Entry([DateTime]$time, [int]$UID, [int]$PID, [double]${%usr}, [double]${%system}, [double]${%guest}, [double]${%wait}, [double]${%CPU}, [int]$CPU, [string]$Command) { + $this.time = $time + $this.UID = $UID + $this.PID = $PID + $this.${%usr} = ${%usr} + $this.${%system} = ${%system} + $this.${%guest} = ${%guest} + $this.${%wait} = ${%wait} + $this.${%CPU} = ${%CPU} + $this.CPU = $CPU + $this.Command = $Command + } +} + +class Process_Memory_Entry { + # columns for mem util: [time], UID, PID, minflt/s, majflt/s, VSZ, RSS, %MEM, Command + [DateTime]$time + [int]$UID = -1 + [int]$PID = -1 + [double]${minflt/s} = 0 + [double]${majflt/s} = 0 + [int]$VSZ = 0 # in KiB + [int]$RSS = 0 # in KiB + [double]${%MEM} = 0 + [string]$Command = '' + Process_Memory_Entry([DateTime]$time, [int]$UID, [int]$PID, [double]${minflt/s}, [double]${majflt/s}, [int]$VSZ, [int]$RSS, [double]${%MEM}, [string]$Command) { + $this.time = $time + $this.UID = $UID + $this.PID = $PID + $this.${minflt/s} = ${minflt/s} + $this.${majflt/s} = ${majflt/s} + $this.VSZ = $VSZ + $this.RSS = $RSS + $this.${%MEM} = ${%MEM} + $this.Command = $Command + } +} + +class Resource_Metric { # for what are sent by K8s metrics server, and cooked entries + [double]$CPU # % + [double]$mem # Mi + Resource_Metric() { + $this.CPU = 0 + $this.mem = 0 + } + Resource_Metric([double]$cpu, [double]$mem) { + $this.CPU = $cpu + $this.mem = $mem + } +} + +if ((-not $console) -and (-not (Test-Path $raw_log_file))) { (@('time') + [Process_CPU_Entry]::col_name) -join ',' | Out-File $raw_log_file } # add csv headers first + +# columns of each row +$cooked_entry = [PSCustomObject]@{ + "time" = $null +} +$CPU_headers = New-Object System.Collections.Generic.List[string] +$mem_headers = New-Object System.Collections.Generic.List[string] +if ($browser_process_filter -ne '') { + $CPU_headers += "CPU:browser:$browser_process_filter" + $mem_headers += "mem:browser:$browser_process_filter" +} +if ($JupyterLab_backend) { + $CPU_headers += "CPU:JupyterLab backend" + $mem_headers += "mem:JupyterLab backend" +} +if ($RStudio_backend) { + $CPU_headers += "CPU:RStudio backend" + $mem_headers += "mem:RStudio backend" +} +if ($vreapi_process_filter -ne '') { + $CPU_headers += "CPU:vreapi:$vreapi_process_filter" + $mem_headers += "mem:vreapi:$vreapi_process_filter" +} +if ($database_process_filter -ne '') { + $CPU_headers += "CPU:database:$database_process_filter" + $mem_headers += "mem:database:$database_process_filter" +} +if ($vreapi_pod_filter -ne '') { + $vreapi_pod_filter = ((kubectl get pod | Select-String $vreapi_pod_filter).Line -split '\s+')[0] + $CPU_headers += "CPU:pod:$vreapi_pod_filter" + $mem_headers += "mem:pod:$vreapi_pod_filter" +} +if ($database_pod_filter -ne '') { + $database_pod_filter = ((kubectl get pod | Select-String $database_pod_filter).Line -split '\s+')[0] + $CPU_headers += "CPU:pod:$database_pod_filter" + $mem_headers += "mem:pod:$database_pod_filter" +} +$CPU_headers + $mem_headers | ForEach-Object { $cooked_entry | Add-Member -MemberType NoteProperty -Name $_ -Value $null } +if ((-not $console) -and (-not (Test-Path $cooked_log_file))) { $cooked_entry | ConvertTo-Csv | Select-Object -First 1 | Out-File $cooked_log_file } # add csv headers first + +$time_format = 'yyyy-MM-dd HH:mm:ss.fff' +$loop_body = { + $time = Get-Date -Format $time_format + $cooked_entry.time = $time + $process_info = (pidstat -ru -p ALL 0) -join "`n" -split "\n\n" | Select-Object -Skip 1 + $util_rows = New-Object System.Collections.Generic.List[System.Collections.Generic.List[string[]]] + for ($i = 0; $i -lt 2; ++$i) { + $raw_rows = $process_info[$i] -split '\n' | Select-Object -Skip 1 + $split_rows = New-Object System.Collections.Generic.List[string[]] + foreach ($raw_row in $raw_rows) { + $columns = $raw_row -split '\s+' + $split_rows.Add($columns) + } + $util_rows.Add($split_rows) + } + if ($browser_process_filter -ne '') { + + } + Start-Sleep($interval) +} + +# main +if ($number_of_records -le 0) { + while ($true) { & $loop_body } +} else { + for ($i = 0; $i -lt $number_of_records; ++$i) { & $loop_body } +} diff --git a/vreapis/tests/perf/record-usage.ps1 b/vreapis/tests/perf/record-usage.ps1 index a8f91be2..d28cfceb 100644 --- a/vreapis/tests/perf/record-usage.ps1 +++ b/vreapis/tests/perf/record-usage.ps1 @@ -35,6 +35,7 @@ if ($console) { } class Simplified_ps_Entry { + static [string] $header_row = 'user,pid,%cpu,rss,command' # complete header row: 'USER,PID,%CPU,%MEM,VSZ,RSS,TTY,STAT,START,TIME,COMMAND' static [string[]] $col_name = [Simplified_ps_Entry]::header_row -replace '%', '' -split ',' static $col_name_index = @{} From 18d4a9a768ab954eb7b7b093bb8d13067f73364a Mon Sep 17 00:00:00 2001 From: Andy Bayer Roswell Date: Wed, 2 Jul 2025 02:00:16 +0200 Subject: [PATCH 138/149] refined performance recording script --- vreapis/tests/perf/rec-utils.ps1 | 105 +++++++++++++++++++++---------- 1 file changed, 72 insertions(+), 33 deletions(-) diff --git a/vreapis/tests/perf/rec-utils.ps1 b/vreapis/tests/perf/rec-utils.ps1 index 873fea2d..6dc318d9 100644 --- a/vreapis/tests/perf/rec-utils.ps1 +++ b/vreapis/tests/perf/rec-utils.ps1 @@ -1,15 +1,15 @@ param( - [string]$browser_process_filter = '', # record resource usage for browser. use this param to specify re pattern to filter browser processes from ps entries - [switch]$JupyterLab_backend = $false, # record resource usage for JupyterLab backend - [switch]$RStudio_backend = $false, # record resource usage for RStudio backend - [alias('v')][string]$vreapi_process_filter = '', # record resource usage for vreapi [common backend]. use this param to specify re pattern to filter vreapi process from ps entries - [alias('d')][string]$database_process_filter = '', # record resource usage for db. use this param to specify re pattern to filter db processes from ps entries - [alias('vp')][string]$vreapi_pod_filter = '', # record resource usage for vreapi pod [common backend]. use this param to specify re pattern to filter vreapi pod from kubectl top pod entries - [alias('dp')][string]$database_pod_filter = '', # record resource usage for db pod [common backend]. use this param to specify re pattern to filter db pod from kubectl top pod entries - [string]$log_dir = '.log', # directory to store log files - [Double]$interval = 1, # interval between 2 adjacent resource usage captures [seconds] - [long]$number_of_records = 0, # 0 or negative means infinite records. positive means number of records to capture - [switch]$console = $false # print usage data to console + [string]$browser_process_filter = '', # record resource usage for browser. use this param to specify re pattern to filter browser processes from ps entries + [switch]$JupyterLab_backend = $false, # record resource usage for JupyterLab backend + [switch]$RStudio_backend = $false, # record resource usage for RStudio backend + [alias('v')][string]$vreapi_process_filter = '', # record resource usage for vreapi [common backend]. use this param to specify re pattern to filter vreapi process from ps entries + [alias('d')][string]$database_process_filter = '', # record resource usage for db. use this param to specify re pattern to filter db processes from ps entries + [alias('vp')][string]$vreapi_pod_filter = '', # record resource usage for vreapi pod [common backend]. use this param to specify re pattern to filter vreapi pod from kubectl top pod entries + [alias('dp')][string]$database_pod_filter = '', # record resource usage for db pod [common backend]. use this param to specify re pattern to filter db pod from kubectl top pod entries + [string]$log_dir = '.log', # directory to store log files + [Double]$interval = 1, # interval between 2 adjacent resource usage captures [seconds] + [long]$number_of_records = 0, # 0 or negative means infinite records. positive means number of records to capture + [switch]$console = $false # print usage data to console ) # intro messages before recording @@ -35,52 +35,52 @@ if ($console) { Write-Host ' second(s) ...' } -class Process_CPU_Entry { # pidstat used here. ps shows average CPU util only while top always show units like m, g, t, p for mem util thus less precise +class Process_CPU_Entry { # pidstat [sudo apt install sysstat] used here. ps shows average CPU util only while top always show units like m, g, t, p for mem util thus less precise # columns for CPU util: [time], UID, PID, %usr, %system, %guest, %wait, %CPU, CPU, Command [DateTime]$time [int]$UID = -1 [int]$PID = -1 - [double]${%usr} = 0 - [double]${%system} = 0 - [double]${%guest} = 0 - [double]${%wait} = 0 - [double]${%CPU} = 0 + [double]$pct_usr = 0 + [double]$pct_system = 0 + [double]$pct_guest = 0 + [double]$pct_wait = 0 + [double]$pct_CPU = 0 [int]$CPU = 0 [string]$Command = '' - Process_CPU_Entry([DateTime]$time, [int]$UID, [int]$PID, [double]${%usr}, [double]${%system}, [double]${%guest}, [double]${%wait}, [double]${%CPU}, [int]$CPU, [string]$Command) { + Process_CPU_Entry([DateTime]$time, [int]$UID, [int]$_PID, [double]$pct_usr, [double]$pct_system, [double]$pct_guest, [double]$pct_wait, [double]$pct_CPU, [int]$CPU, [string]$Command) { $this.time = $time $this.UID = $UID - $this.PID = $PID - $this.${%usr} = ${%usr} - $this.${%system} = ${%system} - $this.${%guest} = ${%guest} - $this.${%wait} = ${%wait} - $this.${%CPU} = ${%CPU} + $this.PID = $_PID + $this.pct_usr = $pct_usr + $this.pct_system = $pct_system + $this.pct_guest = $pct_guest + $this.pct_wait = $pct_wait + $this.pct_CPU = $pct_CPU $this.CPU = $CPU $this.Command = $Command } } -class Process_Memory_Entry { +class Process_Memory_Entry { # pidstat # columns for mem util: [time], UID, PID, minflt/s, majflt/s, VSZ, RSS, %MEM, Command [DateTime]$time [int]$UID = -1 [int]$PID = -1 - [double]${minflt/s} = 0 - [double]${majflt/s} = 0 + [double]$minflt_p_s = 0 + [double]$majflt_p_s = 0 [int]$VSZ = 0 # in KiB [int]$RSS = 0 # in KiB - [double]${%MEM} = 0 + [double]$pct_MEM = 0 [string]$Command = '' - Process_Memory_Entry([DateTime]$time, [int]$UID, [int]$PID, [double]${minflt/s}, [double]${majflt/s}, [int]$VSZ, [int]$RSS, [double]${%MEM}, [string]$Command) { + Process_Memory_Entry([DateTime]$time, [int]$UID, [int]$_PID, [double]$minflt_p_s, [double]$majflt_p_s, [int]$VSZ, [int]$RSS, [double]$pct_MEM, [string]$Command) { $this.time = $time $this.UID = $UID - $this.PID = $PID - $this.${minflt/s} = ${minflt/s} - $this.${majflt/s} = ${majflt/s} + $this.PID = $_PID + $this.minflt_p_s = $minflt_p_s + $this.majflt_p_s = $majflt_p_s $this.VSZ = $VSZ $this.RSS = $RSS - $this.${%MEM} = ${%MEM} + $this.pct_MEM = $pct_MEM $this.Command = $Command } } @@ -140,6 +140,12 @@ $CPU_headers + $mem_headers | ForEach-Object { $cooked_entry | Add-Member -Membe if ((-not $console) -and (-not (Test-Path $cooked_log_file))) { $cooked_entry | ConvertTo-Csv | Select-Object -First 1 | Out-File $cooked_log_file } # add csv headers first $time_format = 'yyyy-MM-dd HH:mm:ss.fff' +if ($env:LC_NUMERIC -eq $null) { + $culture = [System.Globalization.CultureInfo]::CurrentCulture.Name +} else { + $BCP_47_culture_name_for_pidstat = $env:LC_NUMERIC.Substring(0, $env:LC_NUMERIC.IndexOf('.')) # e.g. en-US, nl-NL + $culture = New-Object System.Globalization.CultureInfo($BCP_47_culture_name_for_pidstat) +} $loop_body = { $time = Get-Date -Format $time_format $cooked_entry.time = $time @@ -154,6 +160,39 @@ $loop_body = { } $util_rows.Add($split_rows) } + $CPU_util_rows = $util_rows[0] + $mem_util_rows = $util_rows[1] + $CPU_entries = New-Object System.Collections.Generic.List[Process_CPU_Entry] + $mem_entries = New-Object System.Collections.Generic.List[Process_Memory_Entry] + foreach ($seg in $CPU_util_rows) { + $e = [Process_CPU_Entry]::new( + [DateTime]::ParseExact($time, $time_format, $null), + [int]($seg[1]), + [int]($seg[2]), + [double]::Parse($seg[3], $culture), + [double]::Parse($seg[4], $culture), + [double]::Parse($seg[5], $culture), + [double]::Parse($seg[6], $culture), + [double]::Parse($seg[7], $culture), + [int]($seg[8]), + $seg[9..($seg.Length - 1)] -join ' ' + ) + $CPU_entries.Add($e) + } + foreach ($seg in $mem_util_rows) { + $e = [Process_Memory_Entry]::new( + [DateTime]::ParseExact($time, $time_format, $null), + [int]($seg[1]), + [int]($seg[2]), + [double]::Parse($seg[3], $culture), + [double]::Parse($seg[4], $culture), + [int]($seg[5]), + [int]($seg[6]), + [double]::Parse($seg[7], $culture), + $seg[8..($seg.Length - 1)] -join ' ' + ) + $mem_entries.Add($e) + } if ($browser_process_filter -ne '') { } From 97983557abca79bbfee5f7c8490aad4132afdf20 Mon Sep 17 00:00:00 2001 From: Andy Bayer Roswell Date: Wed, 2 Jul 2025 02:50:35 +0200 Subject: [PATCH 139/149] refined performance recording script --- vreapis/tests/perf/rec-utils.ps1 | 108 +++++++++++++++++++++++++++++-- 1 file changed, 101 insertions(+), 7 deletions(-) diff --git a/vreapis/tests/perf/rec-utils.ps1 b/vreapis/tests/perf/rec-utils.ps1 index 6dc318d9..93b67dcd 100644 --- a/vreapis/tests/perf/rec-utils.ps1 +++ b/vreapis/tests/perf/rec-utils.ps1 @@ -7,7 +7,7 @@ param( [alias('vp')][string]$vreapi_pod_filter = '', # record resource usage for vreapi pod [common backend]. use this param to specify re pattern to filter vreapi pod from kubectl top pod entries [alias('dp')][string]$database_pod_filter = '', # record resource usage for db pod [common backend]. use this param to specify re pattern to filter db pod from kubectl top pod entries [string]$log_dir = '.log', # directory to store log files - [Double]$interval = 1, # interval between 2 adjacent resource usage captures [seconds] + [double]$interval = 1, # interval between 2 adjacent resource usage captures [seconds] [long]$number_of_records = 0, # 0 or negative means infinite records. positive means number of records to capture [switch]$console = $false # print usage data to console ) @@ -23,11 +23,14 @@ if ($console) { if ((Test-Path $log_dir) -eq $false) { & '/usr/bin/mkdir' -p $log_dir } $date = Get-Date -Format 'yyyyMMdd-HHmmss' - $raw_log_file = $log_dir + "/$date.raw.csv" + $CPU_log_file = $log_dir + "/$date.CPU.csv" + $mem_log_file = $log_dir + "/$date.mem.csv" $cooked_log_file = $log_dir + "/$date.cooked.csv" Write-Host -NoNewline 'Recording CPU & mem usage at ' - Write-Host -NoNewline -ForegroundColor Green $raw_log_file + Write-Host -NoNewline -ForegroundColor Green $CPU_log_file + Write-Host -NoNewline ' and ' + Write-Host -NoNewline -ForegroundColor Green $mem_log_file Write-Host -NoNewline ' and ' Write-Host -NoNewline -ForegroundColor Green $cooked_log_file Write-Host -NoNewline ' every ' @@ -98,9 +101,7 @@ class Resource_Metric { # for what are sent by K8s metrics server, and cooked en } } -if ((-not $console) -and (-not (Test-Path $raw_log_file))) { (@('time') + [Process_CPU_Entry]::col_name) -join ',' | Out-File $raw_log_file } # add csv headers first - -# columns of each row +# columns for cooked entries $cooked_entry = [PSCustomObject]@{ "time" = $null } @@ -139,7 +140,10 @@ if ($database_pod_filter -ne '') { $CPU_headers + $mem_headers | ForEach-Object { $cooked_entry | Add-Member -MemberType NoteProperty -Name $_ -Value $null } if ((-not $console) -and (-not (Test-Path $cooked_log_file))) { $cooked_entry | ConvertTo-Csv | Select-Object -First 1 | Out-File $cooked_log_file } # add csv headers first +# main body $time_format = 'yyyy-MM-dd HH:mm:ss.fff' +$JupyterLab_backend_filter = 'jupyter.?lab' +$RStudio_backend_filter = 'rstudio-server' if ($env:LC_NUMERIC -eq $null) { $culture = [System.Globalization.CultureInfo]::CurrentCulture.Name } else { @@ -147,8 +151,11 @@ if ($env:LC_NUMERIC -eq $null) { $culture = New-Object System.Globalization.CultureInfo($BCP_47_culture_name_for_pidstat) } $loop_body = { + # current time point $time = Get-Date -Format $time_format $cooked_entry.time = $time + + # convert text-based results from pidstat to .NET objects $process_info = (pidstat -ru -p ALL 0) -join "`n" -split "\n\n" | Select-Object -Skip 1 $util_rows = New-Object System.Collections.Generic.List[System.Collections.Generic.List[string[]]] for ($i = 0; $i -lt 2; ++$i) { @@ -163,7 +170,9 @@ $loop_body = { $CPU_util_rows = $util_rows[0] $mem_util_rows = $util_rows[1] $CPU_entries = New-Object System.Collections.Generic.List[Process_CPU_Entry] + $CPU_entries_to_export = New-Object System.Collections.Generic.List[Process_CPU_Entry] $mem_entries = New-Object System.Collections.Generic.List[Process_Memory_Entry] + $mem_entries_to_export = New-Object System.Collections.Generic.List[Process_Memory_Entry] foreach ($seg in $CPU_util_rows) { $e = [Process_CPU_Entry]::new( [DateTime]::ParseExact($time, $time_format, $null), @@ -193,9 +202,94 @@ $loop_body = { ) $mem_entries.Add($e) } - if ($browser_process_filter -ne '') { + # filter entries + if ($browser_process_filter -ne '') { + $browser_CPU_entries = $CPU_entries | Where-Object { $_.Command -match $browser_process_filter } + $CPU_entries_to_export.AddRange($browser_CPU_entries) + $browser_mem_entries = $mem_entries | Where-Object { $_.Command -match $browser_process_filter } + $mem_entries_to_export.AddRange($browser_mem_entries) + $browser_metric = [Resource_Metric]::new( + ($browser_CPU_entries | Measure-Object -Property pct_CPU -Sum).Sum, + ($browser_mem_entries | Measure-Object -Property RSS -Sum).Sum / 1024.0 # convert KiB to MiB + ) + $cooked_entry."CPU:browser:$browser_process_filter" = $browser_metric.CPU + $cooked_entry."mem:browser:$browser_process_filter" = $browser_metric.mem + } + if ($JupyterLab_backend) { + $JupyterLab_CPU_entries = $CPU_entries | Where-Object { $_.Command -match $JupyterLab_backend_filter } + $CPU_entries_to_export.AddRange($JupyterLab_CPU_entries) + $JupyterLab_mem_entries = $mem_entries | Where-Object { $_.Command -match $JupyterLab_backend_filter } + $mem_entries_to_export.AddRange($JupyterLab_mem_entries) + $JupyterLab_metric = [Resource_Metric]::new( + ($JupyterLab_CPU_entries | Measure-Object -Property pct_CPU -Sum).Sum, + ($JupyterLab_mem_entries | Measure-Object -Property RSS -Sum).Sum / 1024.0 # convert KiB to MiB + ) + $cooked_entry."CPU:JupyterLab backend" = $JupyterLab_metric.CPU + $cooked_entry."mem:JupyterLab backend" = $JupyterLab_metric.mem + } + if ($RStudio_backend) { + $RStudio_CPU_entries = $CPU_entries | Where-Object { $_.Command -match $RStudio_backend_filter } + $CPU_entries_to_export.AddRange($RStudio_CPU_entries) + $RStudio_mem_entries = $mem_entries | Where-Object { $_.Command -match $RStudio_backend_filter } + $mem_entries_to_export.AddRange($RStudio_mem_entries) + $RStudio_metric = [Resource_Metric]::new( + ($RStudio_CPU_entries | Measure-Object -Property pct_CPU -Sum).Sum, + ($RStudio_mem_entries | Measure-Object -Property RSS -Sum).Sum / 1024.0 # convert KiB to MiB + ) + $cooked_entry."CPU:RStudio backend" = $RStudio_metric.CPU + $cooked_entry."mem:RStudio backend" = $RStudio_metric.mem } + if ($vreapi_process_filter -ne '') { + $vreapi_CPU_entries = $CPU_entries | Where-Object { $_.Command -match $vreapi_process_filter } + $CPU_entries_to_export.AddRange($vreapi_CPU_entries) + $vreapi_mem_entries = $mem_entries | Where-Object { $_.Command -match $vreapi_process_filter } + $mem_entries_to_export.AddRange($vreapi_mem_entries) + $vreapi_metric = [Resource_Metric]::new( + ($vreapi_CPU_entries | Measure-Object -Property pct_CPU -Sum).Sum, + ($vreapi_mem_entries | Measure-Object -Property RSS -Sum).Sum / 1024.0 # convert KiB to MiB + ) + $cooked_entry."CPU:vreapi:$vreapi_process_filter" = $vreapi_metric.CPU + $cooked_entry."mem:vreapi:$vreapi_process_filter" = $vreapi_metric.mem + } + if ($database_process_filter -ne '') { + $database_CPU_entries = $CPU_entries | Where-Object { $_.Command -match $database_process_filter } + $CPU_entries_to_export.AddRange($database_CPU_entries) + $database_mem_entries = $mem_entries | Where-Object { $_.Command -match $database_process_filter } + $mem_entries_to_export.AddRange($database_mem_entries) + $database_metric = [Resource_Metric]::new( + ($database_CPU_entries | Measure-Object -Property pct_CPU -Sum).Sum, + ($database_mem_entries | Measure-Object -Property RSS -Sum).Sum / 1024.0 # convert KiB to MiB + ) + $cooked_entry."CPU:database:$database_process_filter" = $database_metric.CPU + $cooked_entry."mem:database:$database_process_filter" = $database_metric.mem + } + if ($vreapi_pod_filter -ne '') { + $pod_metric = kubectl top pod $vreapi_pod_filter --no-headers | ForEach-Object { + $col = $_ -split '\s+' + [Resource_Metric]::new([double]($col[1].substring(0, $col[1].Length - 'm'.Length)) / 10.0, [double]($col[2]).substring(0, $col[2].Length - 'Mi'.Length)) + } + $cooked_entry."CPU:pod:$vreapi_pod_filter" = $pod_metric.CPU + $cooked_entry."mem:pod:$vreapi_pod_filter" = $pod_metric.mem + } + if ($database_pod_filter -ne '') { + $pod_metric = kubectl top pod $database_pod_filter --no-headers | ForEach-Object { + $col = $_ -split '\s+' + [Resource_Metric]::new([double]($col[1].substring(0, $col[1].Length - 'm'.Length)) / 10.0, [double]($col[2]).substring(0, $col[2].Length - 'Mi'.Length)) + } + $cooked_entry."CPU:pod:$database_pod_filter" = $pod_metric.CPU + $cooked_entry."mem:pod:$database_pod_filter" = $pod_metric.mem + } + + # output + if ($console) { + $cooked_entry | Format-Table | Write-Output + } else { + $CPU_entries_to_export | Export-Csv -Append $CPU_log_file + $mem_entries_to_export | Export-Csv -Append $mem_log_file + $cooked_entry | Export-Csv -Append $cooked_log_file + } + Start-Sleep($interval) } From 8051c6463608d4b2f742478a45b9d4b5d464da16 Mon Sep 17 00:00:00 2001 From: Andy Bayer Roswell Date: Wed, 2 Jul 2025 11:08:26 +0200 Subject: [PATCH 140/149] refined performance recording script --- vreapis/tests/perf/rec-utils.ps1 | 135 +++++++++---------------------- 1 file changed, 39 insertions(+), 96 deletions(-) diff --git a/vreapis/tests/perf/rec-utils.ps1 b/vreapis/tests/perf/rec-utils.ps1 index 93b67dcd..0b7f4f3d 100644 --- a/vreapis/tests/perf/rec-utils.ps1 +++ b/vreapis/tests/perf/rec-utils.ps1 @@ -101,49 +101,48 @@ class Resource_Metric { # for what are sent by K8s metrics server, and cooked en } } -# columns for cooked entries +# add columns according to the designated process/pod filters $cooked_entry = [PSCustomObject]@{ "time" = $null } $CPU_headers = New-Object System.Collections.Generic.List[string] $mem_headers = New-Object System.Collections.Generic.List[string] -if ($browser_process_filter -ne '') { - $CPU_headers += "CPU:browser:$browser_process_filter" - $mem_headers += "mem:browser:$browser_process_filter" +$JupyterLab_backend_filter = 'jupyter.?lab' +$RStudio_backend_filter = 'rstudio-server' +$process_filters = New-Object System.Collections.Generic.List[string] +$pod_filters = New-Object System.Collections.Generic.List[string] +foreach ($s in @( + $browser_process_filter, + $vreapi_process_filter, + $database_process_filter +)) { + if ($s -ne '') { $process_filters.Add($s) } } if ($JupyterLab_backend) { - $CPU_headers += "CPU:JupyterLab backend" - $mem_headers += "mem:JupyterLab backend" + $process_filters.Add($JupyterLab_backend_filter) } if ($RStudio_backend) { - $CPU_headers += "CPU:RStudio backend" - $mem_headers += "mem:RStudio backend" -} -if ($vreapi_process_filter -ne '') { - $CPU_headers += "CPU:vreapi:$vreapi_process_filter" - $mem_headers += "mem:vreapi:$vreapi_process_filter" + $process_filters.Add($RStudio_backend_filter) } -if ($database_process_filter -ne '') { - $CPU_headers += "CPU:database:$database_process_filter" - $mem_headers += "mem:database:$database_process_filter" +foreach ($s in @( + $vreapi_pod_filter, + $database_pod_filter +)) { + if ($s -ne '') { $pod_filters.Add($s) } } -if ($vreapi_pod_filter -ne '') { - $vreapi_pod_filter = ((kubectl get pod | Select-String $vreapi_pod_filter).Line -split '\s+')[0] - $CPU_headers += "CPU:pod:$vreapi_pod_filter" - $mem_headers += "mem:pod:$vreapi_pod_filter" +foreach ($filter in $process_filters) { + $CPU_headers += "CPU:process:$filter" + $mem_headers += "mem:process:$filter" } -if ($database_pod_filter -ne '') { - $database_pod_filter = ((kubectl get pod | Select-String $database_pod_filter).Line -split '\s+')[0] - $CPU_headers += "CPU:pod:$database_pod_filter" - $mem_headers += "mem:pod:$database_pod_filter" +foreach ($filter in $pod_filters) { + $CPU_headers += "CPU:pod:$filter" + $mem_headers += "mem:pod:$filter" } $CPU_headers + $mem_headers | ForEach-Object { $cooked_entry | Add-Member -MemberType NoteProperty -Name $_ -Value $null } if ((-not $console) -and (-not (Test-Path $cooked_log_file))) { $cooked_entry | ConvertTo-Csv | Select-Object -First 1 | Out-File $cooked_log_file } # add csv headers first # main body $time_format = 'yyyy-MM-dd HH:mm:ss.fff' -$JupyterLab_backend_filter = 'jupyter.?lab' -$RStudio_backend_filter = 'rstudio-server' if ($env:LC_NUMERIC -eq $null) { $culture = [System.Globalization.CultureInfo]::CurrentCulture.Name } else { @@ -156,7 +155,7 @@ $loop_body = { $cooked_entry.time = $time # convert text-based results from pidstat to .NET objects - $process_info = (pidstat -ru -p ALL 0) -join "`n" -split "\n\n" | Select-Object -Skip 1 + $process_info = (pidstat -rul -p ALL 0) -join "`n" -split "\n\n" | Select-Object -Skip 1 $util_rows = New-Object System.Collections.Generic.List[System.Collections.Generic.List[string[]]] for ($i = 0; $i -lt 2; ++$i) { $raw_rows = $process_info[$i] -split '\n' | Select-Object -Skip 1 @@ -204,81 +203,25 @@ $loop_body = { } # filter entries - if ($browser_process_filter -ne '') { - $browser_CPU_entries = $CPU_entries | Where-Object { $_.Command -match $browser_process_filter } - $CPU_entries_to_export.AddRange($browser_CPU_entries) - $browser_mem_entries = $mem_entries | Where-Object { $_.Command -match $browser_process_filter } - $mem_entries_to_export.AddRange($browser_mem_entries) - $browser_metric = [Resource_Metric]::new( - ($browser_CPU_entries | Measure-Object -Property pct_CPU -Sum).Sum, - ($browser_mem_entries | Measure-Object -Property RSS -Sum).Sum / 1024.0 # convert KiB to MiB - ) - $cooked_entry."CPU:browser:$browser_process_filter" = $browser_metric.CPU - $cooked_entry."mem:browser:$browser_process_filter" = $browser_metric.mem - } - if ($JupyterLab_backend) { - $JupyterLab_CPU_entries = $CPU_entries | Where-Object { $_.Command -match $JupyterLab_backend_filter } - $CPU_entries_to_export.AddRange($JupyterLab_CPU_entries) - $JupyterLab_mem_entries = $mem_entries | Where-Object { $_.Command -match $JupyterLab_backend_filter } - $mem_entries_to_export.AddRange($JupyterLab_mem_entries) - $JupyterLab_metric = [Resource_Metric]::new( - ($JupyterLab_CPU_entries | Measure-Object -Property pct_CPU -Sum).Sum, - ($JupyterLab_mem_entries | Measure-Object -Property RSS -Sum).Sum / 1024.0 # convert KiB to MiB - ) - $cooked_entry."CPU:JupyterLab backend" = $JupyterLab_metric.CPU - $cooked_entry."mem:JupyterLab backend" = $JupyterLab_metric.mem - } - if ($RStudio_backend) { - $RStudio_CPU_entries = $CPU_entries | Where-Object { $_.Command -match $RStudio_backend_filter } - $CPU_entries_to_export.AddRange($RStudio_CPU_entries) - $RStudio_mem_entries = $mem_entries | Where-Object { $_.Command -match $RStudio_backend_filter } - $mem_entries_to_export.AddRange($RStudio_mem_entries) - $RStudio_metric = [Resource_Metric]::new( - ($RStudio_CPU_entries | Measure-Object -Property pct_CPU -Sum).Sum, - ($RStudio_mem_entries | Measure-Object -Property RSS -Sum).Sum / 1024.0 # convert KiB to MiB + foreach ($filter in $process_filters) { + $new_CPU_entries = $CPU_entries | Where-Object { $_.Command -match $filter } + if ($new_CPU_entries.Count -gt 0) { $CPU_entries_to_export.AddRange($new_CPU_entries) } + $new_mem_entries = $mem_entries | Where-Object { $_.Command -match $filter } + if ($new_mem_entries.Count -gt 0) { $mem_entries_to_export.AddRange($new_mem_entries) } + $metric = [Resource_Metric]::new( + ($new_CPU_entries | Measure-Object -Property pct_CPU -Sum).Sum, + ($new_mem_entries | Measure-Object -Property RSS -Sum).Sum / 1024.0 # convert KiB to MiB ) - $cooked_entry."CPU:RStudio backend" = $RStudio_metric.CPU - $cooked_entry."mem:RStudio backend" = $RStudio_metric.mem - } - if ($vreapi_process_filter -ne '') { - $vreapi_CPU_entries = $CPU_entries | Where-Object { $_.Command -match $vreapi_process_filter } - $CPU_entries_to_export.AddRange($vreapi_CPU_entries) - $vreapi_mem_entries = $mem_entries | Where-Object { $_.Command -match $vreapi_process_filter } - $mem_entries_to_export.AddRange($vreapi_mem_entries) - $vreapi_metric = [Resource_Metric]::new( - ($vreapi_CPU_entries | Measure-Object -Property pct_CPU -Sum).Sum, - ($vreapi_mem_entries | Measure-Object -Property RSS -Sum).Sum / 1024.0 # convert KiB to MiB - ) - $cooked_entry."CPU:vreapi:$vreapi_process_filter" = $vreapi_metric.CPU - $cooked_entry."mem:vreapi:$vreapi_process_filter" = $vreapi_metric.mem - } - if ($database_process_filter -ne '') { - $database_CPU_entries = $CPU_entries | Where-Object { $_.Command -match $database_process_filter } - $CPU_entries_to_export.AddRange($database_CPU_entries) - $database_mem_entries = $mem_entries | Where-Object { $_.Command -match $database_process_filter } - $mem_entries_to_export.AddRange($database_mem_entries) - $database_metric = [Resource_Metric]::new( - ($database_CPU_entries | Measure-Object -Property pct_CPU -Sum).Sum, - ($database_mem_entries | Measure-Object -Property RSS -Sum).Sum / 1024.0 # convert KiB to MiB - ) - $cooked_entry."CPU:database:$database_process_filter" = $database_metric.CPU - $cooked_entry."mem:database:$database_process_filter" = $database_metric.mem - } - if ($vreapi_pod_filter -ne '') { - $pod_metric = kubectl top pod $vreapi_pod_filter --no-headers | ForEach-Object { - $col = $_ -split '\s+' - [Resource_Metric]::new([double]($col[1].substring(0, $col[1].Length - 'm'.Length)) / 10.0, [double]($col[2]).substring(0, $col[2].Length - 'Mi'.Length)) - } - $cooked_entry."CPU:pod:$vreapi_pod_filter" = $pod_metric.CPU - $cooked_entry."mem:pod:$vreapi_pod_filter" = $pod_metric.mem + $cooked_entry."CPU:process:$filter" = $metric.CPU + $cooked_entry."mem:process:$filter" = $metric.mem } - if ($database_pod_filter -ne '') { - $pod_metric = kubectl top pod $database_pod_filter --no-headers | ForEach-Object { + foreach ($filter in $pod_filters) { + $pod_metric = kubectl top pod $filter --no-headers | ForEach-Object { $col = $_ -split '\s+' [Resource_Metric]::new([double]($col[1].substring(0, $col[1].Length - 'm'.Length)) / 10.0, [double]($col[2]).substring(0, $col[2].Length - 'Mi'.Length)) } - $cooked_entry."CPU:pod:$database_pod_filter" = $pod_metric.CPU - $cooked_entry."mem:pod:$database_pod_filter" = $pod_metric.mem + $cooked_entry."CPU:pod:$filter" = $pod_metric.CPU + $cooked_entry."mem:pod:$filter" = $pod_metric.mem } # output From c5271da316b0e08e425eee4c18906b90f1edc54b Mon Sep 17 00:00:00 2001 From: Andy Bayer Roswell Date: Wed, 2 Jul 2025 15:19:10 +0200 Subject: [PATCH 141/149] refined performance recording script --- vreapis/tests/perf/rec-utils.ps1 | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/vreapis/tests/perf/rec-utils.ps1 b/vreapis/tests/perf/rec-utils.ps1 index 0b7f4f3d..44e1e800 100644 --- a/vreapis/tests/perf/rec-utils.ps1 +++ b/vreapis/tests/perf/rec-utils.ps1 @@ -105,12 +105,14 @@ class Resource_Metric { # for what are sent by K8s metrics server, and cooked en $cooked_entry = [PSCustomObject]@{ "time" = $null } -$CPU_headers = New-Object System.Collections.Generic.List[string] -$mem_headers = New-Object System.Collections.Generic.List[string] $JupyterLab_backend_filter = 'jupyter.?lab' $RStudio_backend_filter = 'rstudio-server' +$CPU_headers = New-Object System.Collections.Generic.List[string] +$mem_headers = New-Object System.Collections.Generic.List[string] $process_filters = New-Object System.Collections.Generic.List[string] $pod_filters = New-Object System.Collections.Generic.List[string] +if ($JupyterLab_backend) { $process_filters.Add($JupyterLab_backend_filter) } +if ($RStudio_backend) { $process_filters.Add($RStudio_backend_filter) } foreach ($s in @( $browser_process_filter, $vreapi_process_filter, @@ -118,12 +120,6 @@ foreach ($s in @( )) { if ($s -ne '') { $process_filters.Add($s) } } -if ($JupyterLab_backend) { - $process_filters.Add($JupyterLab_backend_filter) -} -if ($RStudio_backend) { - $process_filters.Add($RStudio_backend_filter) -} foreach ($s in @( $vreapi_pod_filter, $database_pod_filter @@ -139,7 +135,7 @@ foreach ($filter in $pod_filters) { $mem_headers += "mem:pod:$filter" } $CPU_headers + $mem_headers | ForEach-Object { $cooked_entry | Add-Member -MemberType NoteProperty -Name $_ -Value $null } -if ((-not $console) -and (-not (Test-Path $cooked_log_file))) { $cooked_entry | ConvertTo-Csv | Select-Object -First 1 | Out-File $cooked_log_file } # add csv headers first +#if ((-not $console) -and (-not (Test-Path $cooked_log_file))) { $cooked_entry | ConvertTo-Csv | Select-Object -First 1 | Out-File $cooked_log_file } # add csv headers first # main body $time_format = 'yyyy-MM-dd HH:mm:ss.fff' @@ -205,9 +201,9 @@ $loop_body = { # filter entries foreach ($filter in $process_filters) { $new_CPU_entries = $CPU_entries | Where-Object { $_.Command -match $filter } - if ($new_CPU_entries.Count -gt 0) { $CPU_entries_to_export.AddRange($new_CPU_entries) } + if ($new_CPU_entries.Count -gt 0) { $CPU_entries_to_export.AddRange(($new_CPU_entries -as [System.Collections.Generic.List[Process_CPU_Entry]])) } $new_mem_entries = $mem_entries | Where-Object { $_.Command -match $filter } - if ($new_mem_entries.Count -gt 0) { $mem_entries_to_export.AddRange($new_mem_entries) } + if ($new_mem_entries.Count -gt 0) { $mem_entries_to_export.AddRange(($new_mem_entries -as [System.Collections.Generic.List[Process_Memory_Entry]])) } $metric = [Resource_Metric]::new( ($new_CPU_entries | Measure-Object -Property pct_CPU -Sum).Sum, ($new_mem_entries | Measure-Object -Property RSS -Sum).Sum / 1024.0 # convert KiB to MiB @@ -223,7 +219,7 @@ $loop_body = { $cooked_entry."CPU:pod:$filter" = $pod_metric.CPU $cooked_entry."mem:pod:$filter" = $pod_metric.mem } - + # output if ($console) { $cooked_entry | Format-Table | Write-Output From daf2766fec976d68880bee126234881607eddedb Mon Sep 17 00:00:00 2001 From: Andy Bayer Roswell Date: Wed, 2 Jul 2025 15:23:31 +0200 Subject: [PATCH 142/149] refined performance recording script --- vreapis/tests/perf/rec-utils.ps1 | 1 - 1 file changed, 1 deletion(-) diff --git a/vreapis/tests/perf/rec-utils.ps1 b/vreapis/tests/perf/rec-utils.ps1 index 44e1e800..36f7b3c4 100644 --- a/vreapis/tests/perf/rec-utils.ps1 +++ b/vreapis/tests/perf/rec-utils.ps1 @@ -135,7 +135,6 @@ foreach ($filter in $pod_filters) { $mem_headers += "mem:pod:$filter" } $CPU_headers + $mem_headers | ForEach-Object { $cooked_entry | Add-Member -MemberType NoteProperty -Name $_ -Value $null } -#if ((-not $console) -and (-not (Test-Path $cooked_log_file))) { $cooked_entry | ConvertTo-Csv | Select-Object -First 1 | Out-File $cooked_log_file } # add csv headers first # main body $time_format = 'yyyy-MM-dd HH:mm:ss.fff' From fac14d2834af30bf39952b0989a566a4218f0fe6 Mon Sep 17 00:00:00 2001 From: Andy Bayer Roswell Date: Wed, 2 Jul 2025 15:42:52 +0200 Subject: [PATCH 143/149] refined performance recording script --- vreapis/tests/perf/rec-utils.ps1 | 10 +++++----- .../{record-usage.ps1 => record-usage.[obsolete].ps1} | 0 2 files changed, 5 insertions(+), 5 deletions(-) rename vreapis/tests/perf/{record-usage.ps1 => record-usage.[obsolete].ps1} (100%) diff --git a/vreapis/tests/perf/rec-utils.ps1 b/vreapis/tests/perf/rec-utils.ps1 index 36f7b3c4..4bffc906 100644 --- a/vreapis/tests/perf/rec-utils.ps1 +++ b/vreapis/tests/perf/rec-utils.ps1 @@ -71,11 +71,11 @@ class Process_Memory_Entry { # pidstat [int]$PID = -1 [double]$minflt_p_s = 0 [double]$majflt_p_s = 0 - [int]$VSZ = 0 # in KiB - [int]$RSS = 0 # in KiB + [long]$VSZ = 0 # in KiB + [long]$RSS = 0 # in KiB [double]$pct_MEM = 0 [string]$Command = '' - Process_Memory_Entry([DateTime]$time, [int]$UID, [int]$_PID, [double]$minflt_p_s, [double]$majflt_p_s, [int]$VSZ, [int]$RSS, [double]$pct_MEM, [string]$Command) { + Process_Memory_Entry([DateTime]$time, [int]$UID, [int]$_PID, [double]$minflt_p_s, [double]$majflt_p_s, [long]$VSZ, [long]$RSS, [double]$pct_MEM, [string]$Command) { $this.time = $time $this.UID = $UID $this.PID = $_PID @@ -189,8 +189,8 @@ $loop_body = { [int]($seg[2]), [double]::Parse($seg[3], $culture), [double]::Parse($seg[4], $culture), - [int]($seg[5]), - [int]($seg[6]), + [long]($seg[5]), + [long]($seg[6]), [double]::Parse($seg[7], $culture), $seg[8..($seg.Length - 1)] -join ' ' ) diff --git a/vreapis/tests/perf/record-usage.ps1 b/vreapis/tests/perf/record-usage.[obsolete].ps1 similarity index 100% rename from vreapis/tests/perf/record-usage.ps1 rename to vreapis/tests/perf/record-usage.[obsolete].ps1 From 6ae3308347527e16a18347bca2aa16a68ccf1347 Mon Sep 17 00:00:00 2001 From: Andy Bayer Roswell Date: Wed, 2 Jul 2025 18:01:43 +0200 Subject: [PATCH 144/149] refined performance recording script --- vreapis/tests/perf/rec-utils.ps1 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vreapis/tests/perf/rec-utils.ps1 b/vreapis/tests/perf/rec-utils.ps1 index 4bffc906..b6c00b6d 100644 --- a/vreapis/tests/perf/rec-utils.ps1 +++ b/vreapis/tests/perf/rec-utils.ps1 @@ -7,7 +7,7 @@ param( [alias('vp')][string]$vreapi_pod_filter = '', # record resource usage for vreapi pod [common backend]. use this param to specify re pattern to filter vreapi pod from kubectl top pod entries [alias('dp')][string]$database_pod_filter = '', # record resource usage for db pod [common backend]. use this param to specify re pattern to filter db pod from kubectl top pod entries [string]$log_dir = '.log', # directory to store log files - [double]$interval = 1, # interval between 2 adjacent resource usage captures [seconds] + [int]$interval = 1, # interval between 2 adjacent resource usage captures [seconds]. [pidstat only supports integer intervals] [long]$number_of_records = 0, # 0 or negative means infinite records. positive means number of records to capture [switch]$console = $false # print usage data to console ) @@ -150,7 +150,7 @@ $loop_body = { $cooked_entry.time = $time # convert text-based results from pidstat to .NET objects - $process_info = (pidstat -rul -p ALL 0) -join "`n" -split "\n\n" | Select-Object -Skip 1 + $process_info = (pidstat -rul -p ALL $interval 1) -join "`n" -split "\n\n" | Select-Object -Skip 1 $util_rows = New-Object System.Collections.Generic.List[System.Collections.Generic.List[string[]]] for ($i = 0; $i -lt 2; ++$i) { $raw_rows = $process_info[$i] -split '\n' | Select-Object -Skip 1 From 31435baee1cea8ea664063d334a9c0b83d4c16ce Mon Sep 17 00:00:00 2001 From: Andy Bayer Roswell Date: Wed, 2 Jul 2025 19:24:24 +0200 Subject: [PATCH 145/149] refined performance recording script --- vreapis/tests/perf/rec-utils.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vreapis/tests/perf/rec-utils.ps1 b/vreapis/tests/perf/rec-utils.ps1 index b6c00b6d..0843efaf 100644 --- a/vreapis/tests/perf/rec-utils.ps1 +++ b/vreapis/tests/perf/rec-utils.ps1 @@ -6,7 +6,7 @@ param( [alias('d')][string]$database_process_filter = '', # record resource usage for db. use this param to specify re pattern to filter db processes from ps entries [alias('vp')][string]$vreapi_pod_filter = '', # record resource usage for vreapi pod [common backend]. use this param to specify re pattern to filter vreapi pod from kubectl top pod entries [alias('dp')][string]$database_pod_filter = '', # record resource usage for db pod [common backend]. use this param to specify re pattern to filter db pod from kubectl top pod entries - [string]$log_dir = '.log', # directory to store log files + [string]$log_dir = 'log', # directory to store log files [int]$interval = 1, # interval between 2 adjacent resource usage captures [seconds]. [pidstat only supports integer intervals] [long]$number_of_records = 0, # 0 or negative means infinite records. positive means number of records to capture [switch]$console = $false # print usage data to console From 3b672a7629a6a77bd0bb048c3d8d903933ae46ed Mon Sep 17 00:00:00 2001 From: Andy Bayer Roswell Date: Fri, 4 Jul 2025 17:32:32 +0200 Subject: [PATCH 146/149] refined test notebook generation script --- vreapis/tests/perf/cvt-Rmd-to-ipynb.ps1 | 34 +++++++++++++++---------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/vreapis/tests/perf/cvt-Rmd-to-ipynb.ps1 b/vreapis/tests/perf/cvt-Rmd-to-ipynb.ps1 index c941f069..4d820243 100644 --- a/vreapis/tests/perf/cvt-Rmd-to-ipynb.ps1 +++ b/vreapis/tests/perf/cvt-Rmd-to-ipynb.ps1 @@ -1,20 +1,26 @@ param( - [System.IO.FileInfo[]]$pathnames # files to convert + [System.IO.FileInfo[]]$pathnames, # files to convert +# $kernelspec = @{ +# display_name = 'R [conda env:jupyterlab] *' +# language = 'R' +# name = 'conda-env-jupyterlab-r' +# } + $kernelspec = @{ + display_name = 'R' + language = 'R' + name = 'ir' + }, + $language_info = @{ + codemirror_mode = 'r' + file_extension = '.r' + mimetype = 'text/x-r-source' + name = 'R' + pygments_lexer = 'r' +# version = '4.3.3' + version = '4.4.1' + } ) -$kernelspec = @{ - display_name = 'R [conda env:jupyterlab] *' - language = 'R' - name = 'conda-env-jupyterlab-r' -} -$language_info = @{ - codemirror_mode = 'r' - file_extension = '.r' - mimetype = 'text/x-r-source' - name = 'R' - pygments_lexer = 'r' - version = '4.3.3' -} foreach ($pathname in $pathnames) { Write-Host -NoNewline "src: " From 8d61260276e559cc54866589e8b48fca53f71b55 Mon Sep 17 00:00:00 2001 From: Andy Bayer Roswell Date: Fri, 4 Jul 2025 19:14:44 +0200 Subject: [PATCH 147/149] refined test notebook generation script --- vreapis/tests/perf/cvt-Rmd-to-ipynb.ps1 | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/vreapis/tests/perf/cvt-Rmd-to-ipynb.ps1 b/vreapis/tests/perf/cvt-Rmd-to-ipynb.ps1 index 4d820243..4661a217 100644 --- a/vreapis/tests/perf/cvt-Rmd-to-ipynb.ps1 +++ b/vreapis/tests/perf/cvt-Rmd-to-ipynb.ps1 @@ -16,12 +16,10 @@ param( mimetype = 'text/x-r-source' name = 'R' pygments_lexer = 'r' -# version = '4.3.3' - version = '4.4.1' + version = '4.3.3' # keep this unchanged to let JupyterLab not adjust automatically [or it will ask to save the file, which brings impacts on performance test results] } ) - foreach ($pathname in $pathnames) { Write-Host -NoNewline "src: " Write-Host -ForegroundColor Yellow $pathname @@ -30,7 +28,21 @@ foreach ($pathname in $pathnames) { $generated_ipynb = ConvertFrom-Json (Get-Content $dst -Raw) $generated_ipynb.metadata | Add-Member -NotePropertyName kernelspec -NotePropertyValue $kernelspec $generated_ipynb.metadata | Add-Member -NotePropertyName language_info -NotePropertyValue $language_info - $generated_ipynb | ConvertTo-Json -Depth 100 | Set-Content $dst + $serialized_ipynb = ($generated_ipynb | ConvertTo-Json -Depth 100) -split '\n' + $serialized_ipynb | Set-Content $dst +# $reindented_ipynb_builder = New-Object System.Text.StringBuilder +# foreach ($row in $serialized_ipynb) { # adjust indentation to 1 space [JupyterLab's indentation preference] +# $matches = $null +# if ($row -match '^\s+') { +# [void]$reindented_ipynb_builder.Append(' ' * ($matches[0].Length / 2)) +# [void]$reindented_ipynb_builder.Append($row.Substring($matches[0].Length)) +# } else { +# [void]$reindented_ipynb_builder.Append($row) +# } +# [void]$reindented_ipynb_builder.Append("`n") +# } +# [void]$reindented_ipynb_builder.Remove($reindented_ipynb_builder.Length - 1, 1) # remove the last newline +# $reindented_ipynb_builder.ToString() | Set-Content $dst Write-Host -NoNewline "dst: " Write-Host -ForegroundColor Green $dst } From 9a7547e6663d88078510cfa7f26cea79517f5ca0 Mon Sep 17 00:00:00 2001 From: Andy Bayer Roswell Date: Fri, 4 Jul 2025 19:17:37 +0200 Subject: [PATCH 148/149] refined test notebook generation script --- vreapis/tests/perf/cvt-Rmd-to-ipynb.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vreapis/tests/perf/cvt-Rmd-to-ipynb.ps1 b/vreapis/tests/perf/cvt-Rmd-to-ipynb.ps1 index 4661a217..badc5829 100644 --- a/vreapis/tests/perf/cvt-Rmd-to-ipynb.ps1 +++ b/vreapis/tests/perf/cvt-Rmd-to-ipynb.ps1 @@ -16,7 +16,7 @@ param( mimetype = 'text/x-r-source' name = 'R' pygments_lexer = 'r' - version = '4.3.3' # keep this unchanged to let JupyterLab not adjust automatically [or it will ask to save the file, which brings impacts on performance test results] + version = '4.3.3' # keep this unchanged even if the actual R version mismatches to let JupyterLab not adjust automatically [or it will ask to save the file, which brings impacts on performance test results] } ) From 6d0058df601f25e947028ad21d7b86be92da17f9 Mon Sep 17 00:00:00 2001 From: Andy Bayer Roswell Date: Wed, 9 Jul 2025 14:04:27 +0200 Subject: [PATCH 149/149] refined cpu/mem util recording script --- vreapis/tests/perf/rec-utils.ps1 | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/vreapis/tests/perf/rec-utils.ps1 b/vreapis/tests/perf/rec-utils.ps1 index 0843efaf..3add7d7c 100644 --- a/vreapis/tests/perf/rec-utils.ps1 +++ b/vreapis/tests/perf/rec-utils.ps1 @@ -1,15 +1,16 @@ param( - [string]$browser_process_filter = '', # record resource usage for browser. use this param to specify re pattern to filter browser processes from ps entries - [switch]$JupyterLab_backend = $false, # record resource usage for JupyterLab backend - [switch]$RStudio_backend = $false, # record resource usage for RStudio backend - [alias('v')][string]$vreapi_process_filter = '', # record resource usage for vreapi [common backend]. use this param to specify re pattern to filter vreapi process from ps entries - [alias('d')][string]$database_process_filter = '', # record resource usage for db. use this param to specify re pattern to filter db processes from ps entries - [alias('vp')][string]$vreapi_pod_filter = '', # record resource usage for vreapi pod [common backend]. use this param to specify re pattern to filter vreapi pod from kubectl top pod entries - [alias('dp')][string]$database_pod_filter = '', # record resource usage for db pod [common backend]. use this param to specify re pattern to filter db pod from kubectl top pod entries - [string]$log_dir = 'log', # directory to store log files - [int]$interval = 1, # interval between 2 adjacent resource usage captures [seconds]. [pidstat only supports integer intervals] - [long]$number_of_records = 0, # 0 or negative means infinite records. positive means number of records to capture - [switch]$console = $false # print usage data to console + [string]$browser_process_filter = '', # record resource usage for browser. use this param to specify re pattern to filter browser processes from ps entries + [switch]$JupyterLab_backend = $false, # record resource usage for JupyterLab backend + [switch]$RStudio_backend = $false, # record resource usage for RStudio backend + [alias('v')][string]$vreapi_process_filter = '', # record resource usage for vreapi [common backend]. use this param to specify re pattern to filter vreapi process from ps entries + [alias('d')][string]$database_process_filter = '', # record resource usage for db. use this param to specify re pattern to filter db processes from ps entries + [alias('vp')][string]$vreapi_pod_filter = '', # record resource usage for vreapi pod [common backend]. use this param to specify re pattern to filter vreapi pod from kubectl top pod entries + [alias('dp')][string]$database_pod_filter = '', # record resource usage for db pod [common backend]. use this param to specify re pattern to filter db pod from kubectl top pod entries + [string]$log_dir = 'log', # directory to store log files + [string]$log_filename_prefix = (Get-Date -Format 'yyyyMMdd-HHmmss'), # prefix for log filename. if not specified, current date and time will be used + [int]$interval = 1, # interval between 2 adjacent resource usage captures [seconds]. [pidstat only supports integer intervals] + [long]$number_of_records = 0, # 0 or negative means infinite records. positive means number of records to capture + [switch]$console = $false # print usage data to console ) # intro messages before recording @@ -22,10 +23,9 @@ if ($console) { } else { if ((Test-Path $log_dir) -eq $false) { & '/usr/bin/mkdir' -p $log_dir } - $date = Get-Date -Format 'yyyyMMdd-HHmmss' - $CPU_log_file = $log_dir + "/$date.CPU.csv" - $mem_log_file = $log_dir + "/$date.mem.csv" - $cooked_log_file = $log_dir + "/$date.cooked.csv" + $CPU_log_file = $log_dir + "/$log_filename_prefix.CPU.csv" + $mem_log_file = $log_dir + "/$log_filename_prefix.mem.csv" + $cooked_log_file = $log_dir + "/$log_filename_prefix.cooked.csv" Write-Host -NoNewline 'Recording CPU & mem usage at ' Write-Host -NoNewline -ForegroundColor Green $CPU_log_file