diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index 96c6f31d..0ef8173e 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -25,6 +25,8 @@ env: DOCKER_FOLDER: ${{ inputs.docker_folder }} TAG: ${{ inputs.tag }} DOCKERHUB_IMAGE_ID: "qcdis/" + CELL_GITHUB: "https://github.com/QCDIS/NaaVRE-cells-test-3" + CELL_GITHUB_TOKEN: ${{ secrets.CELL_GITHUB_TOKEN }} jobs: build: @@ -33,21 +35,40 @@ jobs: steps: - uses: actions/checkout@v4 + - name: chkenv + run: python -c "import os; print(os.getenv('CELL_GITHUB')); print(os.getenv('BASE_PATH'))" + - name: Build docker run: | + echo '-> envvar:' + env cd $DOCKER_FOLDER && docker build . --file Dockerfile -t $TAG --build-arg "NODE_ENV=${{ inputs.environment }}" - name: Run docker compose - uses: isbang/compose-action@v1.5.1 + # uses: isbang/compose-action@v1.5.1 + uses: hoverkraft-tech/compose-action@v2.0.1 with: compose-file: "${{ inputs.docker_folder }}/docker-compose.yaml" + - name: chkenv2 + run: python -c "import os; print(os.getenv('CELL_GITHUB')); print(os.getenv('BASE_PATH'))" + - name: Wait for healthy app uses: raschmitt/wait-for-healthy-container/@v1 with: container-name: app timeout: 120 + - name: chkenv3 + if: always() + run: python -c "import os; print(os.getenv('CELL_GITHUB')); print(os.getenv('BASE_PATH'))" + + - name: Capture Docker Logs + if: failure() + run: | + echo '-> Capturing Docker logs ...' + docker logs app + - name: Login to github Registry if: ${{ inputs.push }} uses: docker/login-action@v3 diff --git a/tilt/vreapis/Dockerfile b/tilt/vreapis/Dockerfile index 3d9a92c8..fe99d3b8 100644 --- a/tilt/vreapis/Dockerfile +++ b/tilt/vreapis/Dockerfile @@ -1,9 +1,11 @@ -FROM python:3.12.2-slim +# FROM python:3.12.2-slim +FROM python:3.11.9-slim +# Dependency pytype [used by PyExtractor] complains 'Python versions > 3.11 are not yet supported' WORKDIR /app RUN apt update && \ - apt install -y libpq-dev gcc gdal-bin curl net-tools iputils-ping + apt install -y libpq-dev gcc gdal-bin curl net-tools iputils-ping r-base # Install rysnc for Tilt syncback RUN apt install -y rsync @@ -14,6 +16,9 @@ RUN /opt/venv/bin/pip install pip --upgrade COPY ./vreapis/requirements.txt /app RUN /opt/venv/bin/pip install -r requirements.txt +# Set another directory for installation of R packages to avoid permission problem +# RUN echo 'R_LIBS_USER="~/R/library"' >> ~/.Renviron + COPY ./vreapis /app RUN chmod +x entrypoint.dev.sh diff --git a/vreapis/.dockerignore b/vreapis/.dockerignore new file mode 100644 index 00000000..d2e350f5 --- /dev/null +++ b/vreapis/.dockerignore @@ -0,0 +1,6 @@ +.env +export_VARS +.idea +.pytest_cache +__pycache__ +NaaVRE_db.json diff --git a/vreapis/Dockerfile b/vreapis/Dockerfile index 9c730e6e..9cc4a7d7 100644 --- a/vreapis/Dockerfile +++ b/vreapis/Dockerfile @@ -1,9 +1,10 @@ -FROM python:3.12.2-slim +# FROM python:3.12.2-slim +FROM python:3.11.9-slim WORKDIR /app RUN apt update && \ - apt install -y libpq-dev gcc gdal-bin curl net-tools iputils-ping + apt install -y libpq-dev gcc gdal-bin curl net-tools iputils-ping r-base RUN python3 -m venv /opt/venv RUN /opt/venv/bin/pip install pip --upgrade @@ -11,6 +12,8 @@ RUN /opt/venv/bin/pip install pip --upgrade COPY ./requirements.txt /app RUN /opt/venv/bin/pip install -r requirements.txt +RUN Rscript -e "install.packages(c('rlang', 'lobstr', 'purrr', 'renv'), repos = 'https://cloud.r-project.org')" + COPY . /app RUN chmod +x entrypoint.sh diff --git a/vreapis/auth/Keycloak.py b/vreapis/auth/Keycloak.py new file mode 100644 index 00000000..158dc5a4 --- /dev/null +++ b/vreapis/auth/Keycloak.py @@ -0,0 +1,48 @@ +import os +from requests.exceptions import RequestException +from rest_framework.authentication import BaseAuthentication +from rest_framework.exceptions import AuthenticationFailed +from django.contrib.auth.models import User +from django.http import HttpRequest +from django.conf import settings +import jwt +import jwt.algorithms + +import common + + +class KeycloakAuthentication(BaseAuthentication): + @staticmethod + def get_verif_inf() -> dict[str, any]: + return common.session.get(settings.KEYCLOAK_VERIF_URL, verify=settings.ALLOW_INSECURE_TLS).json() + + def authenticate(self, request: HttpRequest): + received_token: str = request.headers.get('Authorization', '') + if received_token == '': + raise AuthenticationFailed('Authorization header not found. Login using Keycloak to get an access token.') + token_header: dict[str, any] = jwt.get_unverified_header(received_token) + try: + verif_inf: dict[str, any] = KeycloakAuthentication.get_verif_inf() + except RequestException as e: + common.logger.error(f'[{e.response.status_code}] Error while getting Keycloak public key.') + return None + kid: str = token_header.get('kid', '') + if kid == '': + common.logger.error(f'JWT Key ID (kid) not found:{os.linesep}{received_token}') + raise AuthenticationFailed(f'Invalid token format: JWT Key ID (kid) not found.') + verif_keys: list[dict[str, any]] = verif_inf.get('keys', []) + key = next((key for key in verif_keys if key['kid'] == kid), None) + if key is None: + common.logger.error(f'Keycloak public key not found.') + return None + pub_key = jwt.algorithms.RSAAlgorithm.from_jwk(key) + access_token: dict[str, any] = jwt.decode(received_token, pub_key, algorithms=key['alg'], audience='account') + username: str = access_token.get('preferred_username', '') + if username == '': + common.logger.error(f'Username (preferred_username) not found in token:{os.linesep}{received_token}') + raise AuthenticationFailed(f'Invalid token format: Username (preferred_username) not found.') + try: + user: User | None = User.objects.get(username=username) + except User.DoesNotExist: + user: User = User.objects.create_user(username) + return user, None diff --git a/vreapis/auth/simple.py b/vreapis/auth/simple.py new file mode 100644 index 00000000..6684bf03 --- /dev/null +++ b/vreapis/auth/simple.py @@ -0,0 +1,24 @@ +from rest_framework.authentication import BaseAuthentication +from rest_framework.exceptions import AuthenticationFailed +from django.contrib.auth.models import User +from django.http import HttpRequest +from django.conf import settings +import string +import random + + +class StaticTokenAuthentication(BaseAuthentication): + dummy_user: User + + def __init__(self): + try: + StaticTokenAuthentication.dummy_user = User.objects.get(username=settings.NAAVRE_API_TOKEN) + except User.DoesNotExist: + StaticTokenAuthentication.dummy_user = User.objects.create_user(settings.NAAVRE_API_TOKEN, password=''.join(random.choice(string.printable) for _ in range(32))) + + def authenticate(self, request: HttpRequest): + access_token: str = request.headers.get('Authorization', '') + print(access_token == f'Token {settings.NAAVRE_API_TOKEN}') + if access_token != f'Token {settings.NAAVRE_API_TOKEN}': + raise AuthenticationFailed(f'Invalid NaaVRE API token') + return StaticTokenAuthentication.dummy_user, None diff --git a/vreapis/auth/test_Keycloak.py b/vreapis/auth/test_Keycloak.py new file mode 100644 index 00000000..b4565ee8 --- /dev/null +++ b/vreapis/auth/test_Keycloak.py @@ -0,0 +1,18 @@ +from django.test import TestCase, RequestFactory +from django.conf import settings +from urllib.parse import urlencode +from auth.Keycloak import KeycloakAuthentication +import common + + +class KeycloakAuthenticationTestCase(TestCase): + def test_authenticate(self): # todo. what if authentication failed? what if refresh token is used? + post_data: dict[str, str] = {'client_id': settings.KEYCLOAK_CLIENT_ID, 'grant_type': 'password', 'scope': 'openid', 'username': 'u', 'password': 'u'} + response_body: dict[str, any] = common.session.post(settings.KEYCLOAK_LOGIN_URL, headers={'Content-Type': 'application/x-www-form-urlencoded'}, data=urlencode(post_data), verify=settings.ALLOW_INSECURE_TLS).json() + access_token: str = response_body['access_token'] + + request_factory = RequestFactory() + request = request_factory.post('', HTTP_AUTHORIZATION=access_token) + authenticator = KeycloakAuthentication() + user, _ = authenticator.authenticate(request) + assert user.username == post_data['username'] diff --git a/vreapis/auth/test_simple.py b/vreapis/auth/test_simple.py new file mode 100644 index 00000000..99789e06 --- /dev/null +++ b/vreapis/auth/test_simple.py @@ -0,0 +1,21 @@ +from django.test import TestCase, RequestFactory +from django.conf import settings +from rest_framework.exceptions import AuthenticationFailed + +from auth.simple import StaticTokenAuthentication + + +class StaticTokenAuthenticationTestCase(TestCase): + tokens_accepted = [f'Token {settings.NAAVRE_API_TOKEN}', ] + tokens_not_accepted = [f'{settings.NAAVRE_API_TOKEN}', f'', f'0123456789abcdef0123456789abcdef01234567', ] + + def test_authenticate(self): + request_factory = RequestFactory() + authenticator = StaticTokenAuthentication() + print(StaticTokenAuthenticationTestCase.tokens_accepted) + for token in StaticTokenAuthenticationTestCase.tokens_accepted: + request = request_factory.post('', HTTP_AUTHORIZATION=token) + user, _ = authenticator.authenticate(request) + for token in StaticTokenAuthenticationTestCase.tokens_not_accepted: + request = request_factory.post('', HTTP_AUTHORIZATION=token) + self.assertRaises(AuthenticationFailed, authenticator.authenticate, request) diff --git a/vreapis/cells/__init__.py b/vreapis/catalog/__init__.py similarity index 100% rename from vreapis/cells/__init__.py rename to vreapis/catalog/__init__.py diff --git a/vreapis/catalog/admin.py b/vreapis/catalog/admin.py new file mode 100644 index 00000000..87770453 --- /dev/null +++ b/vreapis/catalog/admin.py @@ -0,0 +1,7 @@ +from django.contrib import admin + +from . import models + +# Register your models here. + +admin.site.register(models.Cell) diff --git a/vreapis/cells/apps.py b/vreapis/catalog/apps.py similarity index 63% rename from vreapis/cells/apps.py rename to vreapis/catalog/apps.py index 9833cff9..a5993c68 100644 --- a/vreapis/cells/apps.py +++ b/vreapis/catalog/apps.py @@ -1,6 +1,6 @@ from django.apps import AppConfig -class CellsConfig(AppConfig): +class CatalogConfig(AppConfig): default_auto_field = 'django.db.models.BigAutoField' - name = 'cells' + name = 'catalog' diff --git a/vreapis/catalog/management/commands/export-from-psql-pod-in-legacy-db-schema.ps1 b/vreapis/catalog/management/commands/export-from-psql-pod-in-legacy-db-schema.ps1 new file mode 100644 index 00000000..c73bff49 --- /dev/null +++ b/vreapis/catalog/management/commands/export-from-psql-pod-in-legacy-db-schema.ps1 @@ -0,0 +1,30 @@ +$cwd = $PWD + +$project_root = Resolve-Path "$PSScriptRoot/../../.." + +$vreapi_pod_inf = kubectl get pod | sls '^vrepaas-vreapi-\w+-\w+' +$vreapi_pod_name = $vreapi_pod_inf.Matches.Value +$psql_pod_inf = kubectl get pod -owide | sls '^vrepaas-postgresql-0' +$psql_pod_IP = ($psql_pod_inf.Line | sls '\d+\.\d+\.\d+\.\d+').Matches.Value + +cd $project_root +$var = Get-Content export_VARS +$off_DB_HOST = ($var | sls '^export DB_HOST=').LineNumber - 1 +$off_DB_NAME = ($var | sls '^export DB_NAME=').LineNumber - 1 +$off_DB_USER = ($var | sls '^export DB_USER=').LineNumber - 1 +$off_DB_PASSWORD = ($var | sls '^export DB_PASSWORD=').LineNumber - 1 +$var[$off_DB_HOST] = "export DB_HOST=$psql_pod_IP" +$var[$off_DB_NAME] = "export DB_NAME=vrepaas" +$var[$off_DB_USER] = "export DB_USER=vrepaas" +$var[$off_DB_PASSWORD] = "export DB_PASSWORD=vrepaas" + +$var > /tmp/export_VARS +kubectl cp /tmp/export_VARS "${vreapi_pod_name}:/tmp" +kubectl cp "$PSScriptRoot/export-from-psql-pod-in-legacy-db-schema.sh" "${vreapi_pod_name}:/tmp" + +kubectl cp ~/NaaVRE/NaaVRE_db.json "${vreapi_pod_name}:/app" +kubectl exec -it $vreapi_pod_name -- bash "/tmp/export-from-psql-pod-in-legacy-db-schema.sh" + +kubectl cp "${vreapi_pod_name}:/app/NaaVRE_db.json" ~/NaaVRE/NaaVRE_db.json + +cd $cwd diff --git a/vreapis/catalog/management/commands/export-from-psql-pod-in-legacy-db-schema.sh b/vreapis/catalog/management/commands/export-from-psql-pod-in-legacy-db-schema.sh new file mode 100644 index 00000000..1e3d8bf2 --- /dev/null +++ b/vreapis/catalog/management/commands/export-from-psql-pod-in-legacy-db-schema.sh @@ -0,0 +1,3 @@ +source /opt/venv/bin/activate +source /tmp/export_VARS +python manage.py export_in_legacy_db_schema Cell diff --git a/vreapis/catalog/management/commands/export_in_legacy_db_schema.py b/vreapis/catalog/management/commands/export_in_legacy_db_schema.py new file mode 100644 index 00000000..77ef1725 --- /dev/null +++ b/vreapis/catalog/management/commands/export_in_legacy_db_schema.py @@ -0,0 +1,39 @@ +import json + +from django.core.management import CommandParser +from django.core.management.base import BaseCommand, CommandError +from django.db.models import QuerySet + +import common +from catalog.models import Cell +from catalog.serializers import CellSerializer + + +class Command(BaseCommand): + help: str = 'Export designated tables in legacy db schema' + + + def add_arguments(self, parser: CommandParser): + parser.add_argument('tables', type=str, nargs='+', help='Tables to export') + + def handle(self, *args, **options): + for table in options['tables']: + match table: + case 'Cell': + queryset: QuerySet = Cell.objects.all().order_by('task_name') + print(f'Cell count: {queryset.count()}') + no: int = 1 + cells: dict[str, dict] = {} + for cell in queryset: + serializer = CellSerializer(cell) + print(f'task_name: {cell.task_name}') + cells[str(no)] = serializer.data + no += 1 + db_file: str = f'{common.project_root}/NaaVRE_db.json' + with open(db_file) as f: + db = json.load(f) + db['cells'] = cells + with open(db_file, 'w') as f: + json.dump(db, f) + case _: + raise CommandError(f"Table {table} is not supported") diff --git a/vreapis/catalog/migrations/0001_initial.py b/vreapis/catalog/migrations/0001_initial.py new file mode 100644 index 00000000..2dcfca3d --- /dev/null +++ b/vreapis/catalog/migrations/0001_initial.py @@ -0,0 +1,37 @@ +# Generated by Django 5.0.3 on 2024-06-17 18:52 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ] + + operations = [ + migrations.CreateModel( + name='Cell', + fields=[ + ('title', models.CharField(max_length=255)), + ('task_name', models.CharField(max_length=255)), + ('original_source', models.CharField(max_length=255)), + ('types', models.JSONField(blank=True, default=dict, null=True)), + ('base_image', models.JSONField(blank=True, null=True)), + ('inputs', models.JSONField(blank=True, default=list, null=True)), + ('outputs', models.JSONField(blank=True, default=list, null=True)), + ('params', models.JSONField(blank=True, default=list, null=True)), + ('param_values', models.JSONField(blank=True, null=True)), + ('confs', models.JSONField(blank=True, null=True)), + ('dependencies', models.JSONField(blank=True, null=True)), + ('chart_obj', models.JSONField(blank=True, null=True)), + ('node_id', models.CharField(max_length=255, primary_key=True, serialize=False)), + ('container_source', models.CharField(max_length=255)), + ('global_conf', models.JSONField(blank=True, null=True)), + ('kernel', models.CharField(max_length=255)), + ('notebook_dict', models.JSONField(blank=True, null=True)), + ('image_version', models.CharField(max_length=255)), + ], + ), + ] diff --git a/vreapis/catalog/migrations/0002_alter_cell_container_source_alter_cell_image_version.py b/vreapis/catalog/migrations/0002_alter_cell_container_source_alter_cell_image_version.py new file mode 100644 index 00000000..19f92c28 --- /dev/null +++ b/vreapis/catalog/migrations/0002_alter_cell_container_source_alter_cell_image_version.py @@ -0,0 +1,23 @@ +# Generated by Django 5.0.3 on 2024-06-17 19:03 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('catalog', '0001_initial'), + ] + + operations = [ + migrations.AlterField( + model_name='cell', + name='container_source', + field=models.CharField(blank=True, max_length=255, null=True), + ), + migrations.AlterField( + model_name='cell', + name='image_version', + field=models.CharField(blank=True, max_length=255, null=True), + ), + ] diff --git a/vreapis/catalog/migrations/0003_alter_cell_container_source_alter_cell_image_version_and_more.py b/vreapis/catalog/migrations/0003_alter_cell_container_source_alter_cell_image_version_and_more.py new file mode 100644 index 00000000..33701f10 --- /dev/null +++ b/vreapis/catalog/migrations/0003_alter_cell_container_source_alter_cell_image_version_and_more.py @@ -0,0 +1,48 @@ +# Generated by Django 5.0.3 on 2024-06-17 21:00 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('catalog', '0002_alter_cell_container_source_alter_cell_image_version'), + ] + + operations = [ + migrations.AlterField( + model_name='cell', + name='container_source', + field=models.CharField(blank=True, max_length=4000, null=True), + ), + migrations.AlterField( + model_name='cell', + name='image_version', + field=models.CharField(blank=True, max_length=4000, null=True), + ), + migrations.AlterField( + model_name='cell', + name='kernel', + field=models.CharField(max_length=4000), + ), + migrations.AlterField( + model_name='cell', + name='node_id', + field=models.CharField(max_length=4000, primary_key=True, serialize=False), + ), + migrations.AlterField( + model_name='cell', + name='original_source', + field=models.CharField(max_length=4000), + ), + migrations.AlterField( + model_name='cell', + name='task_name', + field=models.CharField(max_length=4000), + ), + migrations.AlterField( + model_name='cell', + name='title', + field=models.CharField(max_length=4000), + ), + ] diff --git a/vreapis/cells/migrations/__init__.py b/vreapis/catalog/migrations/__init__.py similarity index 100% rename from vreapis/cells/migrations/__init__.py rename to vreapis/catalog/migrations/__init__.py diff --git a/vreapis/catalog/models.py b/vreapis/catalog/models.py new file mode 100644 index 00000000..5fa6b22c --- /dev/null +++ b/vreapis/catalog/models.py @@ -0,0 +1,138 @@ +import json +import re + +from slugify import slugify +from django.db import models + +import common + + +class Cell(models.Model): + title = models.CharField(max_length=common.default_varchar_length) + task_name = models.CharField(max_length=common.default_varchar_length) + original_source = models.CharField(max_length=common.default_varchar_length) + types = models.JSONField(blank=True, null=True, default=dict) + base_image = models.JSONField(blank=True, null=True) + inputs = models.JSONField(blank=True, null=True, default=list) + outputs = models.JSONField(blank=True, null=True, default=list) + params = models.JSONField(blank=True, null=True, default=list) + param_values = models.JSONField(blank=True, null=True) + confs = models.JSONField(blank=True, null=True) + dependencies = models.JSONField(blank=True, null=True) + chart_obj = models.JSONField(blank=True, null=True) + node_id = models.CharField(max_length=common.default_varchar_length, primary_key=True) + container_source = models.CharField(max_length=common.default_varchar_length, blank=True, null=True) + global_conf = models.JSONField(blank=True, null=True) + kernel = models.CharField(max_length=common.default_varchar_length) + notebook_dict = models.JSONField(blank=True, null=True, default=dict) + image_version = models.CharField(max_length=common.default_varchar_length, blank=True, null=True) + + def _extract_types(self, vars_dict): + """ Extract types to self.types and return list of var names + + :param vars_dict: {'var1': {'name: 'var1', 'type': 'str'}, 'var2': ...} + :return: ['var1', 'var2', ...] + """ + names = [] + for var_props in vars_dict.values(): + var_type = var_props['type'] + var_name = var_props['name'] + self.types[var_name] = var_type + names.append(var_name) + return names + + def add_inputs(self, inputs): + if isinstance(inputs, dict): + inputs = self._extract_types(inputs) + self.inputs = inputs + + def add_outputs(self, outputs): + if isinstance(outputs, dict): + outputs = self._extract_types(outputs) + self.outputs = outputs + + def add_params(self, params): + if isinstance(params, dict): + params = self._extract_types(params) + self.params = params + + def set_image_version(self, image_version): + self.image_version = image_version + + def add_param_values(self, params): + self.param_values = {} + if isinstance(params, dict): + for param_props in params.values(): + if 'value' in param_props: + self.param_values[param_props['name']] = param_props['value'] + + def concatenate_all_inputs(self): + self.all_inputs = list(self.inputs) + list(self.params) + + def clean_code(self): + indices_to_remove = [] + lines = self.original_source.splitlines() + self.original_source = "" + + for line_i in range(0, len(lines)): + line = lines[line_i] + # Do not remove line that startswith param_ if not in the self.params + if line.startswith('param_'): + # clean param name + pattern = r"\b(param_\w+)\b" + param_name = re.findall(pattern, line)[0] + if param_name in self.params: + indices_to_remove.append(line_i) + if re.match('^\s*(#|import|from)', line): + indices_to_remove.append(line_i) + + for ir in sorted(indices_to_remove, reverse=True): + lines.pop(ir) + + self.original_source = "\n".join(lines) + + def clean_task_name(self): + self.task_name = slugify(self.task_name) + + def clean_title(self): + self.title = slugify(self.title) + + def integrate_configuration(self): + lines = self.original_source.splitlines() + self.original_source = "" + for idx, conf in enumerate(self.generate_configuration()): + lines.insert(idx, conf) + self.original_source = "\n".join(lines) + + def generate_dependencies(self): + resolves = [] + for d in self.dependencies: + resolve_to = "import %s" % d['name'] + if d['module']: + resolve_to = "from %s %s" % (d['module'], resolve_to) + if d['asname']: + resolve_to += " as %s" % d['asname'] + resolves.append(resolve_to) + return resolves + + def generate_configuration(self): + resolves = [] + for c in self.confs: + resolves.append(self.confs[c]) + return resolves + + def generate_configuration_dict(self): + resolves = [] + for c in self.confs: + assignment = self.confs[c].split('=')[1].replace('=', '').strip() + conf = {c: assignment} + resolves.append(conf) + return resolves + + def toJSON(self) -> str: + # return json.dumps(self, default=lambda o: o.__dict__, sort_keys=True, indent=4) + array_fields = ['inputs', 'outputs', 'params', ] + result = {k: v for k, v in self.__dict__.items() if k not in array_fields} + for field in array_fields: + result[field] = list(getattr(self, field)) + return json.dumps(result, default=lambda o: o.__dict__, sort_keys=True, indent=4) diff --git a/vreapis/catalog/serializers.py b/vreapis/catalog/serializers.py new file mode 100644 index 00000000..f8d7c378 --- /dev/null +++ b/vreapis/catalog/serializers.py @@ -0,0 +1,8 @@ +from rest_framework.serializers import ModelSerializer +from . import models + + +class CellSerializer(ModelSerializer): + class Meta: + model = models.Cell + fields = '__all__' diff --git a/vreapis/cells/tests.py b/vreapis/catalog/tests.py similarity index 100% rename from vreapis/cells/tests.py rename to vreapis/catalog/tests.py diff --git a/vreapis/catalog/views.py b/vreapis/catalog/views.py new file mode 100644 index 00000000..91ea44a2 --- /dev/null +++ b/vreapis/catalog/views.py @@ -0,0 +1,3 @@ +from django.shortcuts import render + +# Create your views here. diff --git a/vreapis/cells/admin.py b/vreapis/cells/admin.py deleted file mode 100644 index 21c068ba..00000000 --- a/vreapis/cells/admin.py +++ /dev/null @@ -1,6 +0,0 @@ -from django.contrib import admin - -from cells.models import Catalog, Cell - -admin.site.register(Catalog) -admin.site.register(Cell) diff --git a/vreapis/cells/fixtures/catalogs.yaml b/vreapis/cells/fixtures/catalogs.yaml deleted file mode 100644 index 7c3dac5a..00000000 --- a/vreapis/cells/fixtures/catalogs.yaml +++ /dev/null @@ -1,7 +0,0 @@ -- model: cells.Catalog - pk: 1 - fields: - name: "Summer School Catalog" - description: "Summer School Catalog" - vlab: 1 - diff --git a/vreapis/cells/migrations/0001_initial.py b/vreapis/cells/migrations/0001_initial.py deleted file mode 100644 index ceb260f2..00000000 --- a/vreapis/cells/migrations/0001_initial.py +++ /dev/null @@ -1,38 +0,0 @@ -# Generated by Django 4.0.5 on 2022-06-29 20:00 - -from django.db import migrations, models -import django.db.models.deletion - - -class Migration(migrations.Migration): - - initial = True - - dependencies = [ - ('virtual_labs', '0004_remove_virtuallab_profiles_and_more'), - ] - - operations = [ - migrations.CreateModel( - name='Catalog', - fields=[ - ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('name', models.CharField(max_length=100, null=True)), - ('description', models.TextField(null=True)), - ('vlab', models.ForeignKey(null=True, on_delete=django.db.models.deletion.DO_NOTHING, to='virtual_labs.virtuallab')), - ], - ), - migrations.CreateModel( - name='Cell', - fields=[ - ('uuid', models.UUIDField(primary_key=True, serialize=False)), - ('name', models.CharField(max_length=100, null=True)), - ('description', models.TextField(null=True)), - ('container_image', models.CharField(max_length=100, null=True)), - ('registry_url', models.URLField(null=True)), - ('repository_url', models.URLField(null=True)), - ('cell_metadata', models.JSONField(null=True)), - ('catalog', models.ForeignKey(null=True, on_delete=django.db.models.deletion.DO_NOTHING, to='cells.catalog')), - ], - ), - ] diff --git a/vreapis/cells/models.py b/vreapis/cells/models.py deleted file mode 100644 index 63f885ce..00000000 --- a/vreapis/cells/models.py +++ /dev/null @@ -1,27 +0,0 @@ -from django.db import models -from virtual_labs.models import VirtualLab - - -class Catalog(models.Model): - - name = models.CharField(max_length=100, null=True) - description = models.TextField(null=True) - vlab = models.ForeignKey(VirtualLab, on_delete=models.DO_NOTHING, null=True) - - def __str__(self): - return self.name - - -class Cell(models.Model): - - uuid = models.UUIDField(primary_key=True) - name = models.CharField(max_length=100, null=True) - description = models.TextField(null=True) - container_image = models.CharField(max_length=100, null=True) - registry_url = models.URLField(null=True) - repository_url = models.URLField(null=True) - cell_metadata = models.JSONField(null=True) - catalog = models.ForeignKey(Catalog, on_delete=models.DO_NOTHING, null=True) - - def __str__(self) -> str: - return self.name \ No newline at end of file diff --git a/vreapis/cells/serializers.py b/vreapis/cells/serializers.py deleted file mode 100644 index a27b942e..00000000 --- a/vreapis/cells/serializers.py +++ /dev/null @@ -1,11 +0,0 @@ -from dataclasses import fields -from rest_framework import serializers -from . import models - - -class CellSerializer(serializers.ModelSerializer): - - class Meta: - - model = models.Cell - fields = "__all__" \ No newline at end of file diff --git a/vreapis/cells/views.py b/vreapis/cells/views.py deleted file mode 100644 index 42398932..00000000 --- a/vreapis/cells/views.py +++ /dev/null @@ -1,30 +0,0 @@ -from rest_framework import mixins, viewsets -from rest_framework.permissions import IsAuthenticated - -from vreapis.views import GetSerializerMixin - -from . import models, serializers - - -class CellsViewSet(GetSerializerMixin, - mixins.RetrieveModelMixin, - mixins.UpdateModelMixin, - mixins.ListModelMixin, - mixins.CreateModelMixin, - viewsets.GenericViewSet): - queryset = models.Cell.objects.all() - serializer_class = serializers.CellSerializer - serializer_action_classes = { - 'list': serializers.CellSerializer - } - - def get_permissions(self): - if self.action in ['list', 'retrieve']: - permission_classes = [] - else: - permission_classes = [IsAuthenticated] - return [permission() for permission in permission_classes] - - def create(self, request, *args, **kwargs): - print(request.data) - return super().create(request) diff --git a/vreapis/common.py b/vreapis/common.py new file mode 100644 index 00000000..79a3bbf1 --- /dev/null +++ b/vreapis/common.py @@ -0,0 +1,26 @@ +import logging +import urllib3 +import os +import requests.adapters + +max_retry_count: int = 10 +initial_retry_delay: int | float = 0.1 +max_retry_delay: int | float = 5 + +default_varchar_length: int = 4000 + +project_root: str = os.path.dirname(os.path.abspath(__file__)) + +# customized requests.Session [w/ auto retry] +session = requests.Session() +retry_adapter = requests.adapters.HTTPAdapter(max_retries=urllib3.Retry(total=max_retry_count, backoff_factor=initial_retry_delay, backoff_max=max_retry_delay, status_forcelist=[500, 502, 503, 504])) +session.mount('http://', retry_adapter) +session.mount('https://', retry_adapter) + +# global logger +logger = logging.getLogger(__name__) + + +def retry_delay(cumulated_retry_count: int, initial_delay: int | float = initial_retry_delay, max_delay: int | float = max_retry_delay) -> int | float: + """delay is in seconds""" + return min(initial_delay * 2 ** cumulated_retry_count, max_delay) diff --git a/vreapis/containerizer/RContainerizer.py b/vreapis/containerizer/RContainerizer.py new file mode 100644 index 00000000..f517230e --- /dev/null +++ b/vreapis/containerizer/RContainerizer.py @@ -0,0 +1,101 @@ +import logging +import os + +import jinja2 + +import common +from catalog.models import Cell + +handler = logging.StreamHandler() +handler.setLevel(logging.DEBUG) + +# Create a formatter for the log messages +formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') + +# Add the formatter to the handler +handler.setFormatter(formatter) + +# Add the handler to the logger +common.logger.addHandler(handler) + + +class RContainerizer: + @staticmethod + def get_files_info(cell=None, cells_path=None): + if not os.path.exists(cells_path): + os.mkdir(cells_path) + cell_path = os.path.join(cells_path, cell.task_name) + + cell_file_name = 'task.R' + dockerfile_name = 'Dockerfile' + environment_file_name = 'environment.yaml' + + if os.path.exists(cell_path): + for files in os.listdir(cell_path): + path = os.path.join(cell_path, files) + if os.path.isfile(path): + os.remove(path) + else: + os.mkdir(cell_path) + + cell_file_path = os.path.join(cell_path, cell_file_name) + dockerfile_file_path = os.path.join(cell_path, dockerfile_name) + env_file_path = os.path.join(cell_path, environment_file_name) + return { + 'cell': {'file_name': cell_file_name, 'path': cell_file_path}, + 'dockerfile': {'file_name': dockerfile_name, 'path': dockerfile_file_path}, + 'environment': {'file_name': environment_file_name, 'path': env_file_path}, + } + + @staticmethod + def map_dependencies(dependencies, module_name_mapping): + dependencies = map(lambda x: 'r-' + x['name'], dependencies) # prefix `r-` is naming convention + dependencies = map(lambda x: module_name_mapping.get('r', {}).get(x, x), dependencies) + set_conda_deps = set(dependencies) + set_pip_deps = set() + set_conda_deps.discard(None) + set_conda_deps.discard(None) + return set_conda_deps, set_pip_deps + + @staticmethod + def build_templates(cell: Cell=None, files_info=None, module_name_mapping=None): + # we also want to always add the id to the input parameters + inputs = cell.inputs + if isinstance(cell.types, list): + cell.types = {} # In R, builtin list is used to represent both arrays and dicts. From R side of the frontend, an empty list is serialized into [] rather than {} and it seems difficult to handle this in R. So we correct it here. + types = cell.types + inputs.append('id') + cell.concatenate_all_inputs() + types['id'] = 'str' + common.logger.debug("inputs: " + str(cell.inputs)) + common.logger.debug("types: " + str(cell.types)) + common.logger.debug("params: " + str(cell.params)) + common.logger.debug("outputs: " + str(cell.outputs)) + + common.logger.debug('files_info: ' + str(files_info)) + common.logger.debug('cell.dependencies: ' + str(cell.dependencies)) + + loader = jinja2.FileSystemLoader(searchpath=f'{common.project_root}/templates') + template_env = jinja2.Environment(loader=loader, trim_blocks=True, lstrip_blocks=True) + + template_cell = template_env.get_template('R_cell_template.jinja2') + template_dockerfile = template_env.get_template('dockerfile_template_conda.jinja2') + + compiled_code = template_cell.render(cell=cell, deps=cell.generate_dependencies(), types=cell.types, confs=cell.generate_configuration()) + cell.container_source = compiled_code + dependencies = cell.generate_dependencies() + r_dependencies = [] + for dep in dependencies: + r_dep = dep.replace('import ', '') + install_packages = 'if (!requireNamespace("' + r_dep + '", quietly = TRUE)) {\n\tinstall.packages("' + r_dep + '", repos="http://cran.us.r-project.org")\n}' + r_dependencies.append(install_packages) + library = 'library(' + r_dep + ')' + r_dependencies.append(library) + + template_cell.stream(cell=cell, deps=r_dependencies, types=cell.types, confs=cell.generate_configuration()).dump(files_info['cell']['path']) + template_dockerfile.stream(task_name=cell.task_name, base_image=cell.base_image).dump(files_info['dockerfile']['path']) + + set_conda_deps, set_pip_deps = RContainerizer.map_dependencies(cell.dependencies, module_name_mapping) + common.logger.debug('cell.dependencies.conda: ' + str(cell.dependencies)) + template_conda = template_env.get_template('conda_env_template.jinja2') + template_conda.stream(base_image=cell.base_image, conda_deps=list(set_conda_deps), pip_deps=list(set_pip_deps)).dump(files_info['environment']['path']) diff --git a/vreapis/containerizer/__init__.py b/vreapis/containerizer/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/vreapis/containerizer/admin.py b/vreapis/containerizer/admin.py new file mode 100644 index 00000000..8c38f3f3 --- /dev/null +++ b/vreapis/containerizer/admin.py @@ -0,0 +1,3 @@ +from django.contrib import admin + +# Register your models here. diff --git a/vreapis/containerizer/apps.py b/vreapis/containerizer/apps.py new file mode 100644 index 00000000..eb32c03b --- /dev/null +++ b/vreapis/containerizer/apps.py @@ -0,0 +1,6 @@ +from django.apps import AppConfig + + +class ContainerizerConfig(AppConfig): + default_auto_field = 'django.db.models.BigAutoField' + name = 'containerizer' diff --git a/vreapis/containerizer/jupytext.toml b/vreapis/containerizer/jupytext.toml new file mode 100644 index 00000000..9250219a --- /dev/null +++ b/vreapis/containerizer/jupytext.toml @@ -0,0 +1,2 @@ +[tool.jupytext] +split_at_heading = true diff --git a/vreapis/containerizer/migrations/__init__.py b/vreapis/containerizer/migrations/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/vreapis/containerizer/models.py b/vreapis/containerizer/models.py new file mode 100644 index 00000000..b28b04f6 --- /dev/null +++ b/vreapis/containerizer/models.py @@ -0,0 +1,3 @@ + + + diff --git a/vreapis/containerizer/tests.py b/vreapis/containerizer/tests.py new file mode 100644 index 00000000..28bca5fd --- /dev/null +++ b/vreapis/containerizer/tests.py @@ -0,0 +1,449 @@ +import datetime +import glob +import os +import json +import shlex +import shutil +import subprocess +import sys +import uuid +from pathlib import Path + +from django.test import TestCase, Client +from django.conf import settings +from django.contrib.auth.models import User +from rest_framework.authtoken.models import Token +from urllib.parse import urlencode +import requests +import nbformat +from github import Github +from slugify import slugify +from tornado.gen import sleep + +import common +from containerizer.views import CellsHandler +from services.extractor.pyextractor import PyExtractor +from services.extractor.rextractor import RExtractor +from services.converter import ConverterReactFlowChart +from catalog.models import Cell + +base_path = '' +if os.path.exists('resources'): + base_path = 'resources' +elif os.path.exists('tests/resources/'): + base_path = 'tests/resources/' + +dummy_username = f'test' +dummy_password = '0' + +def create_dummy_credentials(): + dummy_username = f'test' + dummy_password = '0' + try: + dummy_user: User = User.objects.get(username=dummy_username) + except User.DoesNotExist: + dummy_user = User.objects.create_user(dummy_username, password=dummy_password) + django_token, created = Token.objects.get_or_create(user=dummy_user, key=os.getenv("DJANGO_TOKEN")) + + +def get_auth_header() -> dict[str, str]: + # return {'Authorization': f'Token {django_token}'} + return {'Authorization': f'Token {os.getenv("DJANGO_TOKEN")}'} + + +class GetBaseImagesTestCase(TestCase): + @staticmethod + def Keycloak_login() -> dict[str, any]: + return requests.post(settings.KEYCLOAK_LOGIN_URL, headers={'Content-Type': 'application/x-www-form-urlencoded'}, data=urlencode({'client_id': 'myclient', 'grant_type': 'password', 'scope': 'openid', 'username': 'u', 'password': 'u'}), verify=settings.ALLOW_INSECURE_TLS).json() + + def test_get_base_images(self): + client = Client() + create_dummy_credentials() + client.login(username=dummy_username, password=dummy_password) + + response = client.get('/api/containerizer/baseimagetags', headers=get_auth_header()) + self.assertEqual(response.status_code, 200) + images = response.json() + self.assertIsInstance(images, dict) + self.assertGreater(len(images), 0) + + +class ExtractorTestCase(TestCase): + # Reference parameter values for `test_param_values_*.json` + param_values_ref = { + 'param_float': '1.1', + 'param_int': '1', + 'param_list': '[1, 2, 3]', + 'param_string': 'param_string value', + 'param_string_with_comment': 'param_string value', + } + + def setUp(self): # use setUp instead of __init__, or 'uncaught TypeError: __init__() takes 1 positional argument but 2 were given' + super().__init__() + + def create_cell(self, payload_path=None): + with open(payload_path, 'r') as file: + payload = json.load(file) + + cell_index = payload['cell_index'] + notebook = nbformat.reads(json.dumps(payload['notebook']), nbformat.NO_CONVERT) + source = notebook.cells[cell_index].source + if payload['kernel'] == "IRkernel": + extractor = RExtractor(notebook, source) + else: + extractor = PyExtractor(notebook, source) + + title = source.partition('\n')[0] + title = slugify(title) if title and title[ + 0] == "#" else "Untitled" + + if 'JUPYTERHUB_USER' in os.environ: + title += '-' + slugify(os.environ['JUPYTERHUB_USER']) + + ins = {} + outs = {} + params = {} + confs = [] + dependencies = [] + + # Check if cell is code. If cell is for example markdown we get execution from 'extractor.infere_cell_inputs(source)' + if notebook.cells[cell_index].cell_type == 'code': + ins = extractor.infer_cell_inputs() + outs = extractor.infer_cell_outputs() + + confs = extractor.extract_cell_conf_ref() + dependencies = extractor.infer_cell_dependencies(confs) + + node_id = str(uuid.uuid4())[:7] + cell = Cell( + node_id=node_id, + title=title, + task_name=slugify(title.lower()), + original_source=source, + inputs=ins, + outputs=outs, + params=params, + confs=confs, + dependencies=dependencies, + container_source="" + ) + if notebook.cells[cell_index].cell_type == 'code': + cell.integrate_configuration() + params = extractor.extract_cell_params(cell.original_source) + cell.add_params(params) + cell.add_param_values(params) + + return cell + + def extract_cell(self, payload_path): + # Check if file exists + if os.path.exists(payload_path): + cell = self.create_cell(payload_path) + node = ConverterReactFlowChart.get_node(cell.node_id, cell.title, cell.inputs, cell.outputs, cell.params, ) + chart = {'offset': {'x': 0, 'y': 0, }, 'scale': 1, 'nodes': {cell.node_id: node}, 'links': {}, 'selected': {}, 'hovered': {}, } + cell.chart_obj = chart + return cell.toJSON() + return None + + def test_extract_cell(self): + notebooks_json_path = os.path.join(base_path, 'notebooks') + notebooks_files = glob.glob(os.path.join(notebooks_json_path, "*.json")) + for notebook_file in notebooks_files: + cell = self.extract_cell(notebook_file) + if cell: + cell = json.loads(cell) + for conf_name in (cell['confs']): + self.assertFalse('conf_' in cell['confs'][conf_name].split('=')[1], 'conf_ values should not contain conf_ prefix in ''assignment') + # All params should have matching values + for param_name in cell['params']: + self.assertTrue(param_name in cell['param_values']) + + # For notebook_file test_param_values_*.json, extracted params should match with self.param_values_ref + if os.path.basename(notebook_file) in ['test_param_values_Python.json', 'test_param_values_R.json', ]: + for param_name in cell['params']: + self.assertTrue(cell['param_values'][param_name] == self.param_values_ref[param_name]) + + +class ExtractorHandlerTestCase(TestCase): + def test(self): + notebooks_json_path = os.path.join(base_path, 'notebooks') + notebooks_files = glob.glob(os.path.join(notebooks_json_path, "*.json")) + for notebook_file in notebooks_files: + with open(notebook_file, 'r') as file: + notebook = json.load(file) + file.close() + client = Client() + create_dummy_credentials() + response = client.post('/api/containerizer/extract', headers=get_auth_header(), data=notebook, content_type="application/json") + self.assertEqual(response.status_code, 200) + # get JSON response + JSON_response = json.loads(response.data) + self.assertIsNotNone(JSON_response) + cell = notebook['notebook']['cells'][notebook['cell_index']] + print('cell: ', cell) + + +class CellsHandlerTestCase(TestCase): + cells_path = os.path.join(str(Path.home()), 'NaaVRE', 'cells') + github_url_repos = 'https://api.github.com/repos' + + def setUp(self): + self.client = Client() + create_dummy_credentials() + + def create_cell_and_add_to_cat(self, cell_path=None) -> (dict, dict): + print('Creating cell from: ', cell_path) + with open(cell_path, 'r') as file: + cell = json.load(file) + # notebook_dict = {} + # if 'notebook_dict' in cell: + # notebook_dict = cell['notebook_dict'] + # test_cell = Cell(cell['title'], cell['task_name'], cell['original_source'], cell['inputs'], + # cell['outputs'], + # cell['params'], cell['confs'], cell['dependencies'], cell['container_source'], + # cell['chart_obj'], cell['node_id'], cell['kernel'], notebook_dict) + # test_cell.types = cell['types'] + # test_cell.base_image = cell['base_image'] + selected_keys: list[str] = ['title', 'task_name', 'original_source', 'inputs', 'outputs', 'params', 'confs', 'dependencies', 'container_source', 'chart_obj', 'node_id', 'kernel', 'types', 'base_image', 'notebook_dict'] + test_cell = {k: cell[k] for k in selected_keys if k in cell} + return test_cell, cell + + def delete_text(self, file_path, text_to_delete): + # Read the file + with open(file_path, 'r') as file: + lines = file.readlines() + + # Remove the text from each line + updated_lines = [] + for line in lines: + updated_line = line.replace(text_to_delete, '') + updated_lines.append(updated_line) + + # Write the updated lines to the file + with open(file_path, 'w') as file: + file.writelines(updated_lines) + + def wait_for_github_api_resources(self): + github = Github(CellsHandler.get_repositories()[0]['token']) + rate_limit = github.get_rate_limit() + while rate_limit.core.remaining <= 0: + reset = rate_limit.core.reset + # Calculate remaining time for reset + remaining_time = reset.timestamp() - datetime.datetime.now().timestamp() + common.logger.debug(f'Remaining time for reset: {remaining_time} s') + common.logger.debug(f'API rate exceeded, waiting') + common.logger.debug(f'Sleeping for: {remaining_time + 1}') + sleep(remaining_time + 1) + rate_limit = github.get_rate_limit() + + def get_github_workflow_runs(self, owner=None, repository_name=None, t_utc=None, token=None): + workflow_runs_url = CellsHandlerTestCase.github_url_repos + '/' + owner + '/' + repository_name + '/actions/runs' + if t_utc: + t_start = (t_utc - datetime.timedelta(minutes=1)).strftime("%Y-%m-%dT%H:%M:%SZ") + t_stop = (t_utc + datetime.timedelta(minutes=1)).strftime("%Y-%m-%dT%H:%M:%SZ") + workflow_runs_url += f"?created={t_start}..{t_stop}" + headers = {'Accept': 'application/vnd.github.v3+json'} + if token: + headers['Authorization'] = 'Bearer ' + token + workflow_runs = common.session.get(url=workflow_runs_url, verify=False, headers=headers) + if workflow_runs.status_code != 200: + return None + workflow_runs_json = json.loads(workflow_runs.text) + return workflow_runs_json + + def get_github_workflow_jobs(self, jobs_url=None, token=None): + headers = {'Accept': 'application/vnd.github.v3+json'} + if token: + headers['Authorization'] = 'Bearer ' + token + jobs = common.session.get(url=jobs_url, verify=False, headers=headers) + if jobs.status_code == 200: + return json.loads(jobs.text) + else: + raise Exception('Error getting jobs for workflow run: ' + jobs.text) + + def find_job(self, + wf_id=None, + wf_creation_utc=None, + owner=None, + repository_name=None, + token=None, + job_id=None, + ): + f""" Find Github workflow job + + If job_id is set, retrieve it through + https://api.github.com/repos/{owner}/{repository_name}/actions/jobs/{job_id} + + Else, get all workflows runs created around wf_creation_utc through + https://api.github.com/repos/{owner}/{repository_name}/actions/runs + and find the one matching {wf_id} + """ + if job_id: + jobs_url = CellsHandlerTestCase.github_url_repos + '/' + owner + '/' + repository_name + '/actions/jobs/' + str(job_id) + self.wait_for_github_api_resources() + job = self.get_github_workflow_jobs(jobs_url, token=token) + return job + self.wait_for_github_api_resources() + runs = self.get_github_workflow_runs( + owner=owner, + repository_name=repository_name, + t_utc=wf_creation_utc, + token=token) + if not runs: + return None + for run in runs['workflow_runs']: + jobs_url = run['jobs_url'] + self.wait_for_github_api_resources() + jobs = self.get_github_workflow_jobs(jobs_url, token=token) + for job in jobs['jobs']: + if job['name'] == wf_id: + job['head_sha'] = run['head_sha'] + return job + return None + + def wait_for_job(self, + wf_id=None, + wf_creation_utc=None, + owner=None, + repository_name=None, + token=None, + job_id=None, + timeout=200, + wait_for_completion=False, + ): + """ Call find_job until something is returned or timeout is reached + + :param wf_id: passed to find_job + :param wf_creation_utc: passed to find_job + :param owner: passed to find_job + :param repository_name: passed to find_job + :param token: passed to find_job + :param job_id: passed to find_job + :param timeout: timeout in seconds + :param wait_for_completion: wait for the job's status to be 'complete' + + :return: job or None + """ + start_time = datetime.datetime.now().timestamp() # seconds + stop_time = start_time + timeout + while datetime.datetime.now().timestamp() < stop_time: + job = self.find_job( + wf_id=wf_id, + wf_creation_utc=wf_creation_utc, + owner=owner, + repository_name=repository_name, + token=token, + job_id=job_id, + ) + if job: + if not wait_for_completion: + return job + if wait_for_completion and (job['status'] == 'completed'): + return job + sleep(5) + + def test(self): + test_dir: str = '/tmp/data' + if not os.path.exists(test_dir): + os.makedirs(test_dir) + test_file: str = f'{test_dir}/hello.txt' + if not os.path.exists(test_file): + with open(test_file, 'w'): + pass + cells_json_path = os.path.join(base_path, 'cells') + cells_files = os.listdir(cells_json_path) + test_cells = [] + for cell_file in cells_files: + cell_path = os.path.join(cells_json_path, cell_file) + test_cell, cell = self.create_cell_and_add_to_cat(cell_path=cell_path) + response = self.client.post('/api/containerizer/addcell', content_type='application/json', headers=get_auth_header(), data=test_cell) + self.assertEqual(200, response.status_code) + wf_id = response.data['wf_id'] + wf_creation_utc = datetime.datetime.now(tz=datetime.timezone.utc) + dispatched_github_workflow = response.data['dispatched_github_workflow'] + test_cells.append({ + 'wf_id': wf_id, + 'wf_creation_utc': wf_creation_utc, + 'dispatched_github_workflow': dispatched_github_workflow, + }) + if 'skip_exec' in cell and cell['skip_exec']: + continue + if 'python' in test_cell['kernel'] and 'skip_exec': + cell_path = os.path.join(CellsHandlerTestCase.cells_path, test_cell['task_name'], 'task.py') + print('---------------------------------------------------') + print('Executing cell: ', cell_path) + if 'example_inputs' in cell: + exec_args = [sys.executable, cell_path] + cell['example_inputs'] + else: + exec_args = [sys.executable, cell_path] + + cell_exec = subprocess.Popen(exec_args, stdout=subprocess.PIPE) + text = cell_exec.communicate()[0] + print(text) + print("stdout:", cell_exec.stdout) + print("stderr:", cell_exec.stderr) + print("return code:", cell_exec.returncode) + print('---------------------------------------------------') + self.assertEqual(0, cell_exec.returncode, 'Cell execution failed: ' + cell_file) + elif test_cell['kernel'] == 'IRkernel' and 'skip_exec': + cell_path = os.path.join(CellsHandlerTestCase.cells_path, test_cell['task_name'], 'task.R') + run_local_cell_path = os.path.join(CellsHandlerTestCase.cells_path, test_cell['task_name'], 'run_local.R') + shutil.copy(cell_path, run_local_cell_path) + self.delete_text(run_local_cell_path, 'setwd(\'/app\')') + example_inputs = '' + if 'example_inputs' in cell: + example_inputs = ' '.join(cell['example_inputs']) + command = 'Rscript ' + run_local_cell_path + ' ' + example_inputs + R_dependencies: list[str] = ['optparse', 'jsonlite', ] # Some versions [e.g., older ones] may make this test fail + for dependency in R_dependencies: + result = subprocess.run(['Rscript', '-e', f'if(!require("{dependency}")) install.packages("{dependency}")'], capture_output=True, text=True) + print(result.stdout) + print(result.stderr) + self.assertEqual(0, result.returncode, result.stderr) + result = subprocess.run(shlex.split(command), capture_output=True, text=True) + self.assertEqual(0, result.returncode, result.stderr) + + cat_repositories = CellsHandler.get_repositories() + repo = cat_repositories[0] + repo_token = repo['token'] + owner, repository_name = repo['url'].removeprefix('https://github.com/').split('/') + if '.git' in repository_name: + repository_name = repository_name.split('.git')[0] + + updated_cells = list(filter( + lambda cell: cell['dispatched_github_workflow'], + test_cells, + )) + + for cell in updated_cells: + # Get job id (many calls to the GitHub API) + job = self.wait_for_job( + wf_id=cell['wf_id'], + wf_creation_utc=cell['wf_creation_utc'], + owner=owner, + repository_name=repository_name, + token=repo_token, + job_id=None, + timeout=300, + wait_for_completion=False, + ) + cell['job'] = job + + for cell in updated_cells: + # Wait for job completion (fewer calls) + job = self.wait_for_job( + wf_id=cell['wf_id'], + wf_creation_utc=None, + owner=owner, + repository_name=repository_name, + token=repo_token, + job_id=cell['job']['id'], + timeout=300, + wait_for_completion=True, + ) + cell['job'] = job + + for cell in updated_cells: + self.assertIsNotNone(cell['job'], 'Job not found') + self.assertEqual('completed', cell['job']['status'], 'Job not completed') + self.assertEqual('success', cell['job']['conclusion'], 'Job not successful') diff --git a/vreapis/containerizer/urls.py b/vreapis/containerizer/urls.py new file mode 100644 index 00000000..bd4d66d4 --- /dev/null +++ b/vreapis/containerizer/urls.py @@ -0,0 +1,14 @@ +from django.urls import path +from . import views + +urlpatterns = [ + path('baseimagetags', views.get_base_images), + path('extract', views.ExtractorHandler.as_view()), + path('cell', views.CellsHandler.as_view({ + 'get': 'list', + 'post': 'create', + })), + path('addcell', views.CellsHandler.as_view({ + 'post': 'create', + })), +] diff --git a/vreapis/containerizer/views.py b/vreapis/containerizer/views.py new file mode 100644 index 00000000..742022fe --- /dev/null +++ b/vreapis/containerizer/views.py @@ -0,0 +1,565 @@ +import bisect +import importlib +import os +import json +import re +import sys +import copy +import time +import hashlib +import uuid +from typing import Optional +from pathlib import Path +import subprocess + +import autopep8 +from distro import distro +import jinja2 +from django.db.models import QuerySet +from rest_framework.authentication import BaseAuthentication +from rest_framework.exceptions import ValidationError +from rest_framework.permissions import IsAuthenticated, BasePermission +from rest_framework.response import Response +from rest_framework.request import Request +from rest_framework.views import APIView +from rest_framework import status +from rest_framework.decorators import api_view, authentication_classes, permission_classes +from rest_framework import viewsets +from rest_framework.request import Request +import requests +import nbformat +import jsonschema +from slugify import slugify +from github import Github, UnknownObjectException +from django.conf import settings + +from catalog.serializers import CellSerializer +from containerizer.RContainerizer import RContainerizer +from db.repository import Repository +from services.extractor.extractor import DummyExtractor +from services.extractor.pyheaderextractor import PyHeaderExtractor +from services.extractor.rheaderextractor import RHeaderExtractor +from services.extractor.pyextractor import PyExtractor +from services.extractor.rextractor import RExtractor +from services.converter import ConverterReactFlowChart +import utils.cors +# from auth.simple import StaticTokenAuthentication +from catalog.models import Cell + +import common + + +def return_error(err_msg: str = 'Unknown ERROR', e: Optional[Exception] = None, stat: status = status.HTTP_400_BAD_REQUEST) -> Response: + common.logger.error(err_msg, exc_info=e) + return Response(err_msg, status=stat) + + +@api_view(['GET']) +# @authentication_classes([StaticTokenAuthentication]) +@permission_classes([IsAuthenticated]) +def get_base_images(request): + url: str = os.getenv('BASE_IMAGE_TAGS_URL', 'https://github.com/QCDIS/NaaVRE-flavors/releases/latest/download/base_image_tags.json') + common.logger.debug(f'Base image tags URL: {url}') + try: + response = common.session.get(url) + response.raise_for_status() + dat: dict[str, dict[str, str]] = response.json() + except (requests.ConnectionError, requests.HTTPError, requests.JSONDecodeError,) as e: + return return_error(f'Error loading base image tags from {url}', e, stat=status.HTTP_500_INTERNAL_SERVER_ERROR) + return Response(dat, headers=utils.cors.get_CORS_headers(request)) + + +class ExtractorHandler(APIView): + # authentication_classes = [StaticTokenAuthentication] + permission_classes = [IsAuthenticated] + + def extract_cell_by_index(self, notebook: nbformat.notebooknode, cell_index: int) -> nbformat.NotebookNode: + new_nb = copy.deepcopy(notebook) + if cell_index < len(notebook.cells): + new_nb.cells = [notebook.cells[cell_index]] + return new_nb + + def set_notebook_kernel(self, notebook: nbformat.NotebookNode, kernel: str) -> nbformat.NotebookNode: + new_nb: nbformat.NotebookNode = copy.deepcopy(notebook) + # Replace kernel name in the notebook metadata + new_nb.metadata['kernelspec'] = { + 'name': kernel, + 'display_name': kernel, + 'language': kernel, + } + return new_nb + + def get(self, request: Request): + return return_error("Operation not supported.", stat=status.HTTP_400_BAD_REQUEST) + + def post(self, request: Request): + payload = request.data + # common.logger.debug('ExtractorHandler. payload: ' + json.dumps(payload, indent=4)) + if 'rmarkdown' in payload: + # Directly setting `NotebookNode.metadata['jupytext'] = {'split_at_heading': True, }` has no use. I don't know why. So we don't use lib jupytext here. + command_jupytext = f'source {settings.VENV_ACTIVATOR}; jupytext --from Rmd --to ipynb --opt split_at_heading=true -o -' + process_jupytext = subprocess.Popen(command_jupytext, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True, executable='/bin/bash') + stdout, stderr = process_jupytext.communicate(input=payload['rmarkdown'].encode()) + process_jupytext.stdin.close() + process_jupytext.wait() + payload['notebook'] = stdout.decode() + cell_index = payload['cell_index'] - bisect.bisect_right(payload['rmarkdown_offset_indices'], payload['cell_index']) - 1 + else: # if 'notebook' in payload + cell_index = payload['cell_index'] + kernel = payload['kernel'] + if isinstance(payload['notebook'], dict): + payload['notebook'] = json.dumps(payload['notebook']) + notebook = nbformat.reads(payload['notebook'], nbformat.NO_CONVERT) + if 'rmarkdown' in payload: # parsermd ignores md cells with \n only. We need to remove them here to keep consistency + filtered_cells = [notebook.cells[0]] + for i in range(1, len(notebook.cells)): + cell = notebook.cells[i] + if not (cell['cell_type'] == 'markdown' and all(line == os.linesep for line in cell['source']) and notebook.cells[i - 1]['cell_type'] == 'code'): + filtered_cells.append(cell) + notebook.cells = filtered_cells + common.logger.debug(cell_index) + + source = notebook.cells[cell_index].source + + if notebook.cells[cell_index].cell_type != 'code': + # dummy extractor for non-code cells (e.g. markdown) + extractor = DummyExtractor(notebook, source) + else: + # extractor based on the cell header + try: + # extractor = HeaderExtractor(notebook, source) + if 'python' in kernel.lower(): + extractor = PyHeaderExtractor(notebook, source) + elif 'r' in kernel.lower(): + extractor = RHeaderExtractor(notebook, source) + except jsonschema.ValidationError as e: + return return_error('Error in cell header', e, stat=status.HTTP_400_BAD_REQUEST) + + # Extractor based on code analysis. Used if the cell has no header, or if some values are not specified in the header + if not extractor.is_complete(): + if kernel == "IRkernel": + code_extractor = RExtractor(notebook, source) + else: + code_extractor = PyExtractor(notebook, source) + extractor.add_missing_values(code_extractor) + + extracted_nb = self.extract_cell_by_index(notebook, cell_index) + if kernel == "IRkernel": + extracted_nb = self.set_notebook_kernel(extracted_nb, 'R') + else: + extracted_nb = self.set_notebook_kernel(extracted_nb, 'python3') + + # initialize variables + title = source.partition('\n')[0].strip() + title = slugify(title) if title and title[0] == "#" else "Untitled" + + # if 'JUPYTERHUB_USER' in os.environ: + # title += '-' + slugify(os.environ['JUPYTERHUB_USER']) + if 'JUPYTERHUB_USER' in payload: + title += '-' + slugify(payload['JUPYTERHUB_USER']) + + # If any of these change, we create a new cell in the catalog. This matches the cell properties saved in workflows. + cell_identity_dict = {'title': title, 'params': extractor.params, 'inputs': extractor.ins, 'outputs': extractor.outs, } + cell_identity_str = json.dumps(cell_identity_dict, sort_keys=True) + node_id = hashlib.sha1(cell_identity_str.encode()).hexdigest()[:7] + # node_id = str(time.time_ns())[len('0x'):] + + cell = Cell( + node_id=node_id, + title=title, + task_name=slugify(title.lower()), + original_source=source, + inputs=extractor.ins, + outputs=extractor.outs, + params={}, + confs=extractor.confs, + dependencies=extractor.dependencies, + container_source="", + kernel=kernel, + notebook_dict=extracted_nb.dict() + ) + cell.add_inputs(extractor.ins) + cell.add_outputs(extractor.outs) + cell.integrate_configuration() + extractor.params = extractor.extract_cell_params(cell.original_source) + cell.add_params(extractor.params) + cell.add_param_values(extractor.params) + + node = ConverterReactFlowChart.get_node( + node_id, + title, + set(extractor.ins), + set(extractor.outs), + extractor.params, + ) + + chart = {'offset': {'x': 0, 'y': 0, }, 'scale': 1, 'nodes': {node_id: node}, 'links': {}, 'selected': {}, 'hovered': {}, } + + cell.chart_obj = chart + return Response(cell.toJSON()) + + +class CellsHandler(viewsets.ModelViewSet): + common.logger.debug(f"CELL_GITHUB: {os.getenv('CELL_GITHUB')}") + common.logger.debug(f"BASE_PATH: {os.getenv('BASE_PATH')}") + queryset: QuerySet = Cell.objects.all() + serializer_class = CellSerializer + # authentication_classes: list[BaseAuthentication] = [StaticTokenAuthentication] + permission_classes: list[BasePermission] = [IsAuthenticated] + cells_path: str = os.path.join(str(Path.home()), 'NaaVRE', 'cells') + github_url_repos: str = 'https://api.github.com/repos' + github_workflow_file_name: str = 'build-push-docker.yml' + github_url: str = os.getenv('CELL_GITHUB') + github_token: str = os.getenv('CELL_GITHUB_TOKEN') + registry_url: str = os.getenv('REGISTRY_URL') + repo_name: str = github_url.split('https://github.com/')[1] + + def write_cell_to_file(self, current_cell): + Path('/tmp/workflow_cells/cells').mkdir(parents=True, exist_ok=True) + with open('/tmp/workflow_cells/cells/' + current_cell.task_name + '.json', 'w') as f: + f.write(current_cell.toJSON()) + f.close() + + def load_module_name_mapping(self): + module_mapping_url = os.getenv('MODULE_MAPPING_URL') + module_mapping = {} + if module_mapping_url: + resp = common.session.get(module_mapping_url) + module_mapping = json.loads(resp.text) + module_name_mapping_path = os.path.join(str(Path.home()), 'NaaVRE', 'module_name_mapping.json') + if not os.path.exists(module_name_mapping_path): + with open(module_name_mapping_path, 'w') as module_name_mapping_file: + json.dump(module_mapping, module_name_mapping_file, indent=4) + module_name_mapping_file.close() + + module_name_mapping_file = open(module_name_mapping_path) + loaded_module_name_mapping = json.load(module_name_mapping_file) + loaded_module_name_mapping.update(module_mapping) + module_name_mapping_file.close() + return loaded_module_name_mapping + + def get_files_info(self, cell=None): + if not os.path.exists(self.cells_path): + os.mkdir(self.cells_path) + cell_path = os.path.join(self.cells_path, cell.task_name) + + cell_file_name = 'task.py' + dockerfile_name = 'Dockerfile' + environment_file_name = 'environment.yaml' + + notebook_file_name = None + if 'visualize-' in cell.task_name: + notebook_file_name = 'task.ipynb' + if os.path.exists(cell_path): + for files in os.listdir(cell_path): + path = os.path.join(cell_path, files) + if os.path.isfile(path): + os.remove(path) + else: + os.mkdir(cell_path) + + cell_file_path = os.path.join(cell_path, cell_file_name) + dockerfile_file_path = os.path.join(cell_path, dockerfile_name) + env_file_path = os.path.join(cell_path, environment_file_name) + info = { + 'cell': {'file_name': cell_file_name, 'path': cell_file_path}, + 'dockerfile': {'file_name': dockerfile_name, 'path': dockerfile_file_path}, + 'environment': {'file_name': environment_file_name, 'path': env_file_path} + } + if notebook_file_name: + info['notebook'] = {'file_name': notebook_file_name, 'path': os.path.join(cell_path, notebook_file_name)} + return info + + def is_standard_module(self, module_name): + if module_name in sys.builtin_module_names: + return True + installation_path = None + try: + installation_path = importlib.import_module(module_name).__file__ + except ImportError: + return False + linux_os = distro.id() + return 'dist-packages' not in installation_path if linux_os == 'Ubuntu' else 'site-packages' not in installation_path + + def map_dependencies(self, dependencies=None, module_name_mapping=None): + set_conda_deps = set([]) + set_pip_deps = set([]) + for dep in dependencies: + if 'module' in dep and dep['module']: + if '.' in dep['module']: + module_name = dep['module'].split('.')[0] + else: + module_name = dep['module'] + elif 'name' in dep and dep['name']: + module_name = dep['name'] + if module_name: + conda_package = True + pip_package = False + if module_name in module_name_mapping['conda'].keys(): + module_name = module_name_mapping['conda'][module_name] + pip_package = False + conda_package = True + if module_name in module_name_mapping['pip'].keys(): + module_name = module_name_mapping['pip'][module_name] + pip_package = True + conda_package = False + if module_name is None: + continue + if not self.is_standard_module(module_name): + if conda_package: + set_conda_deps.add(module_name) + if pip_package: + set_pip_deps.add(module_name) + return set_conda_deps, set_pip_deps + + def build_templates(self, cell=None, files_info=None, module_name_mapping=None): + common.logger.debug('files_info: ' + str(files_info)) + common.logger.debug('cell.dependencies: ' + str(cell.dependencies)) + set_conda_deps, set_pip_deps = self.map_dependencies(dependencies=cell.dependencies, module_name_mapping=module_name_mapping, ) + loader = jinja2.FileSystemLoader(searchpath=f'{common.project_root}/templates') + template_env = jinja2.Environment(loader=loader, trim_blocks=True, lstrip_blocks=True) + + if cell.title.startswith('visualize-'): + template_cell = template_env.get_template('vis_cell_template.jinja2') + if cell.notebook_dict: + notebook_path = os.path.join(files_info['cell']['path']).replace('.py', '.ipynb') + with open(notebook_path, 'w') as f: + f.write(json.dumps(cell.notebook_dict, indent=4)) + f.close() + else: + template_cell = template_env.get_template('py_cell_template.jinja2') + template_dockerfile = template_env.get_template('dockerfile_template_conda.jinja2') + + compiled_code = template_cell.render(cell=cell, deps=cell.generate_dependencies(), types=cell.types, confs=cell.generate_configuration_dict()) + + compiled_code = autopep8.fix_code(compiled_code) + cell.container_source = compiled_code + + template_cell.stream(cell=cell, deps=cell.generate_dependencies(), types=cell.types, confs=cell.generate_configuration_dict()).dump(files_info['cell']['path']) + template_dockerfile.stream(task_name=cell.task_name, base_image=cell.base_image).dump( + files_info['dockerfile']['path']) + + template_conda = template_env.get_template('conda_env_template.jinja2') + template_conda.stream(base_image=cell.base_image, conda_deps=list(set_conda_deps), pip_deps=list(set_pip_deps)).dump(files_info['environment']['path']) + + def git_hash(self, contents): + s = hashlib.sha1() + s.update(('blob %u\0' % len(contents)).encode('utf-8')) + s.update(contents) + return s.hexdigest() + + def create_or_update_cell_in_repository(self, task_name, repository, files_info): + files_updated = False + code_content_hash = None + for f_type, f_info in files_info.items(): + f_name = f_info['file_name'] + f_path = f_info['path'] + with open(f_path, 'rb') as f: + local_content = f.read() + local_hash = self.git_hash(local_content) + try: + remote_hash = repository.get_contents(path=task_name + '/' + f_name).sha + except UnknownObjectException: + remote_hash = None + common.logger.debug(f'local_hash: {local_hash}; remote_hash: {remote_hash}') + if remote_hash is None: + repository.create_file(path=task_name + '/' + f_name, message=task_name + ' creation', content=local_content, ) + elif remote_hash != local_hash: + repository.update_file(path=task_name + '/' + f_name, message=task_name + ' update', content=local_content, sha=remote_hash, ) + files_updated = True + if f_type == 'cell': + code_content_hash = local_hash + return files_updated, code_content_hash + + def query_registry_for_image(self, image_repo, image_name): + m = re.match(r'^docker.io/(\w+)', image_name) + if m: + # Docker Hub + url = f'https://hub.docker.com/v2/repositories/{m.group(1)}/{image_name}' + headers = {} + else: + # OCI registries + domain = image_repo.split('/')[0] + path = '/'.join(image_repo.split('/')[1:]) + url = f'https://{domain}/v2/{path}/{image_name}/tags/list' + # OCI registries require authentication, even for public registries. + # The token should be set in the $OCI_TOKEN environment variable. + # For ghcr.io, connections still succeed when $OCI_TOKEN is unset (this results in header "Authorization: Bearer None", which grants access to public registries, although it is not officially documented). If this fails, or when accessing private registries, OCI_TOKEN should be a base64-encoded GitHub classic access token with the read:packages scope. + headers = {"Authorization": f"Bearer {os.getenv('OCI_TOKEN')}", } + response = common.session.get(url, headers=headers) + if response.status_code == 200: + return json.loads(response.content.decode('utf-8')) + else: + return None + + def dispatch_github_workflow(self, owner, repository_name, task_name, files_info, repository_token, image, wf_id=None, image_version=None): + url = CellsHandler.github_url_repos + '/' + owner + '/' + repository_name + '/actions/workflows/' + CellsHandler.github_workflow_file_name + '/dispatches' + resp = common.session.post( + url=url, + json={ + 'ref': 'refs/heads/main', + 'inputs': { + 'build_dir': task_name, + 'dockerfile': files_info['dockerfile']['file_name'], + 'image_repo': image, + 'image_tag': task_name, + 'id': wf_id, + 'image_version': image_version, + } + }, + verify=False, + headers={'Accept': 'application/vnd.github.v3+json', 'Authorization': 'token ' + repository_token} + ) + return resp + + @classmethod + def get_registry_url(cls, registry_url, github_url): + """ Convert registry URL + + https://hub.docker.com/u/my_username/ -> docker.io/my_username + oci://ghcr.io/my_username/my_repo/ -> ghcr.io/my_username/my_repo + oci://my_domain/my/custom/path/ -> my_domain/my/custom/path + None -> ghcr.io url, derived from github_url + + Resulting urls can be converted to pullable, e.g.: + + docker pull {url}/{image_name}:{tag} + + where image_name doesn't contain any path information (e.g. my-cell-name) + + """ + if registry_url: + m = re.match(r'^https://hub\.docker\.com/u/(\w+)/?$', registry_url) + if m: + return f"docker.io/{m.group(1)}" + m = re.match(r'^oci://([\w\./-]+?)/?$', registry_url) + if m: + return m.group(1) + raise ValueError(f"Could not parse registry url: {registry_url}") + else: + m = re.match(r'^https://github.com/([\w-]+/[\w-]+)(?:\.git)?', github_url) + if m: + return f"ghcr.io/{m.group(1).lower()}" + + def get_registry_credentials(self) -> list[dict[str, str]]: + return [Repository(CellsHandler.registry_url, CellsHandler.registry_url, None).__dict__] + + @classmethod + def get_repositories(cls) -> list[dict[str, str]]: + return [Repository(CellsHandler.repo_name, CellsHandler.github_url, CellsHandler.github_token).__dict__] + + def create(self, request: Request, *args, **kwargs): + try: + Cell_field_names = [f.name for f in Cell._meta.get_fields()] + filtered_incoming_body = {k: request.data[k] for k in Cell_field_names if k in request.data} + current_cell = Cell(**filtered_incoming_body) + # current_cell = Cell(**request.data) + current_cell.clean_code() + current_cell.clean_title() + current_cell.clean_task_name() + except Exception as ex: + return return_error('Error setting cell', ex) + + # common.logger.debug('current_cell: ' + current_cell.toJSON()) + all_vars = current_cell.params + current_cell.inputs + current_cell.outputs + for param_name in all_vars: + if param_name not in current_cell.types: + return return_error(f'{param_name} not in types') + + if not hasattr(current_cell, 'base_image'): + return return_error(f'{current_cell.task_name} has not selected base image') + try: + serializer: CellSerializer = self.get_serializer(data=request.data) + try: + serializer.is_valid(raise_exception=True) + except ValidationError as ex: + if 'node_id' in ex.detail and str(ex.detail) == 'cell with this node id already exists.': + Cell.objects.update(**serializer.validated_data) + else: + Cell.objects.create(**serializer.validated_data) + # instance, created = Cell.objects.update_or_create(node_id=serializer.validated_data['node_id'], defaults=serializer.validated_data) + except Exception as ex: + return return_error('Error adding or updating cell in catalog', ex) + + if os.getenv('DEBUG'): + self.write_cell_to_file(current_cell) + + if not os.path.exists(self.cells_path): + os.makedirs(self.cells_path, exist_ok=True) + + cell_path = os.path.join(self.cells_path, current_cell.task_name) + + if os.path.exists(cell_path): + for files in os.listdir(cell_path): + path = os.path.join(cell_path, files) + if os.path.isfile(path): + os.remove(path) + else: + os.makedirs(cell_path, exist_ok=True) + + registry_credentials = self.get_registry_credentials() + if not registry_credentials or len(registry_credentials) <= 0: + return return_error('Registry credentials not found') + image_repo = registry_credentials[0]['url'] + if not image_repo: + return return_error(f'Registry not found. Registry credentials:{os.linesep}{registry_credentials}') + + if current_cell.kernel == "IRkernel": + files_info = RContainerizer.get_files_info(cell=current_cell, cells_path=CellsHandler.cells_path) + RContainerizer.build_templates(cell=current_cell, files_info=files_info, module_name_mapping=self.load_module_name_mapping(), ) + elif 'python' in current_cell.kernel.lower(): + files_info = self.get_files_info(cell=current_cell) + self.build_templates(cell=current_cell, files_info=files_info, module_name_mapping=self.load_module_name_mapping(), ) + else: + return return_error(f'Kernel {current_cell.kernel} not supported') + + # upload to GIT + cat_repositories = self.get_repositories() + + repo_token = cat_repositories[0]['token'] + if not repo_token: + return return_error('Repository token not found') + + gh_token = Github(cat_repositories[0]['token']) + url_repos = cat_repositories[0]['url'] + if not url_repos: + return return_error('Repository url not found') + + owner = url_repos.split('https://github.com/')[1].split('/')[0] + repository_name = url_repos.split('https://github.com/')[1].split('/')[1] + if '.git' in repository_name: + repository_name = repository_name.split('.git')[0] + retry_count: int = 0 + while True: + try: + gh_repository = gh_token.get_repo(owner + '/' + repository_name) + break + except Exception as ex: + time.sleep(common.retry_delay(retry_count)) + retry_count += 1 + if retry_count >= common.max_retry_count: + return return_error(f'Error getting repository', ex) + do_dispatch_github_workflow, image_version = self.create_or_update_cell_in_repository(task_name=current_cell.task_name, repository=gh_repository, files_info=files_info, ) + wf_id = str(uuid.uuid4()) + + if os.getenv('DEBUG') and os.getenv('DEBUG').lower() == 'true': + do_dispatch_github_workflow = True + else: + image_info = self.query_registry_for_image(image_repo=image_repo, image_name=current_cell.task_name, ) + common.logger.debug(f'image_info: {image_info}') + if not image_info: + do_dispatch_github_workflow = True + + image_version = image_version[:7] + if do_dispatch_github_workflow: + resp = self.dispatch_github_workflow(owner, repository_name, current_cell.task_name, files_info, repo_token, image_repo, wf_id=wf_id, image_version=image_version) + if resp.status_code != 201 and resp.status_code != 200 and resp.status_code != 204: + return return_error(resp.text) + current_cell.set_image_version(image_version) + Cell.objects.filter(task_name=current_cell.task_name).delete() + current_cell.save() + + return Response({'wf_id': wf_id, 'dispatched_github_workflow': do_dispatch_github_workflow, 'image_version': image_version}) + + +CellsHandler.registry_url = CellsHandler.get_registry_url(CellsHandler.registry_url, CellsHandler.github_url) diff --git a/vreapis/db/__init__.py b/vreapis/db/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/vreapis/db/repository.py b/vreapis/db/repository.py new file mode 100644 index 00000000..cb20d86e --- /dev/null +++ b/vreapis/db/repository.py @@ -0,0 +1,9 @@ +class Repository: + name: str + url: str + token: str + + def __init__(self, name, url, token) -> None: + self.name = name + self.url = url + self.token = token diff --git a/vreapis/db/sdia_credentials.py b/vreapis/db/sdia_credentials.py new file mode 100644 index 00000000..71ab83fe --- /dev/null +++ b/vreapis/db/sdia_credentials.py @@ -0,0 +1,9 @@ +class SDIACredentials: + username: str + password: str + endpoint: str + + def __init__(self, username, password, endpoint) -> None: + self.username = username + self.password = password + self.endpoint = endpoint diff --git a/vreapis/docker-compose.yaml b/vreapis/docker-compose.yaml index 9420187e..7e7ed8ba 100644 --- a/vreapis/docker-compose.yaml +++ b/vreapis/docker-compose.yaml @@ -31,6 +31,9 @@ services: - DJANGO_SUPERUSER_PASSWORD=admin - DJANGO_TOKEN=token - SECRET_KEY=secret + # for GitHub action to read envvar + - CELL_GITHUB=${CELL_GITHUB} + - CELL_GITHUB_TOKEN=${CELL_GITHUB_TOKEN} ports: - '127.0.0.1:8000:8000' healthcheck: diff --git a/vreapis/manage.py b/vreapis/manage.py index 88ef4fb5..4b3bc637 100755 --- a/vreapis/manage.py +++ b/vreapis/manage.py @@ -2,7 +2,7 @@ """Django's command-line utility for administrative tasks.""" import os import sys -import dotenv +# import dotenv # always complains `No module named 'dotenv'` I don't know why def main(): @@ -20,5 +20,5 @@ def main(): if __name__ == '__main__': - dotenv.read_dotenv() + # dotenv.read_dotenv() main() diff --git a/vreapis/requirements.txt b/vreapis/requirements.txt index d6dd3d5f..66fd927a 100644 --- a/vreapis/requirements.txt +++ b/vreapis/requirements.txt @@ -7,7 +7,20 @@ gunicorn==21.2.0 pyyaml==6.0.1 requests==2.31.0 django-dotenv==1.4.2 +# python-dotenv==1.0.1 django-cors-headers==4.3.1 django-extensions==3.2.3 psycopg2==2.9.9 django-probes==1.7.0 +pyjwt==2.8.0 +cryptography==42.0.7 +nbformat==5.10.4 +python-slugify==8.0.4 +colorhash==2.0.0 +rpy2==3.5.11 +pyflakes==3.2.0 +pytype==2024.4.11 +autopep8==2.2.0 +distro==1.9.0 +pygithub==2.3.0 +jupytext==1.16.3 diff --git a/vreapis/services/converter.py b/vreapis/services/converter.py new file mode 100644 index 00000000..8294bc76 --- /dev/null +++ b/vreapis/services/converter.py @@ -0,0 +1,185 @@ +import uuid + +from colorhash import ColorHash + + +class ConverterReactFlowChart: + + @staticmethod + def get_node(node_id, title, ins, outs, params): + + node = {} + position = {} + ports = {} + properties = {} + + node['id'] = node_id + node['type'] = 'input-output' + position['x'] = 35 + position['y'] = 15 + node['position'] = position + properties['title'] = title + properties['vars'] = list() + properties['params'] = list(params) + properties['inputs'] = list(ins) + properties['outputs'] = list(outs) + properties['og_node_id'] = node_id + node['properties'] = properties + + for i in ins: + ports[i] = {} + ports[i]['properties'] = {} + ports[i]['id'] = i + ports[i]['type'] = 'left' + ports[i]['properties']['color'] = ColorHash(i).hex + properties['vars'].append({ + 'name': i, + 'direction': 'input', + 'type': 'datatype', + 'color': ports[i]['properties']['color'] + }) + + for o in outs: + ports[o] = {} + ports[o]['properties'] = {} + ports[o]['id'] = o + ports[o]['type'] = 'right' + ports[o]['properties']['color'] = ColorHash(o).hex + properties['vars'].append({ + 'name': o, + 'direction': 'output', + 'type': 'datatype', + 'color': ports[o]['properties']['color'] + }) + + node['ports'] = ports + + return node + + +class ConverterReactFlow: + + @staticmethod + def get_input_nodes(ins): + nodes = [] + idx = 0 + + for i in ins: + i_node = {} + i_node['data'] = {} + i_node['position'] = {} + i_node['id'] = i + i_node['type'] = 'input' + i_node['data']['label'] = i + i_node['position']['x'] = 10 + idx * 200 + i_node['position']['y'] = 10 + nodes.append(i_node) + idx += 1 + + return nodes + + @staticmethod + def get_output_nodes(outs): + nodes = [] + idx = 0 + + for o in outs: + o_node = {} + o_node['data'] = {} + o_node['position'] = {} + o_node['id'] = o + o_node['type'] = 'output' + o_node['data']['label'] = o + o_node['position']['x'] = 10 + idx * 200 + o_node['position']['y'] = 200 + nodes.append(o_node) + idx += 1 + + return nodes + + @staticmethod + def get_default_node(title, uuid): + d_node = {} + d_node['data'] = {} + d_node['position'] = {} + d_node['id'] = uuid + d_node['data']['label'] = title + d_node['position']['x'] = 100 + d_node['position']['y'] = 100 + + return d_node + + @staticmethod + def get_edges(d_node_id, ins, outs): + edges = [] + + for i in ins: + i_edge = {} + i_edge['id'] = "%s-%s" % (i, d_node_id) + i_edge['source'] = i + i_edge['target'] = d_node_id + i_edge['animated'] = True + edges.append(i_edge) + + for o in outs: + o_edge = {} + o_edge['id'] = "%s-%s" % (d_node_id, o) + o_edge['source'] = d_node_id + o_edge['target'] = o + o_edge['animated'] = True + edges.append(o_edge) + + return edges + + +class ConverterFlume: + + @staticmethod + def get_ports(ins, outs): + colors = [ + 'yellow', + 'orange', + 'red', + 'pink', + 'purple', + 'blue', + 'green', + 'grey' + ] + + ports = set() + ports_types = [] + + ports.update(ins) + ports.update(outs) + + color_i = 0 + for port in ports: + p_type = {} + p_type['type'] = port + p_type['name'] = port + p_type['label'] = port + p_type['color'] = colors[color_i], + color_i = (color_i + 1) % len(colors) + + ports_types.append(p_type) + + return ports_types + + @staticmethod + def get_node(source, ports, ins, outs): + title = source.partition('\n')[0] + short_uuid = str(uuid.uuid4())[:7] + + n_type = {} + n_type['type'] = short_uuid + n_type['label'] = title if title[0] == "#" else "Untitled %s" % short_uuid + n_type['description'] = short_uuid + + ports_in = [p for p in ports if p['name'] in ins] + ports_out = [p for p in ports if p['name'] in outs] + + n_type['inputs'] = ports_in + n_type['outputs'] = ports_out + + return n_type diff --git a/vreapis/services/extractor/__init__.py b/vreapis/services/extractor/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/vreapis/services/extractor/cell_header.schema.json b/vreapis/services/extractor/cell_header.schema.json new file mode 100644 index 00000000..7e84d04e --- /dev/null +++ b/vreapis/services/extractor/cell_header.schema.json @@ -0,0 +1,198 @@ +{ + "$schema": "http://json-schema.org/draft/2020-12/schema", + "$ref": "#/definitions/CellHeader", + "definitions": { + "CellHeader": { + "title": "Cell header", + "type": "object", + "additionalProperties": true, + "properties": { + "NaaVRE": { + "$ref": "#/definitions/NaaVRECellMetadata" + } + }, + "required": [ + "NaaVRE" + ] + }, + "NaaVRECellMetadata": { + "title": "NaaVRE cell metadata", + "type": "object", + "additionalProperties": false, + "properties": { + "cell": { + "$ref": "#/definitions/Cell" + } + }, + "required": [ + "cell" + ] + }, + "Cell": { + "title": "Cell", + "type": "object", + "additionalProperties": false, + "properties": { + "inputs": { + "type": "array", + "items": { + "$ref": "#/definitions/IOElement" + } + }, + "outputs": { + "type": "array", + "items": { + "$ref": "#/definitions/IOElement" + } + }, + "params": { + "type": "array", + "items": { + "$ref": "#/definitions/ParamElement" + } + }, + "confs": { + "type": "array", + "items": { + "$ref": "#/definitions/ConfElement" + } + }, + "dependencies": { + "type": "array", + "items": { + "$ref": "#/definitions/DependencyElement" + } + } + } + }, + "VarType": { + "title": "Variable type", + "type": "string", + "enum": [ + "Integer", + "Float", + "String", + "List" + ] + }, + "VarDefaultValue": { + "title": "Param variable default_value", + "description": "Used to pre-fill workflow parameters", + "type": ["string", "number", "array"] + }, + "VarAssignation": { + "title": "Conf variable assignation property", + "description": "Full variable assignation, e.g. 'conf_answer = 42'", + "type": "string" + }, + "ElementVarName": { + "title": "Variable name", + "description": "simple variable name, as string ('name')", + "type": "string" + }, + "ElementVarNameType": { + "title": "Variable name and type", + "description": "simple variable name and type, as object ({'name': 'type'})", + "type": "object", + "additionalProperties": false, + "patternProperties": { + "^.*$": { + "$ref": "#/definitions/VarType" + } + } + }, + "IOElementVarDict": { + "title": "Full i/o variable description", + "type": "object", + "additionalProperties": false, + "minProperties": 1, + "maxProperties": 1, + "patternProperties": { + "^.*$": { + "type": "object", + "additionalProperties": false, + "properties": { + "type": {"$ref": "#/definitions/VarType"} + } + } + } + }, + "ParamElementVarDict": { + "title": "Full param variable description", + "type": "object", + "additionalProperties": false, + "minProperties": 1, + "maxProperties": 1, + "patternProperties": { + "^param_.*$": { + "type": "object", + "additionalProperties": false, + "properties": { + "type": {"$ref": "#/definitions/VarType"}, + "default_value": {"$ref": "#/definitions/VarDefaultValue"} + } + } + } + }, + "ConfElementVarDict": { + "title": "Full param variable description", + "type": "object", + "additionalProperties": false, + "minProperties": 1, + "maxProperties": 1, + "patternProperties": { + "^conf_.*$": { + "type": "object", + "additionalProperties": false, + "properties": { + "assignation": {"$ref": "#/definitions/VarAssignation"} + }, + "required": ["assignation"] + } + } + }, + "DependencyElementVarDict": { + "title": "Full param variable description", + "type": "object", + "additionalProperties": false, + "properties": { + "name": {"type": "string"}, + "asname": {"type": ["string", "null"]}, + "module": {"type": "string"} + }, + "required": ["name"] + }, + "IOElement": { + "title": "I/O element", + "description": "description input and output variable elements", + "anyOf": [ + {"$ref": "#/definitions/ElementVarName"}, + {"$ref": "#/definitions/ElementVarNameType"}, + {"$ref": "#/definitions/IOElementVarDict"} + ] + }, + "ParamElement": { + "title": "Param element", + "description": "description param variable elements", + "anyOf": [ + {"$ref": "#/definitions/ElementVarName"}, + {"$ref": "#/definitions/ElementVarNameType"}, + {"$ref": "#/definitions/ParamElementVarDict"} + ] + }, + "ConfElement": { + "title": "Conf element", + "description": "description conf variable elements", + "anyOf": [ + {"$ref": "#/definitions/ConfElementVarDict"} + ] + }, + "DependencyElement": { + "title": "Dependency element", + "description": "description dependency elements", + "anyOf": [ + {"$ref": "#/definitions/DependencyElementVarDict"} + ] + } + } +} diff --git a/vreapis/services/extractor/extractor.py b/vreapis/services/extractor/extractor.py new file mode 100644 index 00000000..dcd3fb86 --- /dev/null +++ b/vreapis/services/extractor/extractor.py @@ -0,0 +1,56 @@ +import abc + + +class Extractor(abc.ABC): + ins: dict + outs: dict + params: dict + confs: list + dependencies: list + + def __init__(self, notebook, cell_source): + self.notebook = notebook + self.cell_source = cell_source + + self.ins = self.infer_cell_inputs() + self.outs = self.infer_cell_outputs() + self.params = self.extract_cell_params(cell_source) + self.confs = self.extract_cell_conf_ref() + self.dependencies = self.infer_cell_dependencies(self.confs) + + @abc.abstractmethod + def infer_cell_inputs(self): + pass + + @abc.abstractmethod + def infer_cell_outputs(self): + pass + + @abc.abstractmethod + def extract_cell_params(self, source): + pass + + @abc.abstractmethod + def extract_cell_conf_ref(self): + pass + + @abc.abstractmethod + def infer_cell_dependencies(self, confs): + pass + + +class DummyExtractor(Extractor): + def infer_cell_inputs(self): + return {} + + def infer_cell_outputs(self): + return {} + + def extract_cell_params(self, source): + return {} + + def extract_cell_conf_ref(self): + return [] + + def infer_cell_dependencies(self, confs): + return [] diff --git a/vreapis/services/extractor/headerextractor.py b/vreapis/services/extractor/headerextractor.py new file mode 100644 index 00000000..38429c80 --- /dev/null +++ b/vreapis/services/extractor/headerextractor.py @@ -0,0 +1,245 @@ +import json +import logging +import os +import re +from typing import Literal, Union + +import jsonschema +import yaml + +from .extractor import Extractor + + +class HeaderExtractor(Extractor): + """ Extracts cells using information defined by the user in its header + + Cells should contain a comment with a yaml block defining inputs, outputs, + params and confs. Eg: + + # My cell + # --- + # NaaVRE: + # cell: + # inputs: + # - my_input: + # type: String + # - my_other_input: + # type: Integer + # outputs: + # - my_output: + # type: List + # params: + # - param_something: + # type: String + # default_value: "my default value" + # confs: + # - conf_something_else: + # assignation: "conf_something_else = 'my other value'" + # ... + [cell code] + + The document is validated with the schema `cell_header.schema.json` + + """ + ins: Union[dict, None] + outs: Union[dict, None] + params: Union[dict, None] + confs: Union[list, None] + dependencies: Union[list, None] + + def __init__(self, notebook, cell_source): + self.re_yaml_doc_in_comment = re.compile( + (r"^(?:.*\n)*" + r"\s*#\s*---\s*\n" + r"((?:\s*#.*\n)+?)" + r"\s*#\s*\.\.\.\s*\n" + ), + re.MULTILINE) + self.schema = self._load_schema() + self.cell_header = self._extract_header(cell_source) + self._external_extract_cell_params = None + + super().__init__(notebook, cell_source) + + @staticmethod + def _load_schema(): + filename = os.path.join( + os.path.dirname(__file__), + 'cell_header.schema.json') + with open(filename) as f: + schema = json.load(f) + return schema + + def enabled(self): + return self.cell_header is not None + + def is_complete(self): + return ( + (self.ins is not None) + and (self.outs is not None) + and (self.params is not None) + and (self.confs is not None) + and (self.dependencies is not None) + ) + + def _extract_header(self, cell_source): + # get yaml document from cell comments + m = self.re_yaml_doc_in_comment.match(cell_source) + if not (m and m.groups()): + return None + yaml_doc = m.group(1) + # remove comment symbol + yaml_doc = '\n'.join([ + line.lstrip().lstrip('#') + for line in yaml_doc.splitlines() + ]) + # parse yaml + header = yaml.safe_load(yaml_doc) + # validate schema + try: + jsonschema.validate(header, self.schema) + except jsonschema.ValidationError as e: + logging.getLogger().debug(f"Cell header validation error: {e}") + raise e + return header + + def add_missing_values(self, extractor: Extractor): + """ Add values not specified in the header from another extractor + (e.g. PyExtractor or RExtractor) + """ + if self.ins is None: + self.ins = extractor.ins + if self.outs is None: + self.outs = extractor.outs + if self.params is None: + self.params = extractor.params + # We store a reference to extractor.extract_cell_params because + # self.extract_cell_params is called after self.add_missing_values + # in component_containerizer.handlers.ExtractorHandler.post() + self._external_extract_cell_params = extractor.extract_cell_params + if self.confs is None: + self.confs = extractor.confs + if self.dependencies is None: + self.dependencies = extractor.dependencies + + @staticmethod + def _parse_inputs_outputs_param_items( + item: Union[str, dict], + item_type: Literal['inputs', 'outputs', 'params'], + ) -> dict: + """ Parse inputs, outputs, or params items from the header + + They can have either format + - ElementVarName: 'my_name' + - ElementVarNameType {'my_name': 'my_type'} + - IOElementVarDict {'my_name': {'type': 'my_type'}} + or ParamElementVarDict {'my_name': {'type': 'my_type', + 'default_value': 'my_value'}} + + Returns + - if item_type is 'inputs' or 'outputs': + {'name': 'my_name', 'type': 'my_type'} + - if item_type is 'params': + {'name': 'my_name', 'type': 'my_type', 'value': 'my_value'} + """ + var_dict = {} + + # ElementVarName + if isinstance(item, str): + var_dict = { + 'name': item, + 'type': None, + 'value': None, + } + elif isinstance(item, dict): + if len(item.keys()) != 1: + # this should have been caught by the schema validation + raise ValueError(f"Unexpected item in {item_type}: {item}") + var_name = list(item.keys())[0] + var_props = item[var_name] + # ElementVarNameType + if isinstance(var_props, str): + var_dict = { + 'name': var_name, + 'type': var_props, + 'value': None, + } + # IOElementVarDict or ParamElementVarDict + elif isinstance(var_props, dict): + var_dict = { + 'name': var_name, + 'type': var_props.get('type'), + 'value': var_props.get('default_value'), + } + + # Convert types + types_conversion = { + 'Integer': 'int', + 'Float': 'float', + 'String': 'str', + 'List': 'list', + None: None, + } + var_dict['type'] = types_conversion[var_dict['type']] + + # 'value' should only be kept for params + if item_type not in ['params']: + del var_dict['value'] + + return var_dict + + def _infer_cell_inputs_outputs_params( + self, + header: Union[dict, None], + item_type: Literal['inputs', 'outputs', 'params'], + ) -> Union[dict, None]: + if header is None: + return None + items = header['NaaVRE']['cell'].get(item_type) + if items is None: + return None + items = [self._parse_inputs_outputs_param_items(it, item_type) + for it in items] + return {it['name']: it for it in items} + + def infer_cell_inputs(self): + return self._infer_cell_inputs_outputs_params( + self.cell_header, + 'inputs', + ) + + def infer_cell_outputs(self): + return self._infer_cell_inputs_outputs_params( + self.cell_header, + 'outputs', + ) + + def extract_cell_params(self, source): + if self._external_extract_cell_params is not None: + return self._external_extract_cell_params(source) + return self._infer_cell_inputs_outputs_params( + self._extract_header(source), + 'params', + ) + + def extract_cell_conf_ref(self): + if self.cell_header is None: + return None + items = self.cell_header['NaaVRE']['cell'].get('confs') + if items is None: + return None + return {k: v['assignation'] for it in items for k, v in it.items()} + + def infer_cell_dependencies(self, confs): + if self.cell_header is None: + return None + items = self.cell_header['NaaVRE']['cell'].get('dependencies') + if items is None: + return None + return [ + { + 'name': it.get('name'), + 'asname': it.get('asname', None), + 'module': it.get('module', ''), + } + for it in items] diff --git a/vreapis/services/extractor/pyextractor.py b/vreapis/services/extractor/pyextractor.py new file mode 100644 index 00000000..93bc408d --- /dev/null +++ b/vreapis/services/extractor/pyextractor.py @@ -0,0 +1,282 @@ +import ast +import logging +import re +from functools import lru_cache + +from pyflakes import reporter as pyflakes_reporter, api as pyflakes_api +from pytype import config as pytype_config +from pytype.tools.annotate_ast import annotate_ast + +from .extractor import Extractor + + +class PyExtractor(Extractor): + sources: list + imports: dict + configurations: dict + global_params: dict + undefined: dict + + def __init__(self, notebook, cell_source): + # If cell_type is code and not starting with '!' + self.sources = [nbcell.source for nbcell in notebook.cells if + nbcell.cell_type == 'code' and len(nbcell.source) > 0 and nbcell.source[0] != '!'] + self.notebook_names = self.__extract_cell_names( + '\n'.join(self.sources), + infer_types=True, + ) + self.imports = self.__extract_imports(self.sources) + self.configurations = self.__extract_configurations(self.sources) + self.global_params = self.__extract_params(self.sources) + self.undefined = dict() + for source in self.sources: + self.undefined.update(self.__extract_cell_undefined(source)) + + super().__init__(notebook, cell_source) + + def __extract_imports(self, sources): + imports = {} + for s in sources: + tree = ast.parse(s) + for node in ast.walk(tree): + if isinstance(node, (ast.Import, ast.ImportFrom,)): + for n in node.names: + key = n.asname if n.asname else n.name + if key not in imports: + imports[key] = { + 'name': n.name, + 'asname': n.asname or None, + 'module': node.module if isinstance(node, ast.ImportFrom) else "" + } + return imports + + def __extract_configurations(self, sources): + configurations = {} + for s in sources: + lines = s.splitlines() + tree = ast.parse(s) + for node in ast.walk(tree): + if isinstance(node, ast.Assign): + target = node.targets[0] + if hasattr(target, 'id'): + name = node.targets[0].id + prefix = name.split('_')[0] + if prefix == 'conf' and name not in configurations: + conf_line = '' + for line in lines[node.lineno - 1:node.end_lineno]: + conf_line += line.strip() + configurations[name] = conf_line + return self.__resolve_configurations(configurations) + + def __extract_params(self, sources): + params = dict() + for s in sources: + lines = s.splitlines() + tree = ast.parse(s) + for node in ast.walk(tree): + if isinstance(node, ast.Assign) and hasattr(node.targets[0], 'id'): + name = node.targets[0].id + prefix = name.split('_')[0] + if prefix == 'param': + param_line = '' + for line in lines[node.lineno - 1:node.end_lineno]: + param_line += line.strip() + param_value = ast.unparse(node.value) + try: + # remove quotes around strings + param_value = str(ast.literal_eval(param_value)) + except ValueError: + # when param_value can't safely be parsed, + pass + params[name] = { + 'name': name, + 'type': self.notebook_names[name]['type'], + 'value': param_value, + } + return params + + def infer_cell_outputs(self): + cell_names = self.__extract_cell_names(self.cell_source) + return { + name: properties + for name, properties in cell_names.items() + if name not in self.__extract_cell_undefined(self.cell_source) + and name not in self.imports + and name in self.undefined + and name not in self.configurations + and name not in self.global_params + } + + def infer_cell_inputs(self): + cell_undefined = self.__extract_cell_undefined(self.cell_source) + return { + und: properties + for und, properties in cell_undefined.items() + if und not in self.imports + and und not in self.configurations + and und not in self.global_params + } + + def infer_cell_dependencies(self, confs): + dependencies = [] + names = self.__extract_cell_names(self.cell_source) + + for ck in confs: + names.update(self.__extract_cell_names(confs[ck])) + + for name in names: + if name in self.imports: + dependencies.append(self.imports.get(name)) + + return dependencies + + def infer_cell_conf_dependencies(self, confs): + dependencies = [] + for ck in confs: + for name in self.__extract_cell_names(confs[ck]): + if name in self.imports: + dependencies.append(self.imports.get(name)) + + return dependencies + + @staticmethod + @lru_cache + def __get_annotated_ast(cell_source): + return annotate_ast.annotate_source( + cell_source, ast, pytype_config.Options.create()) + + def __convert_type_annotation(self, type_annotation): + """ Convert type annotation to the ones supported for cell interfaces + + :param type_annotation: type annotation obtained by e.g. pytype + :return: converted type: 'int', 'float', 'str', 'list', or None + """ + # logging.getLogger(__name__).debug(f'type_annotation = {type_annotation}') + if type_annotation is None: + # logging.getLogger(__name__).debug(f'type_annotation is None') + return None + + patterns = { + 'int': [ + re.compile(r'^int$'), + ], + 'float': [ + re.compile(r'^float$'), + ], + 'str': [ + re.compile(r'^str$'), + ], + 'list': [ + re.compile(r'^List\['), + ], + None: [ + re.compile(r'^Any$'), + re.compile(r'^Dict'), + re.compile(r'^Callable'), + ] + } + for type_name, regs in patterns.items(): + for reg in regs: + if reg.match(type_annotation): + # logging.getLogger(__name__).debug(f'type_name = {type_name}') + return type_name + + logging.getLogger(__name__).debug(f'Unmatched type: {type_annotation}') + return None + + def __extract_cell_names(self, cell_source, infer_types=False): + names = dict() + if infer_types: + tree = self.__get_annotated_ast(cell_source) + else: + tree = ast.parse(cell_source) + for module in ast.walk(tree): + if isinstance(module, (ast.Name,)): + var_name = module.id + if infer_types: + try: + var_type = self.__convert_type_annotation(module.resolved_annotation) + except AttributeError: + print('__extract_cell_names failed', var_name) + var_type = None + else: + var_type = self.notebook_names[var_name]['type'] + names[module.id] = { + 'name': var_name, + 'type': var_type, + } + return names + + def __extract_cell_undefined(self, cell_source): + + flakes_stdout = StreamList() + flakes_stderr = StreamList() + rep = pyflakes_reporter.Reporter( + flakes_stdout.reset(), + flakes_stderr.reset()) + pyflakes_api.check(cell_source, filename="temp", reporter=rep) + + if rep._stderr(): + raise SyntaxError("Flakes reported the following error:" + "\n{}".format('\t' + '\t'.join(rep._stderr()))) + p = r"'(.+?)'" + + out = rep._stdout() + undef_vars = dict() + + for line in filter(lambda a: a != '\n' and 'undefined name' in a, out): + var_search = re.search(p, line) + var_name = var_search.group(1) + undef_vars[var_name] = { + 'name': var_name, + 'type': self.notebook_names[var_name]['type'], + } + return undef_vars + + def extract_cell_params(self, cell_source): + params = {} + cell_unds = self.__extract_cell_undefined(cell_source) + param_unds = [und for und in cell_unds if und in self.global_params] + for u in param_unds: + if u not in params: + params[u] = self.global_params[u] + return params + + def extract_cell_conf_ref(self): + confs = {} + cell_unds = self.__extract_cell_undefined(self.cell_source) + conf_unds = [und for und in cell_unds if und in self.configurations] + for u in conf_unds: + if u not in confs: + confs[u] = self.configurations[u] + return confs + + def __resolve_configurations(self, configurations): + resolved_configurations = {} + for k, assignment in configurations.items(): + while 'conf_' in assignment.split('=')[1]: + for conf_name, replacing_assignment in configurations.items(): + if conf_name in assignment.split('=')[1]: + assignment = assignment.replace( + conf_name, + replacing_assignment.split('=')[1], + ) + resolved_configurations[k] = assignment + configurations.update(resolved_configurations) + return configurations + + +class StreamList: + + def __init__(self): + self.out = list() + + def write(self, text): + self.out.append(text) + + def reset(self): + self.out = list() + return self + + def __call__(self): + return self.out diff --git a/vreapis/services/extractor/pyheaderextractor.py b/vreapis/services/extractor/pyheaderextractor.py new file mode 100644 index 00000000..dc7e2a56 --- /dev/null +++ b/vreapis/services/extractor/pyheaderextractor.py @@ -0,0 +1,245 @@ +import json +import logging +import os +import re +from typing import Literal, Union + +import jsonschema +import yaml + +from .extractor import Extractor + + +class PyHeaderExtractor(Extractor): + """ Extracts cells using information defined by the user in its header + + Cells should contain a comment with a yaml block defining inputs, outputs, + params and confs. Eg: + + # My cell + # --- + # NaaVRE: + # cell: + # inputs: + # - my_input: + # type: String + # - my_other_input: + # type: Integer + # outputs: + # - my_output: + # type: List + # params: + # - param_something: + # type: String + # default_value: "my default value" + # confs: + # - conf_something_else: + # assignation: "conf_something_else = 'my other value'" + # ... + [cell code] + + The document is validated with the schema `cell_header.schema.json` + + """ + ins: Union[dict, None] + outs: Union[dict, None] + params: Union[dict, None] + confs: Union[list, None] + dependencies: Union[list, None] + + def __init__(self, notebook, cell_source): + self.re_yaml_doc_in_comment = re.compile( + (r"^(?:.*\n)*" + r"\s*#\s*---\s*\n" + r"((?:\s*#.*\n)+?)" + r"\s*#\s*\.\.\.\s*\n" + ), + re.MULTILINE) + self.schema = self._load_schema() + self.cell_header = self._extract_header(cell_source) + self._external_extract_cell_params = None + + super().__init__(notebook, cell_source) + + @staticmethod + def _load_schema(): + filename = os.path.join( + os.path.dirname(__file__), + 'cell_header.schema.json') + with open(filename) as f: + schema = json.load(f) + return schema + + def enabled(self): + return self.cell_header is not None + + def is_complete(self): + return ( + (self.ins is not None) + and (self.outs is not None) + and (self.params is not None) + and (self.confs is not None) + and (self.dependencies is not None) + ) + + def _extract_header(self, cell_source): + # get yaml document from cell comments + m = self.re_yaml_doc_in_comment.match(cell_source) + if not (m and m.groups()): + return None + yaml_doc = m.group(1) + # remove comment symbol + yaml_doc = '\n'.join([ + line.lstrip().lstrip('#') + for line in yaml_doc.splitlines() + ]) + # parse yaml + header = yaml.safe_load(yaml_doc) + # validate schema + try: + jsonschema.validate(header, self.schema) + except jsonschema.ValidationError as e: + logging.getLogger().debug(f"Cell header validation error: {e}") + raise e + return header + + def add_missing_values(self, extractor: Extractor): + """ Add values not specified in the header from another extractor + (e.g. PyExtractor or RExtractor) + """ + if self.ins is None: + self.ins = extractor.ins + if self.outs is None: + self.outs = extractor.outs + if self.params is None: + self.params = extractor.params + # We store a reference to extractor.extract_cell_params because + # self.extract_cell_params is called after self.add_missing_values + # in component_containerizer.handlers.ExtractorHandler.post() + self._external_extract_cell_params = extractor.extract_cell_params + if self.confs is None: + self.confs = extractor.confs + if self.dependencies is None: + self.dependencies = extractor.dependencies + + @staticmethod + def _parse_inputs_outputs_param_items( + item: Union[str, dict], + item_type: Literal['inputs', 'outputs', 'params'], + ) -> dict: + """ Parse inputs, outputs, or params items from the header + + They can have either format + - ElementVarName: 'my_name' + - ElementVarNameType {'my_name': 'my_type'} + - IOElementVarDict {'my_name': {'type': 'my_type'}} + or ParamElementVarDict {'my_name': {'type': 'my_type', + 'default_value': 'my_value'}} + + Returns + - if item_type is 'inputs' or 'outputs': + {'name': 'my_name', 'type': 'my_type'} + - if item_type is 'params': + {'name': 'my_name', 'type': 'my_type', 'value': 'my_value'} + """ + var_dict = {} + + # ElementVarName + if isinstance(item, str): + var_dict = { + 'name': item, + 'type': None, + 'value': None, + } + elif isinstance(item, dict): + if len(item.keys()) != 1: + # this should have been caught by the schema validation + raise ValueError(f"Unexpected item in {item_type}: {item}") + var_name = list(item.keys())[0] + var_props = item[var_name] + # ElementVarNameType + if isinstance(var_props, str): + var_dict = { + 'name': var_name, + 'type': var_props, + 'value': None, + } + # IOElementVarDict or ParamElementVarDict + elif isinstance(var_props, dict): + var_dict = { + 'name': var_name, + 'type': var_props.get('type'), + 'value': var_props.get('default_value'), + } + + # Convert types + types_conversion = { + 'Integer': 'int', + 'Float': 'float', + 'String': 'str', + 'List': 'list', + None: None, + } + var_dict['type'] = types_conversion[var_dict['type']] + + # 'value' should only be kept for params + if item_type not in ['params']: + del var_dict['value'] + + return var_dict + + def _infer_cell_inputs_outputs_params( + self, + header: Union[dict, None], + item_type: Literal['inputs', 'outputs', 'params'], + ) -> Union[dict, None]: + if header is None: + return None + items = header['NaaVRE']['cell'].get(item_type) + if items is None: + return None + items = [self._parse_inputs_outputs_param_items(it, item_type) + for it in items] + return {it['name']: it for it in items} + + def infer_cell_inputs(self): + return self._infer_cell_inputs_outputs_params( + self.cell_header, + 'inputs', + ) + + def infer_cell_outputs(self): + return self._infer_cell_inputs_outputs_params( + self.cell_header, + 'outputs', + ) + + def extract_cell_params(self, source): + if self._external_extract_cell_params is not None: + return self._external_extract_cell_params(source) + return self._infer_cell_inputs_outputs_params( + self._extract_header(source), + 'params', + ) + + def extract_cell_conf_ref(self): + if self.cell_header is None: + return None + items = self.cell_header['NaaVRE']['cell'].get('confs') + if items is None: + return None + return {k: v['assignation'] for it in items for k, v in it.items()} + + def infer_cell_dependencies(self, confs): + if self.cell_header is None: + return None + items = self.cell_header['NaaVRE']['cell'].get('dependencies') + if items is None: + return None + return [ + { + 'name': it.get('name'), + 'asname': it.get('asname', None), + 'module': it.get('module', ''), + } + for it in items] diff --git a/vreapis/services/extractor/rextractor.py b/vreapis/services/extractor/rextractor.py new file mode 100644 index 00000000..7ea963c1 --- /dev/null +++ b/vreapis/services/extractor/rextractor.py @@ -0,0 +1,460 @@ +import os +import re +import tempfile + +import rpy2.rinterface as rinterface +import rpy2.robjects as robjects +import rpy2.robjects.packages as rpackages +from rpy2.robjects.packages import importr +from rpy2.rinterface_lib.callbacks import logger as rpy2_logger +from rpy2.robjects import conversion, default_converter +import logging + +rpy2_logger.setLevel(logging.WARNING) + +from .extractor import Extractor + +# Create an R environment +r_env = robjects.globalenv + +robject_converter = default_converter # + conversion.localconverter(default_converter) +# robjects.conversion.set_conversion(robject_converter) + +# install R packages +# robjects.r(''' +# install_package_with_retry <- function(package_name, max_attempts = 5) { +# for(i in 1:max_attempts) { +# print(paste("Attempt", i, "to install", package_name)) +# tryCatch({ +# install.packages(package_name, quiet = TRUE) +# print(paste(package_name, "installed successfully.")) +# return(TRUE) +# }, warning = function(w) { +# print(paste("Warning while installing", package_name, ":", w)) +# Sys.sleep(2) +# }, error = function(e) { +# print(paste("Failed to install", package_name, ":", e)) +# Sys.sleep(2) +# }) +# } +# return(FALSE) +# } +# ''') +# packnames = ('rlang', 'lobstr', 'purrr', 'renv',) +# for p in packnames: +# if not rpackages.isinstalled(p): +# robjects.r(f'install_package_with_retry("{p}")') + +# This R code is used to obtain all assignment variables (source https://adv-r.hadley.nz/expressions.html) +r_env["result"] = robjects.r(""" +library(rlang) +library(lobstr) +library(purrr) + +expr_type <- function(x) { + if (rlang::is_syntactic_literal(x)) { + "constant" + } else if (is.symbol(x)) { + "symbol" + } else if (is.call(x)) { + "call" + } else if (is.pairlist(x)) { + "pairlist" + } else { + typeof(x) + } +} + +switch_expr <- function(x, ...) { + switch(expr_type(x), + ..., + stop("Don't know how to handle type ", typeof(x), call. = FALSE) + ) +} + +recurse_call <- function(x) { + switch_expr(x, + # Base cases + symbol = , + constant = , + + # Recursive cases + call = , + pairlist = + ) +} + +logical_abbr_rec <- function(x) { + switch_expr(x, + constant = FALSE, + symbol = as_string(x) %in% c("F", "T") + ) +} + +logical_abbr <- function(x) { + logical_abbr_rec(enexpr(x)) +} + +find_assign_rec <- function(x) { + switch_expr(x, + constant = , + symbol = character() + ) +} +find_assign <- function(x) unique(find_assign_rec(enexpr(x))) + +flat_map_chr <- function(.x, .f, ...) { + purrr::flatten_chr(purrr::map(.x, .f, ...)) +} + +find_assign_rec <- function(x) { + switch_expr(x, + # Base cases + constant = , + symbol = character(), + + # Recursive cases + pairlist = flat_map_chr(as.list(x), find_assign_rec), + call = { + if (is_call(x, "<-") || is_call(x, "=")) { # TODO: also added is_call(x, "=") here + if (typeof(x[[2]]) == "symbol"){ # TODO: added the type check here + as_string(x[[2]]) + } + } else { + flat_map_chr(as.list(x), find_assign_rec) + } + } + ) +} +""") + +# Load the base R package for parsing and evaluation +base = importr('base') + + +# TODO: create an interface such that it can be easily extended to other kernels + +class RExtractor(Extractor): + sources: list + imports: dict + configurations: dict + global_params: dict + undefined: dict + + def __init__(self, notebook, cell_source): + self.sources = [nbcell.source for nbcell in notebook.cells if + nbcell.cell_type == 'code' and len(nbcell.source) > 0] + + self.imports = self.__extract_imports(self.sources) + self.configurations = self.__extract_configurations(self.sources) + self.global_params = self.__extract_params(self.sources) + self.undefined = dict() + for source in self.sources: + self.undefined.update(self.__extract_cell_undefined(source)) + + super().__init__(notebook, cell_source) + + def __extract_imports(self, sources): + robjects.conversion.set_conversion(robject_converter) + renv = rpackages.importr('renv') + + imports = {} + for s in sources: + packages = [] + + ''' Approach 1: Simple regex. + this matches the following cases: require(pkg), library(pkg), library("pkg"), library(package=pkg), library(package="pkg") + ''' + # packages = re.findall(r'(?:library|require)\((?:package=)?(?:")?(\w+)(?:")?\)', s) + + ''' Approach 2: Static analysis using 'renv' package. + this approach is more safe as it covers more cases and checks comments + ''' + with tempfile.NamedTemporaryFile(delete=False, suffix='.R') as tmp_file: + tmp_file.write(s.encode()) + tmp_file.flush() + function_list = renv.dependencies(tmp_file.name) + + # transpose renv dependencies to readable dependencies + transposed_list = list(map(list, zip(*function_list))) + packages = [row[1] for row in transposed_list] + tmp_file.close() + os.remove(tmp_file.name) + + # format the packages + for package in packages: + imports[package] = { + # as name and module are specific to Python packages. So you can probably leave them out here + 'name': package, + 'asname': '', + 'module': '' + } + return imports + + def __extract_configurations(self, sources): + configurations = {} + for s in sources: + parsed_expr = base.parse(text=s, keep_source=True) + parsed_expr_py = robjects.conversion.rpy2py(parsed_expr) + lines = s.splitlines() + + # loop through all assignment variables + assignment_variables = self.assignment_variables(s) + for variable in assignment_variables: + + # the prefix should be 'conf' + if not (variable.split("_")[0] == "conf"): + continue + + # find the line of the assignment. (TODO) this approach assumes that there is only one expression in one line. + # this might not work when we have something like: a <- 3; b = 7 + for line in lines: + matches = re.findall(r'{}\s*(=|<-)'.format(variable), line) + + if len(matches) > 0 and variable not in configurations: + configurations[variable] = line + break + return configurations + + def __extract_params(self, sources): # check source https://adv-r.hadley.nz/expressions.html) + params = {} + for s in sources: + lines = s.splitlines() + + '''Approach 1: Naive way + Find all variable assignments with a prefix of "param"''' + # pattern = r"param_[a-zA-Z0-9_]{0,}" + # matches = re.findall(pattern, s) + # Extract the variable names from the matches + # for match in matches: + # params.add(match) + + '''Approach 2: Look at the AST''' + assignment_variables = self.assignment_variables(s) + for variable in assignment_variables: + + # the prefix should be 'param' + if not (variable.split("_")[0] == "param"): + continue + + # find the line of the assignment. (TODO) this approach assumes that there is only one expression in one line. + # this might not work when we have something like: a <- 3; b = 7 + param_value = '' + for line in lines: + m = re.match(r'{}\s*(?:=|<-)(.*?)\s*(#.*?)?$'.format(variable), line) + if m: + param_value = m.group(1).strip(" \"' ") + + params[variable] = { + 'name': variable, + 'type': None, + 'value': param_value, + } + return params + + def infer_cell_outputs(self): + cell_names = self.__extract_cell_names(self.cell_source) + return { + name: properties + for name, properties in cell_names.items() + if name not in self.__extract_cell_undefined(self.cell_source) + and name not in self.imports + and name in self.undefined + and name not in self.configurations + and name not in self.global_params + } + + def infer_cell_inputs(self): + cell_undefined = self.__extract_cell_undefined(self.cell_source) + return { + und: properties + for und, properties in cell_undefined.items() + if und not in self.imports + and und not in self.configurations + and und not in self.global_params + } + + def infer_cell_dependencies(self, confs): + # TODO: check this code, you have removed logic. + # we probably like to only use dependencies that are necessary to execute the cell + # however this is challenging in R as functions are non-scoped + dependencies = [] + for name in self.imports: + dependencies.append(self.imports.get(name)) + return dependencies + + def infer_cell_conf_dependencies(self, confs): + dependencies = [] + for ck in confs: + for name in self.__extract_cell_names(confs[ck]): + if name in self.imports: + dependencies.append(self.imports.get(name)) + + return dependencies + + def get_function_parameters(self, cell_source): + result = [] + + # Approach 1: Naive Regex + functions = re.findall(r'function\s*\((.*?)\)', cell_source) + for params in functions: + result.extend(re.findall(r'\b\w+\b', params)) + + # Approach 2: AST based + # TODO + + return list(set(result)) + + def get_iterator_variables(self, cell_source): + result = [] + + # Approach 1: Naive Regex. This means that iterator variables are in the following format: + # for ( .....) + result = re.findall(r'for\s*\(\s*([a-zA-Z0-9.]+)\s+in', cell_source) + + # Approach 2: Parse AST. Much cleaner option as iterator variables can appear in differen syntaxes. + # TODO + + return result + + def __extract_cell_names(self, cell_source): + parsed_r = robjects.r['parse'](text=cell_source) + vars_r = robjects.r['all.vars'](parsed_r) + + # Challenge 1: Function Parameters + function_parameters = self.get_function_parameters(cell_source) + vars_r = list(filter(lambda x: x not in function_parameters, vars_r)) + + # Challenge 2: Built-in Constants + built_in_cons = ["T", "F", "pi", "is.numeric", "mu", "round"] + vars_r = list(filter(lambda x: x not in built_in_cons, vars_r)) + + # Challenge 3: Iterator Variables + iterator_variables = self.get_iterator_variables(cell_source) + vars_r = list(filter(lambda x: x not in iterator_variables, vars_r)) + + # Challenge 4: Apply built-in functions + # MANUALLY SOLVED + + # Challenge 5: Libraries + vars_r = list(filter(lambda x: x not in self.imports, vars_r)) + + # Challenge 6: Variable-based data access + # MANUALLY SOLVED + + vars_r = { + name: { + 'name': name, + 'type': None, + } + for name in vars_r + } + + return vars_r + + # This is a very inefficient approach to obtain all assignment variables (Solution 1) + def recursive_variables(self, my_expr, result): + if isinstance(my_expr, rinterface.LangSexpVector): + # check if there are enough data values. for an assignment there must be three namely VARIABLE SYMBOL VALUE. e.g. a = 3 + if len(my_expr) >= 3: + + # check for matches + c = str(my_expr[0]) + variable = my_expr[1] + + # Check if assignment. + if (c == "<-" or c == "="): + if isinstance(my_expr[1], rinterface.SexpSymbol): + result.add(str(variable)) + try: + for expr in my_expr: + result = self.recursive_variables(expr, result) + except Exception as e: + pass + return result + + def assignment_variables(self, text): + result = [] + + # Solution 1 (Native-Python): Write our own recursive function that in Python that parses the Abstract Syntax Tree of the R cell + # This is a very inefficient solution + # parsed_expr = base.parse(text=text, keep_source=True) + # parsed_expr_py = robjects.conversion.rpy2py(parsed_expr) + # result = list(self.recursive_variables(parsed_expr_py, set())) + + # Solution 2 (Native-R): Use built-in recursive cases of R (source https://adv-r.hadley.nz/expressions.html). This method is significantly faster. + output_r = robjects.r("""find_assign({ + %s + })""" % text) + result = re.findall(r'"([^"]*)"', str(output_r)) + + # Return the result + return result + + def __extract_cell_undefined(self, cell_source): + # Approach 1: get all vars and substract the ones with the approach as in + cell_names = self.__extract_cell_names(cell_source) + assignment_variables = self.assignment_variables(cell_source) + undef_vars = set(cell_names).difference(set(assignment_variables)) + + # Approach 2: (TODO) dynamic analysis approach. this is complex for R as functions + # as they are not scoped (which is the case in python). As such, we might have to include + # all the libraries to make sure that those functions work + + undef_vars = { + name: { + 'name': name, + 'type': None, + } + for name in undef_vars + } + + return undef_vars + + def extract_cell_params(self, cell_source): + params = {} + cell_unds = self.__extract_cell_undefined(cell_source) + param_unds = [und for und in cell_unds if und in self.global_params] + for u in param_unds: + if u not in params: + params[u] = self.global_params[u] + return params + + def extract_cell_conf_ref(self): + confs = {} + cell_unds = self.__extract_cell_undefined(self.cell_source) + conf_unds = [und for und in cell_unds if und in self.configurations] + for u in conf_unds: + if u not in confs: + confs[u] = self.configurations[u] + return confs + + def __resolve_configurations(self, configurations): + resolved_configurations = {} + for k, assignment in configurations.items(): + while 'conf_' in assignment.split('=')[1]: + for conf_name, replacing_assignment in configurations.items(): + if conf_name in assignment.split('=')[1]: + assignment = assignment.replace( + conf_name, + replacing_assignment.split('=')[1], + ) + resolved_configurations[k] = assignment + configurations.update(resolved_configurations) + return configurations + + +class StreamList: + + def __init__(self): + self.out = list() + + def write(self, text): + self.out.append(text) + + def reset(self): + self.out = list() + return self + + def __call__(self): + return self.out diff --git a/vreapis/services/extractor/rheaderextractor.py b/vreapis/services/extractor/rheaderextractor.py new file mode 100644 index 00000000..113cfb19 --- /dev/null +++ b/vreapis/services/extractor/rheaderextractor.py @@ -0,0 +1,259 @@ +import json +import logging +import os +import re +from typing import Literal, Union + +import jsonschema +import yaml + +from .extractor import Extractor + + +class RHeaderExtractor(Extractor): + """ Extracts cells using information defined by the user in its header + + Cells should contain a comment with a yaml block defining inputs, outputs, + params and confs. Eg: + + # My cell + # --- + # NaaVRE: + # cell: + # inputs: + # - my_input: + # type: String + # - my_other_input: + # type: Integer + # outputs: + # - my_output: + # type: List + # params: + # - param_something: + # type: String + # default_value: "my default value" + # confs: + # - conf_something_else: + # assignation: "conf_something_else = 'my other value'" + # ... + [cell code] + + The document is validated with the schema `cell_header.schema.json` + + """ + ins: Union[dict, None] + outs: Union[dict, None] + params: Union[dict, None] + confs: Union[list, None] + dependencies: Union[list, None] + + def __init__(self, notebook, cell_source): + self.re_yaml_doc_in_comment = re.compile( + (r"^(?:.*\n)*" + r"\s*#\s*---\s*\n" + r"((?:\s*#.*\n)+?)" + r"\s*#\s*\.\.\.\s*\n" + ), + re.MULTILINE) + self.schema = self._load_schema() + self.cell_header = self._extract_header(cell_source) + self._external_extract_cell_params = None + + super().__init__(notebook, cell_source) + + @staticmethod + def _load_schema(): + filename = os.path.join( + os.path.dirname(__file__), + 'cell_header.schema.json') + with open(filename) as f: + schema = json.load(f) + return schema + + def enabled(self): + return self.cell_header is not None + + def is_complete(self): + return ( + (self.ins is not None) + and (self.outs is not None) + and (self.params is not None) + and (self.confs is not None) + and (self.dependencies is not None) + ) + + def _extract_header(self, cell_source): + # get yaml document from cell comments + m = self.re_yaml_doc_in_comment.match(cell_source) + if not (m and m.groups()): + return None + yaml_doc = m.group(1) + # remove comment symbol + yaml_doc = '\n'.join([ + line.lstrip().lstrip('#') + for line in yaml_doc.splitlines() + ]) + # parse yaml + header = yaml.safe_load(yaml_doc) + # validate schema + try: + jsonschema.validate(header, self.schema) + except jsonschema.ValidationError as e: + logging.getLogger().debug(f"Cell header validation error: {e}") + raise e + return header + + def add_missing_values(self, extractor: Extractor): + """ Add values not specified in the header from another extractor + (e.g. PyExtractor or RExtractor) + """ + if self.ins is None: + self.ins = extractor.ins + if self.outs is None: + self.outs = extractor.outs + if self.params is None: + self.params = extractor.params + # We store a reference to extractor.extract_cell_params because + # self.extract_cell_params is called after self.add_missing_values + # in component_containerizer.handlers.ExtractorHandler.post() + self._external_extract_cell_params = extractor.extract_cell_params + if self.confs is None: + self.confs = extractor.confs + if self.dependencies is None: + self.dependencies = extractor.dependencies + + @staticmethod + def _parse_inputs_outputs_param_items( + item: Union[str, dict], + item_type: Literal['inputs', 'outputs', 'params'], + ) -> dict: + """ Parse inputs, outputs, or params items from the header + + They can have either format + - ElementVarName: 'my_name' + - ElementVarNameType {'my_name': 'my_type'} + - IOElementVarDict {'my_name': {'type': 'my_type'}} + or ParamElementVarDict {'my_name': {'type': 'my_type', + 'default_value': 'my_value'}} + + Returns + - if item_type is 'inputs' or 'outputs': + {'name': 'my_name', 'type': 'my_type'} + - if item_type is 'params': + {'name': 'my_name', 'type': 'my_type', 'value': 'my_value'} + """ + var_dict = {} + + # ElementVarName + if isinstance(item, str): + var_dict = { + 'name': item, + 'type': None, + 'value': None, + } + elif isinstance(item, dict): + if len(item.keys()) != 1: + # this should have been caught by the schema validation + raise ValueError(f"Unexpected item in {item_type}: {item}") + var_name = list(item.keys())[0] + var_props = item[var_name] + # ElementVarNameType + if isinstance(var_props, str): + var_dict = { + 'name': var_name, + 'type': var_props, + 'value': None, + } + # IOElementVarDict or ParamElementVarDict + elif isinstance(var_props, dict): + var_type = var_props.get('type') + default_value = var_props.get('default_value') + if var_type == 'List': + # Convert list to string representation + default_value = json.dumps(default_value) + var_dict = { + 'name': var_name, + 'type': var_type, + 'value': default_value, + } + + # Convert types + types_conversion = { + 'Integer': 'int', + 'Float': 'float', + 'String': 'str', + 'List': 'list', + None: None, + } + var_dict['type'] = types_conversion[var_dict['type']] + + # 'value' should only be kept for params + if item_type not in ['params']: + del var_dict['value'] + return var_dict + + def _infer_cell_inputs_outputs_params( + self, + header: Union[dict, None], + item_type: Literal['inputs', 'outputs', 'params'], + ) -> Union[dict, None]: + if header is None: + return None + items = header['NaaVRE']['cell'].get(item_type) + if items is None: + return None + items = [self._parse_inputs_outputs_param_items(it, item_type) + for it in items] + inputs_outputs_params = {it['name']: it for it in items} + return inputs_outputs_params + + def infer_cell_inputs(self): + return self._infer_cell_inputs_outputs_params( + self.cell_header, + 'inputs', + ) + + def infer_cell_outputs(self): + return self._infer_cell_inputs_outputs_params( + self.cell_header, + 'outputs', + ) + + def extract_cell_params(self, source): + if self._external_extract_cell_params is not None: + return self._external_extract_cell_params(source) + return self._infer_cell_inputs_outputs_params( + self._extract_header(source), + 'params', + ) + + def extract_cell_conf_ref(self): + if self.cell_header is None: + return None + items = self.cell_header['NaaVRE']['cell'].get('confs') + if items is None: + return None + for item in items: + for k, v in item.items(): + if 'assignation' in v: + assignation = v.get('assignation') + if '[' in assignation and ']' in assignation: + # Replace to R list format + assignation = assignation.replace('[', 'list(').replace(']', ')') + item[k]['assignation'] = assignation + cell_conf = {k: v['assignation'] for it in items for k, v in it.items()} + return cell_conf + + def infer_cell_dependencies(self, confs): + if self.cell_header is None: + return None + items = self.cell_header['NaaVRE']['cell'].get('dependencies') + if items is None: + return None + return [ + { + 'name': it.get('name'), + 'asname': it.get('asname', None), + 'module': it.get('module', ''), + } + for it in items] diff --git a/vreapis/templates/R_cell_template.jinja2 b/vreapis/templates/R_cell_template.jinja2 new file mode 100644 index 00000000..c99660b5 --- /dev/null +++ b/vreapis/templates/R_cell_template.jinja2 @@ -0,0 +1,71 @@ +setwd('/app') + +# retrieve input parameters + +library(optparse) +library(jsonlite) +{% for dep in deps%} +{{ dep }} +{% endfor %} + + +option_list = list( + +{% for in_var in cell.all_inputs|sort %} +{% if types[in_var] == list or types[in_var] == 'list' %} +{% set type = 'character' %} +{% elif types[in_var] == str or types[in_var] == 'str' %} +{% set type = 'character' %} +{% elif types[in_var] == int or types[in_var] == 'int' %} +{% set type = 'integer' %} +{% elif types[in_var] == float or types[in_var] == 'float' %} +{% set type = 'numeric' %} +{% elif types[in_var] == bool or types[in_var] == 'bool' %} +{% set type = 'logical' %} +{% else %} +{% set type = 'character' %} +{% endif %} +{% set option = 'make_option(c("--' + in_var+'"), action="store", default=NA, type="'+type+'", help="my description")' %} +{% if not loop.last %} +{% set option = option + ', '%} +{% endif %} +{{ option }} +{% endfor %} + +) + +# set input parameters accordingly +opt = parse_args(OptionParser(option_list=option_list)) + +{% for in_var in cell.inputs|sort %} +{% if types[in_var] == list or types[in_var] == 'list' %} +{{ in_var }} = fromJSON(opt${{ in_var }}) +{% elif types[in_var] == str or types[in_var] == 'str' %} +{{ in_var }} <- gsub('"', '', opt${{ in_var }}) +{% else %} +{{ in_var }} = opt${{ in_var }} +{% endif %} +{% endfor %} + +{% for param in cell.params|sort %} +{{ param }} = opt${{ param }} +{% endfor %} + + +{% for c in confs %} +{{ c }} +{% endfor %} + + +{{ cell.original_source }} + + + +{% if cell.outputs|length > 0 %} +# capturing outputs +{% for out in cell.outputs %} +file <- file(paste0('/tmp/{{out}}_', id, '.json')) +writeLines(toJSON({{out}}, auto_unbox=TRUE), file) +close(file) +{% endfor %} +{% endif %} \ No newline at end of file diff --git a/vreapis/templates/conda_env_template.jinja2 b/vreapis/templates/conda_env_template.jinja2 new file mode 100644 index 00000000..9c3fdcd8 --- /dev/null +++ b/vreapis/templates/conda_env_template.jinja2 @@ -0,0 +1,20 @@ +name: venv +channels: + - conda-forge + - r + - bioconda + - defaults +dependencies: + - pip + - python>=3.8 +{% for d in conda_deps %} + - {{ d }} +{% endfor %} +{% if pip_deps|length > 0 %} + - pip: + {% for d in pip_deps %} + - {{ d }} + {% endfor %} +{% endif %} + - papermill + - ipykernel \ No newline at end of file diff --git a/vreapis/templates/dockerfile_template_conda.jinja2 b/vreapis/templates/dockerfile_template_conda.jinja2 new file mode 100644 index 00000000..4c0c53f2 --- /dev/null +++ b/vreapis/templates/dockerfile_template_conda.jinja2 @@ -0,0 +1,18 @@ +FROM {{ base_image.build }} AS build +COPY --chown=$MAMBA_USER:$MAMBA_USER environment.yaml . +RUN micromamba install -y -n venv -f environment.yaml +ARG MAMBA_DOCKERFILE_ACTIVATE=1 +USER root +RUN conda-pack -p /opt/conda/envs/venv -o /tmp/env.tar && \ + mkdir /venv && cd /venv && tar xf /tmp/env.tar && \ + rm /tmp/env.tar +RUN /venv/bin/conda-unpack + +{% if task_name.startswith('visualize-')%} +FROM jupyter/base-notebook AS runtime +{% else %} +FROM {{ base_image.runtime }} AS runtime +{% endif %} +COPY --from=build /venv /venv +WORKDIR /app +COPY . . \ No newline at end of file diff --git a/vreapis/templates/py_cell_template.jinja2 b/vreapis/templates/py_cell_template.jinja2 new file mode 100644 index 00000000..0c6ee9c5 --- /dev/null +++ b/vreapis/templates/py_cell_template.jinja2 @@ -0,0 +1,62 @@ +{% if deps|length >0 %} +{% for d in deps %} +{{ d }} +{% endfor %} +{% endif %} + +import argparse +arg_parser = argparse.ArgumentParser() + +arg_parser.add_argument('--id', action='store', type=str, required=True, dest='id') + +{% for in_var in cell.inputs|sort %} + +{% if types[in_var] == list or types[in_var] == 'list' %} +arg_parser.add_argument('--{{ in_var }}', action='store', type=str, required=True, dest='{{ in_var }}') +{% else %} +arg_parser.add_argument('--{{ in_var }}', action='store', type={{ types[in_var] }}, required=True, dest='{{ in_var }}') +{% endif %} +{% endfor %} + +{% for param in cell.params|sort %} +arg_parser.add_argument('--{{ param }}', action='store', type={{ types[param] }}, required=True, dest='{{ param }}') +{% endfor %} + +args = arg_parser.parse_args() +print(args) + +id = args.id + +{% for in_var in cell.inputs|sort %} +{% if types[in_var] == list or types[in_var] == 'list' %} +import json +{{ in_var }} = json.loads(args.{{ in_var }}) +{% elif types[in_var] == str or types[in_var] == 'str' %} +{{ in_var }} = args.{{ in_var }}.replace('"','') +{% else %} +{{ in_var }} = args.{{ in_var }} +{% endif %} +{% endfor %} + +{% for param in cell.params|sort %} +{{ param }} = args.{{ param }} +{% endfor %} + +{% for c in confs %} +{% for c_name in c %} +{{ c_name }} = {{ c[c_name] }} +{% endfor %} + +{% endfor %} + +{{ cell.original_source }} + +{% if cell.outputs|length > 0 %} +import json +{% for out in cell.outputs %} +filename = "/tmp/{{out}}_" + id + ".json" +file_{{out}} = open(filename, "w") +file_{{out}}.write(json.dumps({{out}})) +file_{{out}}.close() +{% endfor %} +{% endif %} \ No newline at end of file diff --git a/vreapis/templates/vis_cell_template.jinja2 b/vreapis/templates/vis_cell_template.jinja2 new file mode 100644 index 00000000..26e5a3cf --- /dev/null +++ b/vreapis/templates/vis_cell_template.jinja2 @@ -0,0 +1,72 @@ +{% if deps|length >0 %} +{% for d in deps %} +{{ d }} +{% endfor %} +{% endif %} + +import argparse +import papermill as pm + +arg_parser = argparse.ArgumentParser() + +arg_parser.add_argument('--id', action='store', type=str, required=True, dest='id') + +{% for in_var in cell.inputs|sort %} + +{% if types[in_var] == list or types[in_var] == 'list' %} +arg_parser.add_argument('--{{ in_var }}', action='store', type=str, required=True, dest='{{ in_var }}') +{% else %} +arg_parser.add_argument('--{{ in_var }}', action='store', type={{ types[in_var] }}, required=True, dest='{{ in_var }}') +{% endif %} +{% endfor %} + +{% for param in cell.params|sort %} +arg_parser.add_argument('--{{ param }}', action='store', type={{ types[param] }}, required=True, dest='{{ param }}') +{% endfor %} + +args = arg_parser.parse_args() +print(args) + +id = args.id +parameters = {} + +{% for in_var in cell.inputs|sort %} +{% if types[in_var] == list or types[in_var] == 'list' %} +import json +{{ in_var }} = json.loads(args.{{ in_var }}) +{% elif types[in_var] == str or types[in_var] == 'str' %} +{{ in_var }} = args.{{ in_var }}.replace('"','') +{% else %} +{{ in_var }} = args.{{ in_var }} +{% endif %} +parameters['{{ in_var}}'] = {{ in_var }} +{% endfor %} + +{% for param in cell.params|sort %} +{{ param }} = args.{{ param }} +parameters['{{ param}}'] = {{ param }} +{% endfor %} + +{% for c in confs %} +{% for c_name in c %} +{{ c_name }} = {{ c[c_name] }} +parameters['{{ c_name }}'] = {{ c_name }} +{% endfor %} +{% endfor %} + +pm.execute_notebook( + '{{ cell.title }}.ipynb', + '{{ cell.title }}-output.ipynb', + prepare_only=True, + parameters=dict(parameters) +) + +{% if cell.outputs|length > 0 %} +import json +{% for out in cell.outputs %} +filename = "/tmp/{{out}}_" + id + ".json" +file_{{out}} = open(filename, "w") +file_{{out}}.write(json.dumps({{out}})) +file_{{out}}.close() +{% endfor %} +{% endif %} \ No newline at end of file diff --git a/vreapis/templates/workflow_template_v2.jinja2 b/vreapis/templates/workflow_template_v2.jinja2 new file mode 100644 index 00000000..e78c57d9 --- /dev/null +++ b/vreapis/templates/workflow_template_v2.jinja2 @@ -0,0 +1,199 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Workflow +metadata: + generateName: {{ workflow_name }}- + labels: + vlab_slug: {{ vlab_slug }} +spec: + entrypoint: {{ workflow_name }} + serviceAccountName: {{ workflow_service_account }} + volumeClaimTemplates: + - metadata: + name: workdir + spec: + accessModes: [ "ReadWriteMany" ] + resources: + requests: + storage: {{ workdir_storage_size }}Gi + arguments: + parameters: + {% for p, value in global_params.items() %} + - name: {{ p }} + value: {{ value }} + {% endfor %} + templates: + - name: {{ workflow_name }} + dag: + tasks: + {% for nid in nodes %} + {% set is_special_node = nodes[nid]['type'] == 'splitter' or nodes[nid]['type'] == 'merger' or nodes[nid]['type'] == 'visualizer' %} + {% set is_splitter = nodes[nid]['type'] == 'splitter' %} + {% set is_visualizer = nodes[nid]['type'] == 'visualizer' %} + {% set task_name = nodes[nid]['type'] + "-" + nid[:7] if is_special_node else cells[nid]['task_name']+ "-" + nid[:7] %} + {% set params = [] if is_special_node else cells[nid]['params'] %} + {# {% if 'visualize-' in task_name %} + - name: {{ task_name }}-server + dependencies: [{{ task_name }}] + template: {{ task_name }}-server-tmp + {% endif %} #} + + - name: {{ task_name }} + {% if deps_dag[nid]|length > 0 %} + dependencies: [{% for d in deps_dag[nid] %} {{ d['task_name'] }}{% if not loop.last %},{% endif %} {% endfor %}] + {% endif %} + template: {{ task_name }}-tmp + {% if deps_dag[nid]|length > 0 or params|length > 0 %} + arguments: + parameters: + {% for d in deps_dag[nid] %} + - {name: {{ d['port_id'] }}, value: {% if d['type'] == "splitter" %}{{ '"{{item}}"' }}{% else %}{{ '"{{tasks.' }}{{ d['task_name'] }}{{ '.outputs.parameters.' }}{{ d['port_id'] }}{{'}}"'}}{% endif %}} + {% endfor %} + {% for p in params %} + - {name: {{ p }}, value: {{ '"{{workflow.parameters.' }}{{ p }}{{ '}}"' }}} + {% endfor %} + {% for d in deps_dag[nid] %} + {% if d['type'] == "splitter" %} + withParam: {{ '"{{tasks.' }}{{ d['task_name'] }}{{ '.outputs.parameters.' }}{{ d['port_id'] }}{{'}}"'}} + {% endif %} + {% endfor %} + {% endif %} + {% endfor %} + {% for nid in nodes %} + {% set is_special_node = nodes[nid]['type'] == 'splitter' or nodes[nid]['type'] == 'merger' or nodes[nid]['type'] == 'visualizer' %} + {% set is_splitter = nodes[nid]['type'] == 'splitter' %} + {% set is_visualizer = nodes[nid]['type'] == 'visualizer' %} + {% set task_name = nodes[nid]['type'] + "-" + nid[:7] if is_special_node else cells[nid]['task_name'] + "-" + nid[:7] %} + {% set params = [] if is_special_node else cells[nid]['params'] %} + {% set ports = nodes[nid]['ports'] %} + {# {% if 'visualize-' in task_name %} + - name: {{ task_name }}-server-tmp + container: + image: busybox + imagePullPolicy: IfNotPresent + command: ["/bin/bash", "-c"] + args: + - echo "Starting server" + {% endif %} #} + - name: {{ task_name }}-tmp + {% if deps_dag[nid]|length > 0 %} + inputs: + parameters: + {% for d in deps_dag[nid] %} + - name: {{ d['port_id'] }} + {% endfor %} + {% for p in params %} + - name: {{ p }} + {% endfor %} + {% endif %} + outputs: + parameters: + {% for p in ports %} + {% if ports[p]['type'] == 'right' %} + - name: {{ ports[p]['id'] }} + valueFrom: + path: /tmp/{{ ports[p]['id'] }}.json + {% endif %} + {% endfor %} + {% if is_special_node and not is_visualizer %} + {% set special_dep = deps_dag[nid][0] %} + script: + image: python:alpine3.9 + imagePullPolicy: IfNotPresent + command: [python] + source: | + import json + {{ special_dep['port_id'] }} = {{ '{{inputs.parameters.' }}{{ special_dep['port_id'] }}{{ '}}' }} + {% for p in ports %} + {% if ports[p]['type'] == 'right' %} + {% if is_splitter %} + {# splitter logic #} + list_of_lists = [] + for elem in {{special_dep['port_id']}}: + list = [elem] + list_of_lists.append(list) + f_out = open("/tmp/{{ports[p]['id']}}.json", "w") + f_out.write(json.dumps(list_of_lists)) + f_out.close() + {% else %} + {# Merger logic #} + {{special_dep['port_id']}} = [item for items in {{special_dep['port_id']}} for item in json.loads(items)] + f_out = open("/tmp/{{ports[p]['id']}}.json", "w") + f_out.write(json.dumps({{special_dep['port_id']}})) + f_out.close() + {% endif %} + {% endif %} + {% endfor %} + {% elif is_visualizer %} + container: + image: "qcdis/geotiff_viewer:v0.2.6" + imagePullPolicy: IfNotPresent + command: ["/bin/sh", "-c"] + args: + - python /app/python_scripts/main.py + {% for d in deps_dag[nid] %} + {% set is_from_special = d['type'] == 'splitter' or d['type'] == 'merger' %} + {% set stdinname = d['og_port_id'] if is_from_special else d['port_id'] %} + {% set stdinname_base = stdinname.split('_')[0] %} + --{{ stdinname_base }} '{{ '{{inputs.parameters.' }}{{ d['port_id'] }}{{ '}}' }}' + {% endfor %} + {% for p in params %} + --{{ p }} '{{ '{{workflow.parameters.' }}{{ p }}{{ '}}' }}' + {% endfor %} + && npm start + metadata: + labels: + app: naavre-visualizer + {% else %} + container: + image: "{{ image_repo }}/{{ cells[nid]['task_name'] }}:{{ cells[nid]['image_version'] }}" + imagePullPolicy: Always + command: ["/bin/bash", "-c"] + args: + {% if 'visualize-' in cells[nid]['task_name']%} + - source /venv/bin/activate; ipython kernel install --user; python /app/task.py + {% for d in deps_dag[nid] %} + {% set is_from_special = d['type'] == 'splitter' or d['type'] == 'merger' %} + {% set stdinname = d['og_port_id'] if is_from_special else d['port_id'] %} + {% set stdinname_base = stdinname.split('_')[0] %} + --{{ stdinname_base }} '{{ '{{inputs.parameters.' }}{{ d['port_id'] }}{{ '}}' }}' + {% endfor %} + {% for p in params %} + --{{ p }} '{{ '{{workflow.parameters.' }}{{ p }}{{ '}}' }}' + {% endfor %} + --id {{' '}}{{cells[nid]['node_id']}}{{''}}{{";"}} + jupyter execute /app/task-output.ipynb --allow-errors; + jupyter nbconvert --no-input --execute /app/task-output.ipynb --to html; + jupyter notebook --port 5173 --NotebookApp.ip='0.0.0.0' --NotebookApp.allow_origin='*' --NotebookApp.base_url=/naavre-visualizer-notebook + metadata: + labels: + app: naavre-visualizer-notebook + {% elif cells[nid]['kernel'] == 'ipython' %} + - source /venv/bin/activate; python /app/task.py + {% for d in deps_dag[nid] %} + {% set is_from_special = d['type'] == 'splitter' or d['type'] == 'merger' %} + {% set stdinname = d['og_port_id'] if is_from_special else d['port_id'] %} + {% set stdinname_base = stdinname.split('_')[0] %} + --{{ stdinname_base }} '{{ '{{inputs.parameters.' }}{{ d['port_id'] }}{{ '}}' }}' + {% endfor %} + {% for p in params %} + --{{ p }} '{{ '{{workflow.parameters.' }}{{ p }}{{ '}}' }}' + {% endfor %} + --id {{' '}}{{cells[nid]['node_id']}}{{''}}{{";"}} + {% elif cells[nid]['kernel'] == 'IRkernel' %} + - source /venv/bin/activate; Rscript /app/task.R + {% for d in deps_dag[nid] %} + {% set is_from_special = d['type'] == 'splitter' or d['type'] == 'merger' %} + {% set stdinname = d['og_port_id'] if is_from_special else d['port_id'] %} + {% set stdinname_base = stdinname.split('_')[0] %} + --{{ stdinname_base }} '{{ '{{inputs.parameters.' }}{{ d['port_id'] }}{{ '}}' }}' + {% endfor %} + {% for p in params %} + --{{ p }} '{{ '{{workflow.parameters.' }}{{ p }}{{ '}}' }}' + {% endfor %} + --id {{' '}}{{cells[nid]['node_id']}}{{''}}{{";"}} + {% endif %} + volumeMounts: + - name: workdir + mountPath: /tmp/data + {% endif %} + {% endfor %} \ No newline at end of file diff --git a/vreapis/tests/emulated-frontend/containerizer.py b/vreapis/tests/emulated-frontend/containerizer.py new file mode 100644 index 00000000..068b1711 --- /dev/null +++ b/vreapis/tests/emulated-frontend/containerizer.py @@ -0,0 +1,57 @@ +import requests +import json +import os +import time +import argparse +import re + +arg_parser = argparse.ArgumentParser(description=''' +These tests emulate a frontend to test the endpoints of VRE PaaS API. +At this moment, folder `dat` contains all the test input data. Each file represents an entire request body. +Some files contain only 1 segment of extension `.json`. This indicates that the file is the only test case. +Some files contain 2 segments of extension (ex. `.1.json`). The substring between 2 dots is the ID of the test case (`1` in this example). +'''.strip()) +arg_parser.add_argument('in_def', nargs='+', help=''' +Default: Test the designated endpoint using all the existing request bodies. +'''.strip()) +arg_parser.add_argument('--in-id', action='append', nargs='+', help=''' +Specify test cases by ID: Test the designated endpoint using the request bodies with the specified IDs. +'''.strip()) +args = arg_parser.parse_args() + +script_path: str = os.path.dirname(os.path.realpath(__file__)) +API_ENDPOINT: str = f"{os.getenv('API_ENDPOINT')}/api/containerizer" +headers: dict = { + "Content-Type": "application/json", + "Authorization": f"Token {os.getenv('DJANGO_TOKEN')}", +} +session = requests.Session() + + +def test_post(endpoint: str, files: list[str]): + for file in files: + with open(f'{script_path}/dat/{file}') as f: + body: dict[str, any] = json.load(f) + # match endpoint: + # case 'addcell': + # body['node_id'] = str(hex(time.time_ns())[len('0x'):]) # use a unique node_id + response = session.post(f'{API_ENDPOINT}/{endpoint}', json.dumps(body), headers=headers, verify=False) + print(response.text) + + +for endpoint in args.in_def: + match endpoint: + case 'baseimagetags': + response = session.get(f'{API_ENDPOINT}/{endpoint}', headers=headers, verify=False) + print(response.text) + case _: + file_pattern = re.compile(fr'{endpoint}(\..+)?.json') + files: list[str] = os.listdir(f'{script_path}/dat') + request_body_files: list[str] = [file for file in files if file_pattern.match(file)] + test_post(endpoint, request_body_files) + +if args.in_id is not None: + for test_info in args.in_id: + endpoint: str = test_info[0] + files: list[str] = [f'{endpoint}.{id}.json' for id in test_info[1:]] + test_post(endpoint, files) diff --git a/vreapis/tests/emulated-frontend/dat/addcell.1.json b/vreapis/tests/emulated-frontend/dat/addcell.1.json new file mode 100644 index 00000000..9bfc8e78 --- /dev/null +++ b/vreapis/tests/emulated-frontend/dat/addcell.1.json @@ -0,0 +1,113 @@ +{ + "_state": {}, + "base_image": { + "build": "ghcr.io/qcdis/naavre/naavre-cell-build-python:v0.14", + "runtime": "ghcr.io/qcdis/naavre/naavre-cell-runtime-python:v0.14" + }, + "chart_obj": { + "hovered": {}, + "links": {}, + "nodes": { + "19096734787431788": { + "id": "19096734787431788", + "ports": { + "names": { + "id": "names", + "properties": { + "color": "#ac5397" + }, + "type": "right", + "position": { + "x": 225.5, + "y": 74.5 + } + } + }, + "position": { + "x": 35, + "y": 15 + }, + "properties": { + "inputs": [], + "og_node_id": "19096734787431788", + "outputs": [ + "names" + ], + "params": [], + "title": "input-names-user0", + "vars": [ + { + "color": "#ac5397", + "direction": "output", + "name": "names", + "type": "datatype" + } + ] + }, + "type": "input-output", + "size": { + "width": 250, + "height": 150 + } + } + }, + "offset": { + "x": 0, + "y": 0 + }, + "scale": 1, + "selected": {} + }, + "confs": {}, + "container_source": "", + "dependencies": [], + "global_conf": null, + "image_version": null, + "inputs": [], + "kernel": "ipython", + "node_id": "19096734787431788", + "notebook_dict": { + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "87f87efd-50e8-41cd-b4f6-2ba14d442cfb", + "metadata": {}, + "outputs": [], + "source": "# input names\nnames = [\"Alice\", \"Bob\"]" + } + ], + "metadata": { + "kernelspec": { + "display_name": "python3", + "language": "python3", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.9" + } + }, + "nbformat": 4, + "nbformat_minor": 5 + }, + "original_source": "# input names\nnames = [\"Alice\", \"Bob\"]", + "outputs": [ + "names" + ], + "param_values": {}, + "params": [], + "task_name": "input-names-user0", + "title": "input-names-user0", + "types": { + "names": "list" + } +} \ No newline at end of file diff --git a/vreapis/tests/emulated-frontend/dat/addcell.2.json b/vreapis/tests/emulated-frontend/dat/addcell.2.json new file mode 100644 index 00000000..8223d09b --- /dev/null +++ b/vreapis/tests/emulated-frontend/dat/addcell.2.json @@ -0,0 +1,113 @@ +{ + "_state": {}, + "base_image": { + "build": "ghcr.io/qcdis/naavre/naavre-cell-build-python:v0.14", + "runtime": "ghcr.io/qcdis/naavre/naavre-cell-runtime-python:v0.14" + }, + "chart_obj": { + "hovered": {}, + "links": {}, + "nodes": { + "603faa7": { + "id": "603faa7", + "ports": { + "names": { + "id": "names", + "properties": { + "color": "#ac5397" + }, + "type": "left", + "position": { + "x": 22.5, + "y": 74.5 + } + } + }, + "position": { + "x": 35, + "y": 15 + }, + "properties": { + "inputs": [ + "names" + ], + "og_node_id": "603faa7", + "outputs": [], + "params": [], + "title": "print-names-user0", + "vars": [ + { + "color": "#ac5397", + "direction": "input", + "name": "names", + "type": "datatype" + } + ] + }, + "type": "input-output", + "size": { + "width": 250, + "height": 150 + } + } + }, + "offset": { + "x": 0, + "y": 0 + }, + "scale": 1, + "selected": {} + }, + "confs": {}, + "container_source": "", + "dependencies": [], + "global_conf": null, + "image_version": null, + "inputs": [ + "names" + ], + "kernel": "ipython", + "node_id": "603faa7", + "notebook_dict": { + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "848c006a-bf2b-439d-8fec-ed16c5eae66d", + "metadata": {}, + "outputs": [], + "source": "# print names\nfor name in names:\n print(f\"Hello, {name}!\")" + } + ], + "metadata": { + "kernelspec": { + "display_name": "python3", + "language": "python3", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.9" + } + }, + "nbformat": 4, + "nbformat_minor": 5 + }, + "original_source": "# print names\nfor name in names:\n print(f\"Hello, {name}!\")", + "outputs": [], + "param_values": {}, + "params": [], + "task_name": "print-names-user0", + "title": "print-names-user0", + "types": { + "names": "list" + } +} \ No newline at end of file diff --git a/vreapis/tests/emulated-frontend/dat/baseimage.json b/vreapis/tests/emulated-frontend/dat/baseimage.json new file mode 100644 index 00000000..a5f93371 --- /dev/null +++ b/vreapis/tests/emulated-frontend/dat/baseimage.json @@ -0,0 +1 @@ +{"image": {"build": "ghcr.io/qcdis/naavre/naavre-cell-build-python:v0.14", "runtime": "ghcr.io/qcdis/naavre/naavre-cell-runtime-python:v0.14"}} \ No newline at end of file diff --git a/vreapis/tests/emulated-frontend/dat/extract.json b/vreapis/tests/emulated-frontend/dat/extract.json new file mode 100644 index 00000000..6350a958 --- /dev/null +++ b/vreapis/tests/emulated-frontend/dat/extract.json @@ -0,0 +1,50 @@ +{ + "save": false, + "kernel": "ipython", + "cell_index": 0, + "notebook": { + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.9" + } + }, + "nbformat_minor": 5, + "nbformat": 4, + "cells": [ + { + "cell_type": "code", + "source": "# input names\nnames = [\"Alice\", \"Bob\"]", + "metadata": { + "trusted": true + }, + "execution_count": null, + "outputs": [], + "id": "87f87efd-50e8-41cd-b4f6-2ba14d442cfb" + }, + { + "cell_type": "code", + "source": "# print names\nfor name in names:\n print(f\"Hello, {name}!\")", + "metadata": { + "trusted": true + }, + "execution_count": null, + "outputs": [], + "id": "848c006a-bf2b-439d-8fec-ed16c5eae66d" + } + ] + } +} diff --git a/vreapis/tests/extract-notebook.ps1 b/vreapis/tests/extract-notebook.ps1 new file mode 100644 index 00000000..60b12d3d --- /dev/null +++ b/vreapis/tests/extract-notebook.ps1 @@ -0,0 +1,17 @@ +# extract notebook contents from test json files +param( + [System.IO.FileInfo[]]$input_files, + [System.IO.FileInfo]$output_dir +) +New-Item -i Directory -f $output_dir +foreach ($input_file in $input_files) { + $root = ConvertFrom-Json (Get-Content -raw $input_file) -d 100 + $serialized_nb = ConvertTo-Json $root.data.notebook -d 100 + $serialized_nb > $output_dir/$($input_file.BaseName).ipynb + switch ($root.data.kernel) { + 'ipython' {} + 'IRkernel' { + $serialized_nb | jupytext --from ipynb --to Rmd > $output_dir/$($input_file.BaseName).Rmd + } + } +} diff --git a/vreapis/tests/perf/cvt-Rmd-to-ipynb.ps1 b/vreapis/tests/perf/cvt-Rmd-to-ipynb.ps1 new file mode 100644 index 00000000..badc5829 --- /dev/null +++ b/vreapis/tests/perf/cvt-Rmd-to-ipynb.ps1 @@ -0,0 +1,48 @@ +param( + [System.IO.FileInfo[]]$pathnames, # files to convert +# $kernelspec = @{ +# display_name = 'R [conda env:jupyterlab] *' +# language = 'R' +# name = 'conda-env-jupyterlab-r' +# } + $kernelspec = @{ + display_name = 'R' + language = 'R' + name = 'ir' + }, + $language_info = @{ + codemirror_mode = 'r' + file_extension = '.r' + mimetype = 'text/x-r-source' + name = 'R' + pygments_lexer = 'r' + version = '4.3.3' # keep this unchanged even if the actual R version mismatches to let JupyterLab not adjust automatically [or it will ask to save the file, which brings impacts on performance test results] + } +) + +foreach ($pathname in $pathnames) { + Write-Host -NoNewline "src: " + Write-Host -ForegroundColor Yellow $pathname + $dst = "$($pathname.DirectoryName)/$($pathname.BaseName).ipynb" + Get-Content -Raw $pathname | jupytext --from Rmd --to ipynb --opt split_at_heading=true -o - > $dst + $generated_ipynb = ConvertFrom-Json (Get-Content $dst -Raw) + $generated_ipynb.metadata | Add-Member -NotePropertyName kernelspec -NotePropertyValue $kernelspec + $generated_ipynb.metadata | Add-Member -NotePropertyName language_info -NotePropertyValue $language_info + $serialized_ipynb = ($generated_ipynb | ConvertTo-Json -Depth 100) -split '\n' + $serialized_ipynb | Set-Content $dst +# $reindented_ipynb_builder = New-Object System.Text.StringBuilder +# foreach ($row in $serialized_ipynb) { # adjust indentation to 1 space [JupyterLab's indentation preference] +# $matches = $null +# if ($row -match '^\s+') { +# [void]$reindented_ipynb_builder.Append(' ' * ($matches[0].Length / 2)) +# [void]$reindented_ipynb_builder.Append($row.Substring($matches[0].Length)) +# } else { +# [void]$reindented_ipynb_builder.Append($row) +# } +# [void]$reindented_ipynb_builder.Append("`n") +# } +# [void]$reindented_ipynb_builder.Remove($reindented_ipynb_builder.Length - 1, 1) # remove the last newline +# $reindented_ipynb_builder.ToString() | Set-Content $dst + Write-Host -NoNewline "dst: " + Write-Host -ForegroundColor Green $dst +} diff --git a/vreapis/tests/perf/dup-test-files.ps1 b/vreapis/tests/perf/dup-test-files.ps1 new file mode 100644 index 00000000..334fecee --- /dev/null +++ b/vreapis/tests/perf/dup-test-files.ps1 @@ -0,0 +1,31 @@ +# duplicate rmd test files for test repetitions +# At present, Cell Containerizer from NaaVRE automatically generates Docker image name for a cell using login username and its 1st line comment. To bring more convenience to test repetitions, this script directly duplicates designated test files with suffix added to the 1st line comment. +# Example: +# ./dup-test-files a.Rmd +# Results: +# 'a.Rmd' -> 'a.0.Rmd', 'a.1.Rmd', 'a.2.Rmd', ..., 'a.9.Rmd' +# Convert to ipynb: +# ls a*.Rmd | % { cat -raw $_ | jupytext --from Rmd --to ipynb --opt split_at_heading=true -o - > "$($_.BaseName).ipynb" } + +param( + [Parameter(Mandatory = $true)][System.IO.FileInfo[]]$pathnames, # files to dup + [int]$copies = 10, # number of copies + [string]$extension_prefix_prefix = '.', + [string]$extension_prefix_suffix = '', + [string]$magic_prefix = ' # C25B14E054D65738 [rpt ', # to conveniently recognise copies + [string]$magic_suffix = '] ' +) + +foreach ($pathname in $pathnames) { + $content = Get-Content -Raw $pathname + $re = '((^|[\r\n])```{r[^\r\n]+[\r\n]+(\s#\|[^\r\n]*[\r\n]+)*#[^\r\n]*)' # code chunk header to 1st line comment [chunk options w/ prefix #| skipped] [see https://yihui.org/knitr/options/?utm_source=chatgpt.com] + # $res = $content | Select-String -AllMatches $re + # $res.Matches | ForEach-Object { Write-Host $_.Value } + for ($i = 0; $i -lt $copies; ++$i) { + $new_content = $content -replace $re, "`$1$magic_prefix$i$magic_suffix" + $dst = "$($pathname.DirectoryName)/$($pathname.BaseName)$extension_prefix_prefix$i$extension_prefix_suffix$($pathname.Extension)" + Set-Content $dst $new_content + Write-Host -NoNewline "new replica: " + Write-Host -ForegroundColor Green $dst + } +} diff --git a/vreapis/tests/perf/rec-utils.ps1 b/vreapis/tests/perf/rec-utils.ps1 new file mode 100644 index 00000000..3add7d7c --- /dev/null +++ b/vreapis/tests/perf/rec-utils.ps1 @@ -0,0 +1,239 @@ +param( + [string]$browser_process_filter = '', # record resource usage for browser. use this param to specify re pattern to filter browser processes from ps entries + [switch]$JupyterLab_backend = $false, # record resource usage for JupyterLab backend + [switch]$RStudio_backend = $false, # record resource usage for RStudio backend + [alias('v')][string]$vreapi_process_filter = '', # record resource usage for vreapi [common backend]. use this param to specify re pattern to filter vreapi process from ps entries + [alias('d')][string]$database_process_filter = '', # record resource usage for db. use this param to specify re pattern to filter db processes from ps entries + [alias('vp')][string]$vreapi_pod_filter = '', # record resource usage for vreapi pod [common backend]. use this param to specify re pattern to filter vreapi pod from kubectl top pod entries + [alias('dp')][string]$database_pod_filter = '', # record resource usage for db pod [common backend]. use this param to specify re pattern to filter db pod from kubectl top pod entries + [string]$log_dir = 'log', # directory to store log files + [string]$log_filename_prefix = (Get-Date -Format 'yyyyMMdd-HHmmss'), # prefix for log filename. if not specified, current date and time will be used + [int]$interval = 1, # interval between 2 adjacent resource usage captures [seconds]. [pidstat only supports integer intervals] + [long]$number_of_records = 0, # 0 or negative means infinite records. positive means number of records to capture + [switch]$console = $false # print usage data to console +) + +# intro messages before recording +if ($console) { + Write-Host -NoNewline 'Recording CPU & mem usage at ' + Write-Host -NoNewline -ForegroundColor Green 'console' + Write-Host -NoNewline ' every ' + Write-Host -NoNewline -ForegroundColor Green $interval + Write-Host ' second(s) ...' +} else { + if ((Test-Path $log_dir) -eq $false) { & '/usr/bin/mkdir' -p $log_dir } + + $CPU_log_file = $log_dir + "/$log_filename_prefix.CPU.csv" + $mem_log_file = $log_dir + "/$log_filename_prefix.mem.csv" + $cooked_log_file = $log_dir + "/$log_filename_prefix.cooked.csv" + + Write-Host -NoNewline 'Recording CPU & mem usage at ' + Write-Host -NoNewline -ForegroundColor Green $CPU_log_file + Write-Host -NoNewline ' and ' + Write-Host -NoNewline -ForegroundColor Green $mem_log_file + Write-Host -NoNewline ' and ' + Write-Host -NoNewline -ForegroundColor Green $cooked_log_file + Write-Host -NoNewline ' every ' + Write-Host -NoNewline -ForegroundColor Green $interval + Write-Host ' second(s) ...' +} + +class Process_CPU_Entry { # pidstat [sudo apt install sysstat] used here. ps shows average CPU util only while top always show units like m, g, t, p for mem util thus less precise + # columns for CPU util: [time], UID, PID, %usr, %system, %guest, %wait, %CPU, CPU, Command + [DateTime]$time + [int]$UID = -1 + [int]$PID = -1 + [double]$pct_usr = 0 + [double]$pct_system = 0 + [double]$pct_guest = 0 + [double]$pct_wait = 0 + [double]$pct_CPU = 0 + [int]$CPU = 0 + [string]$Command = '' + Process_CPU_Entry([DateTime]$time, [int]$UID, [int]$_PID, [double]$pct_usr, [double]$pct_system, [double]$pct_guest, [double]$pct_wait, [double]$pct_CPU, [int]$CPU, [string]$Command) { + $this.time = $time + $this.UID = $UID + $this.PID = $_PID + $this.pct_usr = $pct_usr + $this.pct_system = $pct_system + $this.pct_guest = $pct_guest + $this.pct_wait = $pct_wait + $this.pct_CPU = $pct_CPU + $this.CPU = $CPU + $this.Command = $Command + } +} + +class Process_Memory_Entry { # pidstat + # columns for mem util: [time], UID, PID, minflt/s, majflt/s, VSZ, RSS, %MEM, Command + [DateTime]$time + [int]$UID = -1 + [int]$PID = -1 + [double]$minflt_p_s = 0 + [double]$majflt_p_s = 0 + [long]$VSZ = 0 # in KiB + [long]$RSS = 0 # in KiB + [double]$pct_MEM = 0 + [string]$Command = '' + Process_Memory_Entry([DateTime]$time, [int]$UID, [int]$_PID, [double]$minflt_p_s, [double]$majflt_p_s, [long]$VSZ, [long]$RSS, [double]$pct_MEM, [string]$Command) { + $this.time = $time + $this.UID = $UID + $this.PID = $_PID + $this.minflt_p_s = $minflt_p_s + $this.majflt_p_s = $majflt_p_s + $this.VSZ = $VSZ + $this.RSS = $RSS + $this.pct_MEM = $pct_MEM + $this.Command = $Command + } +} + +class Resource_Metric { # for what are sent by K8s metrics server, and cooked entries + [double]$CPU # % + [double]$mem # Mi + Resource_Metric() { + $this.CPU = 0 + $this.mem = 0 + } + Resource_Metric([double]$cpu, [double]$mem) { + $this.CPU = $cpu + $this.mem = $mem + } +} + +# add columns according to the designated process/pod filters +$cooked_entry = [PSCustomObject]@{ + "time" = $null +} +$JupyterLab_backend_filter = 'jupyter.?lab' +$RStudio_backend_filter = 'rstudio-server' +$CPU_headers = New-Object System.Collections.Generic.List[string] +$mem_headers = New-Object System.Collections.Generic.List[string] +$process_filters = New-Object System.Collections.Generic.List[string] +$pod_filters = New-Object System.Collections.Generic.List[string] +if ($JupyterLab_backend) { $process_filters.Add($JupyterLab_backend_filter) } +if ($RStudio_backend) { $process_filters.Add($RStudio_backend_filter) } +foreach ($s in @( + $browser_process_filter, + $vreapi_process_filter, + $database_process_filter +)) { + if ($s -ne '') { $process_filters.Add($s) } +} +foreach ($s in @( + $vreapi_pod_filter, + $database_pod_filter +)) { + if ($s -ne '') { $pod_filters.Add($s) } +} +foreach ($filter in $process_filters) { + $CPU_headers += "CPU:process:$filter" + $mem_headers += "mem:process:$filter" +} +foreach ($filter in $pod_filters) { + $CPU_headers += "CPU:pod:$filter" + $mem_headers += "mem:pod:$filter" +} +$CPU_headers + $mem_headers | ForEach-Object { $cooked_entry | Add-Member -MemberType NoteProperty -Name $_ -Value $null } + +# main body +$time_format = 'yyyy-MM-dd HH:mm:ss.fff' +if ($env:LC_NUMERIC -eq $null) { + $culture = [System.Globalization.CultureInfo]::CurrentCulture.Name +} else { + $BCP_47_culture_name_for_pidstat = $env:LC_NUMERIC.Substring(0, $env:LC_NUMERIC.IndexOf('.')) # e.g. en-US, nl-NL + $culture = New-Object System.Globalization.CultureInfo($BCP_47_culture_name_for_pidstat) +} +$loop_body = { + # current time point + $time = Get-Date -Format $time_format + $cooked_entry.time = $time + + # convert text-based results from pidstat to .NET objects + $process_info = (pidstat -rul -p ALL $interval 1) -join "`n" -split "\n\n" | Select-Object -Skip 1 + $util_rows = New-Object System.Collections.Generic.List[System.Collections.Generic.List[string[]]] + for ($i = 0; $i -lt 2; ++$i) { + $raw_rows = $process_info[$i] -split '\n' | Select-Object -Skip 1 + $split_rows = New-Object System.Collections.Generic.List[string[]] + foreach ($raw_row in $raw_rows) { + $columns = $raw_row -split '\s+' + $split_rows.Add($columns) + } + $util_rows.Add($split_rows) + } + $CPU_util_rows = $util_rows[0] + $mem_util_rows = $util_rows[1] + $CPU_entries = New-Object System.Collections.Generic.List[Process_CPU_Entry] + $CPU_entries_to_export = New-Object System.Collections.Generic.List[Process_CPU_Entry] + $mem_entries = New-Object System.Collections.Generic.List[Process_Memory_Entry] + $mem_entries_to_export = New-Object System.Collections.Generic.List[Process_Memory_Entry] + foreach ($seg in $CPU_util_rows) { + $e = [Process_CPU_Entry]::new( + [DateTime]::ParseExact($time, $time_format, $null), + [int]($seg[1]), + [int]($seg[2]), + [double]::Parse($seg[3], $culture), + [double]::Parse($seg[4], $culture), + [double]::Parse($seg[5], $culture), + [double]::Parse($seg[6], $culture), + [double]::Parse($seg[7], $culture), + [int]($seg[8]), + $seg[9..($seg.Length - 1)] -join ' ' + ) + $CPU_entries.Add($e) + } + foreach ($seg in $mem_util_rows) { + $e = [Process_Memory_Entry]::new( + [DateTime]::ParseExact($time, $time_format, $null), + [int]($seg[1]), + [int]($seg[2]), + [double]::Parse($seg[3], $culture), + [double]::Parse($seg[4], $culture), + [long]($seg[5]), + [long]($seg[6]), + [double]::Parse($seg[7], $culture), + $seg[8..($seg.Length - 1)] -join ' ' + ) + $mem_entries.Add($e) + } + + # filter entries + foreach ($filter in $process_filters) { + $new_CPU_entries = $CPU_entries | Where-Object { $_.Command -match $filter } + if ($new_CPU_entries.Count -gt 0) { $CPU_entries_to_export.AddRange(($new_CPU_entries -as [System.Collections.Generic.List[Process_CPU_Entry]])) } + $new_mem_entries = $mem_entries | Where-Object { $_.Command -match $filter } + if ($new_mem_entries.Count -gt 0) { $mem_entries_to_export.AddRange(($new_mem_entries -as [System.Collections.Generic.List[Process_Memory_Entry]])) } + $metric = [Resource_Metric]::new( + ($new_CPU_entries | Measure-Object -Property pct_CPU -Sum).Sum, + ($new_mem_entries | Measure-Object -Property RSS -Sum).Sum / 1024.0 # convert KiB to MiB + ) + $cooked_entry."CPU:process:$filter" = $metric.CPU + $cooked_entry."mem:process:$filter" = $metric.mem + } + foreach ($filter in $pod_filters) { + $pod_metric = kubectl top pod $filter --no-headers | ForEach-Object { + $col = $_ -split '\s+' + [Resource_Metric]::new([double]($col[1].substring(0, $col[1].Length - 'm'.Length)) / 10.0, [double]($col[2]).substring(0, $col[2].Length - 'Mi'.Length)) + } + $cooked_entry."CPU:pod:$filter" = $pod_metric.CPU + $cooked_entry."mem:pod:$filter" = $pod_metric.mem + } + + # output + if ($console) { + $cooked_entry | Format-Table | Write-Output + } else { + $CPU_entries_to_export | Export-Csv -Append $CPU_log_file + $mem_entries_to_export | Export-Csv -Append $mem_log_file + $cooked_entry | Export-Csv -Append $cooked_log_file + } + + Start-Sleep($interval) +} + +# main +if ($number_of_records -le 0) { + while ($true) { & $loop_body } +} else { + for ($i = 0; $i -lt $number_of_records; ++$i) { & $loop_body } +} diff --git a/vreapis/tests/perf/record-usage.[obsolete].ps1 b/vreapis/tests/perf/record-usage.[obsolete].ps1 new file mode 100644 index 00000000..d28cfceb --- /dev/null +++ b/vreapis/tests/perf/record-usage.[obsolete].ps1 @@ -0,0 +1,227 @@ +param( + [string]$browser_process_filter = '', # record resource usage for browser. use this param to specify re pattern to filter browser processes from ps entries + [switch]$JupyterLab_backend = $false, # record resource usage for JupyterLab backend + [switch]$RStudio_backend = $false, # record resource usage for RStudio backend + [alias('v')][string]$vreapi_process_filter = '', # record resource usage for vreapi [common backend]. use this param to specify re pattern to filter vreapi process from ps entries + [alias('d')][string]$database_process_filter = '', # record resource usage for db. use this param to specify re pattern to filter db processes from ps entries + [alias('vp')][string]$vreapi_pod_filter = '', # record resource usage for vreapi pod [common backend]. use this param to specify re pattern to filter vreapi pod from kubectl top pod entries + [alias('dp')][string]$database_pod_filter = '', # record resource usage for db pod [common backend]. use this param to specify re pattern to filter db pod from kubectl top pod entries + [string]$log_dir = '.log', # directory to store log files + [Double]$interval = 1, # interval between 2 adjacent resource usage captures [seconds] + [long]$number_of_records = 0, # 0 or negative means infinite records. positive means number of records to capture + [switch]$console = $false # print usage data to console +) + +if ($console) { + Write-Host -NoNewline 'Recording CPU & mem usage at ' + Write-Host -NoNewline -ForegroundColor Green 'console' + Write-Host -NoNewline ' every ' + Write-Host -NoNewline -ForegroundColor Green $interval + Write-Host ' second(s) ...' +} else { + if ((Test-Path $log_dir) -eq $false) { & '/usr/bin/mkdir' -p $log_dir } + + $date = Get-Date -Format 'yyyyMMdd-HHmmss' + $raw_log_file = $log_dir + "/$date.raw.csv" + $cooked_log_file = $log_dir + "/$date.cooked.csv" + + Write-Host -NoNewline 'Recording CPU & mem usage at ' + Write-Host -NoNewline -ForegroundColor Green $raw_log_file + Write-Host -NoNewline ' and ' + Write-Host -NoNewline -ForegroundColor Green $cooked_log_file + Write-Host -NoNewline ' every ' + Write-Host -NoNewline -ForegroundColor Green $interval + Write-Host ' second(s) ...' +} + +class Simplified_ps_Entry { + + static [string] $header_row = 'user,pid,%cpu,rss,command' # complete header row: 'USER,PID,%CPU,%MEM,VSZ,RSS,TTY,STAT,START,TIME,COMMAND' + static [string[]] $col_name = [Simplified_ps_Entry]::header_row -replace '%', '' -split ',' + static $col_name_index = @{} + static [void] Init() { + for ([int]$i = 0; $i -lt [Simplified_ps_Entry]::col_name.Count; ++$i) { + [Simplified_ps_Entry]::col_name_index[[Simplified_ps_Entry]::col_name[$i]] = $i + } + } + [string]$time = '' + [string]$USER = '' + [int]$PID = -1 + [double]$CPU = 0.0 + [int]$RSS = 0 # KB + [string]$COMMAND = '' + [void]constructor([string]$user, [int]$process_ID, [double]$cpu, [int]$rss, [string]$command) { + $this.USER = $user + $this.PID = $process_ID + $this.CPU = $cpu + $this.RSS = $rss + $this.COMMAND = $command + } + Simplified_ps_Entry([string]$time, [string]$user, [int]$process_ID, [double]$cpu, [int]$rss, [string]$command) { + $this.time = $time + $this.constructor($user, $process_ID, $cpu, $rss, $command) + } + [void]constructor([string]$raw_entry) { + $cooked_entry = $raw_entry -split '\s+' + $this.USER = $cooked_entry[[Simplified_ps_Entry]::col_name_index['user']] + $this.PID = [int]$cooked_entry[[Simplified_ps_Entry]::col_name_index['pid']] + $this.CPU = [double]$cooked_entry[[Simplified_ps_Entry]::col_name_index['cpu']] + $this.RSS = [int]$cooked_entry[[Simplified_ps_Entry]::col_name_index['rss']] + $this.COMMAND = $cooked_entry[[Simplified_ps_Entry]::col_name_index['command']..($cooked_entry.count - 1)] + } + Simplified_ps_Entry([string]$time, [string]$raw_entry) { + $this.time = $time + $this.constructor($raw_entry) + } +} +[Simplified_ps_Entry]::Init() + +class Resource_Metric { + [double]$CPU # % + [double]$mem # Mi + Resource_Metric() { + $this.CPU = 0 + $this.mem = 0 + } + Resource_Metric([double]$cpu, [double]$mem) { + $this.CPU = $cpu + $this.mem = $mem + } +} + +if ((-not $console) -and (-not (Test-Path $raw_log_file))) { (@('time') + [Simplified_ps_Entry]::col_name) -join ',' | Out-File $raw_log_file } # add csv headers first + +$compound_resource_metric = [PSCustomObject]@{ + "time" = $null +} +$cpu_headers = $mem_headers = @() +if ($browser_process_filter -ne '') { + $cpu_headers += "CPU:browser:$browser_process_filter" + $mem_headers += "mem:browser:$browser_process_filter" +} +if ($JupyterLab_backend) { + $cpu_headers += "CPU:JupyterLab backend" + $mem_headers += "mem:JupyterLab backend" +} +if ($RStudio_backend) { + $cpu_headers += "CPU:RStudio backend" + $mem_headers += "mem:RStudio backend" +} +if ($vreapi_process_filter -ne '') { + $cpu_headers += "CPU:vreapi:$vreapi_process_filter" + $mem_headers += "mem:vreapi:$vreapi_process_filter" +} +if ($database_process_filter -ne '') { + $cpu_headers += "CPU:database:$database_process_filter" + $mem_headers += "mem:database:$database_process_filter" +} +if ($vreapi_pod_filter -ne '') { + $vreapi_pod_filter = ((kubectl get pod | Select-String $vreapi_pod_filter).Line -split '\s+')[0] + $cpu_headers += "CPU:pod:$vreapi_pod_filter" + $mem_headers += "mem:pod:$vreapi_pod_filter" +} +if ($database_pod_filter -ne '') { + $database_pod_filter = ((kubectl get pod | Select-String $database_pod_filter).Line -split '\s+')[0] + $cpu_headers += "CPU:pod:$database_pod_filter" + $mem_headers += "mem:pod:$database_pod_filter" +} +$cpu_headers + $mem_headers | ForEach-Object { $compound_resource_metric | Add-Member -MemberType NoteProperty -Name $_ -Value $null } +if ((-not $console) -and (-not (Test-Path $cooked_log_file))) { $compound_resource_metric | ConvertTo-Csv | Select-Object -First 1 | Out-File $cooked_log_file } # add csv headers first + +$ps_pathname = '/usr/bin/ps' +$time_format = 'yyyy-MM-dd HH:mm:ss.fff' + +$loop_body = { + $time = Get-Date -Format $time_format + $compound_resource_metric.time = $time + $new_raw_ps_entries = New-Object System.Collections.Generic.List[Simplified_ps_Entry] + if ($browser_process_filter -ne '') { + $browser_processes = & $ps_pathname 'exo' $([Simplified_ps_Entry]::header_row) | Select-String -a $browser_process_filter + $browser_metric = [Resource_Metric]::new() + foreach ($process in $browser_processes) { + $m = [Simplified_ps_Entry]::new($time, $process) + $new_raw_ps_entries.Add($m) + $browser_metric.CPU += $m.CPU + $browser_metric.mem += $m.RSS * 1000 / 1024.0 / 1024 + } + $compound_resource_metric."CPU:browser:$browser_process_filter" = $browser_metric.CPU + $compound_resource_metric."mem:browser:$browser_process_filter" = $browser_metric.mem + } + if ($JupyterLab_backend) { + $JupyterLab_backend_processes = & $ps_pathname 'exo' $([Simplified_ps_Entry]::header_row) | Select-String -a 'jupyter.?lab' + $JupyterLab_backend_metric = [Resource_Metric]::new() + foreach ($process in $JupyterLab_backend_processes) { + $m = [Simplified_ps_Entry]::new($time, $process) + $new_raw_ps_entries.Add($m) + $JupyterLab_backend_metric.CPU += [double]$m.CPU + $JupyterLab_backend_metric.mem += [double]$m.RSS * 1000 / 1024.0 / 1024 + } + $compound_resource_metric."CPU:JupyterLab backend" = $JupyterLab_backend_metric.CPU + $compound_resource_metric."mem:JupyterLab backend" = $JupyterLab_backend_metric.mem + } + if ($RStudio_backend) { + $RStudio_backend_processes = & $ps_pathname 'axo' $([Simplified_ps_Entry]::header_row) | Select-String -a 'rstudio-server' # here 'axo' not 'exo' since all process got by the latter are contained in those got by the former + $RStudio_backend_metric = [Resource_Metric]::new() + foreach ($process in $RStudio_backend_processes) { + $m = [Simplified_ps_Entry]::new($time, $process) + $new_raw_ps_entries.Add($m) + $RStudio_backend_metric.CPU += [double]$m.CPU + $RStudio_backend_metric.mem += [double]$m.RSS * 1000 / 1024.0 / 1024 + } + $compound_resource_metric."CPU:RStudio backend" = $RStudio_backend_metric.CPU + $compound_resource_metric."mem:RStudio backend" = $RStudio_backend_metric.mem + } + if ($vreapi_process_filter -ne '') { + $vreapi_processes = & $ps_pathname 'exo' $([Simplified_ps_Entry]::header_row) | Select-String -a $vreapi_process_filter + $vreapi_metric = [Resource_Metric]::new() + foreach ($process in $vreapi_processes) { + $m = [Simplified_ps_Entry]::new($time, $process) + $new_raw_ps_entries.Add($m) + $vreapi_metric.CPU += $m.CPU + $vreapi_metric.mem += $m.RSS * 1000 / 1024.0 / 1024 + } + $compound_resource_metric."CPU:vreapi:$vreapi_process_filter" = $vreapi_metric.CPU + $compound_resource_metric."mem:vreapi:$vreapi_process_filter" = $vreapi_metric.mem + } + if ($database_process_filter -ne '') { + $database_processes = & $ps_pathname 'axo' $([Simplified_ps_Entry]::header_row) | Select-String -a $database_process_filter # axo works fine to filter postgres processes. not sure when it comes to other db systems + $database_metric = [Resource_Metric]::new() + foreach ($process in $database_processes) { + $m = [Simplified_ps_Entry]::new($time, $process) + $new_raw_ps_entries.Add($m) + $database_metric.CPU += $m.CPU + $database_metric.mem += $m.RSS * 1000 / 1024.0 / 1024 + } + $compound_resource_metric."CPU:database:$database_process_filter" = $database_metric.CPU + $compound_resource_metric."mem:database:$database_process_filter" = $database_metric.mem + } + if ($vreapi_pod_filter -ne '') { + $pod_metric = kubectl top pod $vreapi_pod_filter --no-headers | ForEach-Object { + $col = $_ -split '\s+' + [Resource_Metric]::new([double]($col[1].substring(0, $col[1].Length - 'm'.Length)) / 10.0, [double]($col[2]).substring(0, $col[2].Length - 'Mi'.Length)) + } + $compound_resource_metric."CPU:pod:$vreapi_pod_filter" = $pod_metric.CPU + $compound_resource_metric."mem:pod:$vreapi_pod_filter" = $pod_metric.mem + } + if ($database_pod_filter -ne '') { + $pod_metric = kubectl top pod $database_pod_filter --no-headers | ForEach-Object { + $col = $_ -split '\s+' + [Resource_Metric]::new([double]($col[1].substring(0, $col[1].Length - 'm'.Length)) / 10.0, [double]($col[2]).substring(0, $col[2].Length - 'Mi'.Length)) + } + $compound_resource_metric."CPU:pod:$database_pod_filter" = $pod_metric.CPU + $compound_resource_metric."mem:pod:$database_pod_filter" = $pod_metric.mem + } + if ($console) { + $compound_resource_metric | Format-Table | Write-Output + } else { + $new_raw_ps_entries | Export-Csv -Append $raw_log_file + $compound_resource_metric | Export-Csv -Append $cooked_log_file + } + Start-Sleep($interval) +} + +if ($number_of_records -le 0) { + while ($true) { & $loop_body } +} else { + for ($i = 0; $i -lt $number_of_records; ++$i) { & $loop_body } +} diff --git a/vreapis/tests/resources/cells/add1-dev-user-name-at-domain-com.json b/vreapis/tests/resources/cells/add1-dev-user-name-at-domain-com.json new file mode 100644 index 00000000..adf237b8 --- /dev/null +++ b/vreapis/tests/resources/cells/add1-dev-user-name-at-domain-com.json @@ -0,0 +1,92 @@ +{ + "all_inputs": [ + "count" + ], + "base_image": {"build":"ghcr.io/qcdis/naavre/naavre-cell-build-python:latest","runtime":"ghcr.io/qcdis/naavre/naavre-cell-runtime-python:latest"}, + "chart_obj": { + "hovered": {}, + "links": {}, + "nodes": { + "49d5dda": { + "id": "49d5dda", + "ports": { + "a": { + "id": "a", + "properties": { + "color": "#77862d" + }, + "type": "right" + }, + "count": { + "id": "count", + "properties": { + "color": "#783d3a" + }, + "type": "left" + } + }, + "position": { + "x": 35, + "y": 15 + }, + "properties": { + "inputs": [ + "count" + ], + "og_node_id": "49d5dda", + "outputs": [ + "a" + ], + "params": [], + "title": "Add1-dev-user-name-at-domain-com", + "vars": [ + { + "color": "#783d3a", + "direction": "input", + "name": "count", + "type": "datatype" + }, + { + "color": "#77862d", + "direction": "output", + "name": "a", + "type": "datatype" + } + ] + }, + "type": "input-output" + } + }, + "offset": { + "x": 0, + "y": 0 + }, + "scale": 1, + "selected": {} + }, + "confs": {}, + "container_source": "", + "dependencies": [], + "inputs": [ + "count" + ], + "kernel": "ipython", + "node_id": "49d5dda", + "original_source": "\na = count + 1", + "outputs": [ + "a" + ], + "params": [], + "task_name": "add1-dev-user-name-at-domain-com", + "title": "Add1-dev-user-name-at-domain-com", + "types": { + "a": "int", + "count": "int" + }, + "example_inputs" : [ + "--id", + "0", + "--count", + "1" + ] +} diff --git a/vreapis/tests/resources/cells/add1-r-dev-user-name-at-domain-com.json b/vreapis/tests/resources/cells/add1-r-dev-user-name-at-domain-com.json new file mode 100644 index 00000000..42f74cac --- /dev/null +++ b/vreapis/tests/resources/cells/add1-r-dev-user-name-at-domain-com.json @@ -0,0 +1,74 @@ +{ + "all_inputs": [ + "count" + ], + "base_image": {"build":"ghcr.io/qcdis/naavre/naavre-cell-build-r:latest","runtime":"ghcr.io/qcdis/naavre/naavre-cell-runtime-r:latest"}, + "chart_obj": { + "hovered": {}, + "links": {}, + "nodes": { + "d9acfa2": { + "id": "d9acfa2", + "ports": { + "count": { + "id": "count", + "properties": { + "color": "#783d3a" + }, + "type": "left" + } + }, + "position": { + "x": 35, + "y": 15 + }, + "properties": { + "inputs": [ + "count" + ], + "og_node_id": "d9acfa2", + "outputs": [], + "params": [], + "title": "Add1 R-dev-user-name-at-domain-com", + "vars": [ + { + "color": "#783d3a", + "direction": "input", + "name": "count", + "type": "datatype" + } + ] + }, + "type": "input-output" + } + }, + "offset": { + "x": 0, + "y": 0 + }, + "scale": 1, + "selected": {} + }, + "confs": {}, + "container_source": "", + "dependencies": [], + "inputs": [ + "count" + ], + "kernel": "IRkernel", + "node_id": "d9acfa2", + "original_source": "\na = count + 1", + "outputs": [], + "params": [], + "task_name": "add1-r-dev-user-name-at-domain-com", + "title": "Add1 R-dev-user-name-at-domain-com", + "types": { + "count": "int" + }, + "example_inputs" : [ + "--id", + "0", + "--count", + "1" + ] +} diff --git a/vreapis/tests/resources/cells/anti-pattern-dev-user-name-at-domain-com.json b/vreapis/tests/resources/cells/anti-pattern-dev-user-name-at-domain-com.json new file mode 100644 index 00000000..7d005f14 --- /dev/null +++ b/vreapis/tests/resources/cells/anti-pattern-dev-user-name-at-domain-com.json @@ -0,0 +1,104 @@ +{ + "all_inputs": [ + "count", + "a" + ], + "base_image": {"build":"ghcr.io/qcdis/naavre/naavre-cell-build-python:latest","runtime":"ghcr.io/qcdis/naavre/naavre-cell-runtime-python:latest"}, + "chart_obj": { + "hovered": {}, + "links": {}, + "nodes": { + "44dec46": { + "id": "44dec46", + "ports": { + "a": { + "id": "a", + "properties": { + "color": "#77862d" + }, + "type": "left" + }, + "count": { + "id": "count", + "properties": { + "color": "#783d3a" + }, + "type": "left" + }, + "msg": { + "id": "msg", + "properties": { + "color": "#7bd279" + }, + "type": "right" + } + }, + "position": { + "x": 35, + "y": 15 + }, + "properties": { + "inputs": [ + "a", + "count" + ], + "og_node_id": "44dec46", + "outputs": [ + "msg" + ], + "params": [], + "title": "Anti-pattern-dev-user-name-at-domain-com", + "vars": [ + { + "color": "#77862d", + "direction": "input", + "name": "a", + "type": "datatype" + }, + { + "color": "#783d3a", + "direction": "input", + "name": "count", + "type": "datatype" + }, + { + "color": "#7bd279", + "direction": "output", + "name": "msg", + "type": "datatype" + } + ] + }, + "type": "input-output" + } + }, + "offset": { + "x": 0, + "y": 0 + }, + "scale": 1, + "selected": {} + }, + "confs": {}, + "container_source": "", + "dependencies": [], + "inputs": [ + "count", + "a" + ], + "kernel": "ipython", + "node_id": "44dec46", + "original_source": "some_list = range(count, a+1)\n\nmsg = '1'", + "outputs": [ + "msg" + ], + "params": [], + "task_name": "anti-pattern-dev-user-name-at-domain-com", + "title": "Anti-pattern-dev-user-name-at-domain-com", + "types": { + "a": "int", + "count": "int", + "msg": "str" + }, + "skip_exec": true +} diff --git a/vreapis/tests/resources/cells/consume-climwin-dev-user-name-at-domain-com.json b/vreapis/tests/resources/cells/consume-climwin-dev-user-name-at-domain-com.json new file mode 100644 index 00000000..21c7d451 --- /dev/null +++ b/vreapis/tests/resources/cells/consume-climwin-dev-user-name-at-domain-com.json @@ -0,0 +1,133 @@ +{ + "all_inputs": [ + "rolling_mean_temp_str", + "temperature_data_str" + ], + "base_image": {"build":"ghcr.io/qcdis/naavre/naavre-cell-build-lter-life-veluwe:latest","runtime":"ghcr.io/qcdis/naavre/naavre-cell-runtime-lter-life-veluwe:latest"}, + "chart_obj": { + "hovered": {}, + "links": {}, + "nodes": { + "dd373f8": { + "id": "dd373f8", + "ports": { + "rolling_mean_temp_str": { + "id": "rolling_mean_temp_str", + "properties": { + "color": "#c5a987" + }, + "type": "left" + }, + "temperature_data_str": { + "id": "temperature_data_str", + "properties": { + "color": "#aca953" + }, + "type": "left" + } + }, + "position": { + "x": 35, + "y": 15 + }, + "properties": { + "inputs": [ + "rolling_mean_temp_str", + "temperature_data_str" + ], + "og_node_id": "dd373f8", + "outputs": [], + "params": [], + "title": "consume climwin-dev-user-name-at-domain-com", + "vars": [ + { + "color": "#c5a987", + "direction": "input", + "name": "rolling_mean_temp_str", + "type": "datatype" + }, + { + "color": "#aca953", + "direction": "input", + "name": "temperature_data_str", + "type": "datatype" + } + ] + }, + "type": "input-output" + } + }, + "offset": { + "x": 0, + "y": 0 + }, + "scale": 1, + "selected": {} + }, + "confs": {}, + "container_source": "", + "dependencies": [ + { + "asname": "", + "module": "", + "name": "climwin" + }, + { + "asname": "", + "module": "", + "name": "zoo" + } + ], + "inputs": [ + "rolling_mean_temp_str", + "temperature_data_str" + ], + "kernel": "IRkernel", + "node_id": "dd373f8", + "notebook_dict": { + "cells": [ + { + "cell_type": "code", + "execution_count": 2, + "id": "84eee104-ceac-4ccb-8a66-d75b6715c4f8", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": "Original Temperature Data:\n 12.1976217672389, 13.8491125525836, 22.7935415707456, 15.3525419571229, 15.6464386758047, 23.5753249344164, 17.304581029946, 8.67469382696733, 11.5657357405324, 12.7716901495002, 21.1204089871973, 16.7990691352868, 17.0038572529703, 15.5534135797256, 12.2207943262296, 23.9345656840154, 17.4892523911462, 5.16691421685181, 18.5067795078184, 12.6360429613603, 9.66088147006577, 13.9101254267085, 9.8699777584638, 11.3555438535443, 11.8748036607537, 6.56653344628793, 19.1889352224726, 15.7668655891826, 9.30931531494026, 21.2690746053496, 17.1323211073841, 13.5246425850386, 19.4756283052251, 19.3906674376652, 19.1079054081874, 18.4432012705005, 17.7695882676879, 14.6904414471164, 13.4701866813004, 13.0976449949381, 11.5264651053974, 13.960413609902, 8.67301824215868, 25.8447798266926, 21.039809991525, 9.38445708398325, 12.9855758235046, 12.6667232318839, 18.8998255916816, 14.5831546676409, 16.2665925699738, 14.8572662232565, 14.7856477135434, 21.8430114200723, 13.8711450717037, 22.5823530221477, 7.25623597884889, 17.9230687481803, 15.6192712192231, 16.0797078437199, 16.8981974137994, 12.4883827344535, 13.3339630816529, 9.90712308446456, 9.64104386762211, 16.5176432070213, 17.2410488931471, 15.2650211336525, 19.6113373393987, 25.2504234281357, 12.5448441697173, 3.45415562179594, 20.0286926223113, 11.453996187088, 11.5599569176632, 20.1278568484835, 13.576134964745, 8.89641143872732, 15.9065173987458, 14.3055431878048, 15.0288209294994, 16.9264020056317, 13.146699841038, 18.2218827425942, 13.8975671909062, 16.6589098195785, 20.4841950657467, 17.175907454169, 13.3703420723439, 20.7440380922555, 19.9675192798106, 17.7419847975404, 16.1936586755572, 11.8604696198031, 21.80326224265, 11.9987020642644, 25.9366649650829, 22.6630531309259, 13.8214982044976, 9.8678954984661, 11.4479671815035, 16.2844185457826, 13.7665406076881, 13.2622870030113, 10.2419071636749, 14.7748613759554, 11.0754776527146, 6.66029031705932, 13.0988673985612, 19.5949830453038, 12.123265186958, 18.0398216111252, 6.91058645855418, 14.7221901723773, 17.5970360197173, 16.5057668108336, 15.5283809707447, 11.7964699584731, 10.7514782698321, 9.87935604697543, 15.5882329855006, 10.262626929076, 12.5472127814967, 13.7195390390088, 24.219310026161, 11.7402504915227, 16.1769328614243, 15.3898042478186, 10.1907168293494, 14.643459569382, 22.2227542921167, 17.2575202653961, 15.2061646099647, 12.8875158383019, 4.73376389229742, 20.6566860670709, 7.69679964537589, 18.6997375543867, 24.5455178460874, 7.780534195141, 18.5089216768736, 13.6890125529877, 7.13927920427256, 7.42666173109124, 6.99231913212703, 12.3454673891485, 7.6912220750205, 18.4395838648791, 25.5005447026284, 8.56484761982411, 18.9386942373759, 18.8452112050045, 16.6610128947506, 9.95811695861496, 14.4027369668467, 13.5980233241488, 17.8149476661024, 13.1378062194809, 19.8848669334281, 13.1270957111649, 20.2635573278967, 9.75411496666967, 8.69922377620944, 31.205199674712, 12.9157120591978, 16.4911379577036, 18.1828483701692, 12.5810968714563, 17.584310221568, 16.8448226369254, 13.9230974617915, 15.3264651676266, 14.8296637313077, 25.6422594950809, 11.2933195186359, 9.52001866462668, 15.1889419958554, 16.5524037472157, 17.1826173945509, 12.7081733364445, 9.68336933014404, 21.3159258804475, 13.2517480602322, 10.6724356867331, 13.8186021552945, 14.0141205282572, 20.5496014485682, 15.423686460986, 18.7702689259226, 12.5035399141387, 16.072226547908, 13.3765704425458, 15.4729176408679, 10.5231832101123, 8.44599233336014, 24.9860669237398, 18.0035441183621, 8.74364319187528, 11.9441704165979, 9.07259957701345, 25.9940517444186, 21.5620648821676, 13.6742747165182, 17.7159702961604, 12.9283002604057, 12.6187655269221, 11.0569858107488, 12.0269136627024, 23.2545373366834, 14.7298593745728, 15.5962261821379, 16.2184371479955, 21.1623793924267, 12.4196808452761, 10.0374642480398, 23.3784846620159, 12.7941839154736, 11.3846701503006, 8.81863440558353, 8.57642138841102, 12.1301326035101, 18.0899290858326, 20.5492406946486, 18.5379417691779, 13.1817135145237, 15.298749686923, 11.4770176815996, 11.4139091921299, 19.4232524948846, 9.92203710698228, 24.7764698274623, 14.5484020301707, 16.0726941331461, 11.3073614763021, 12.1280565511837, 8.41491933847379, 14.0853730581364, 17.0949120246223, 16.6215217208069, 11.0923175647262, 11.05689014573, 12.4890064082857, 22.4803033492318, 9.31348189667128, 14.104742028099, 24.5118091083946, 14.495125573356, 8.20079648089303, 11.6761528236297, 17.4272998945244, 13.1219856416511, 12.1906181822511, 13.2804138293577, 15.4524832356961, 22.9925438557291, 14.5571744393056, 20.4039974807576, 18.1537705782528, 14.4318005224693, 7.33548998554701, 12.3944134122374, 12.5506477343076, 15.2357721638076, 21.5009933883341, 26.4653948691555, 22.7379052949188, 14.3342451783553, 6.21736302221181, 13.0561006796413, 15.4460361153665, 19.2250650203372, 19.8126398424214, 18.4215471470823, 8.02362825100266, 19.2482152281668, 12.7672139178639, 15.8740135008063, 15.3727558858687, 17.1408338248525, 15.1233749141307, 6.66262451207169, 18.6824798238672, 16.9301328417484, 13.6717418736089, 15.5907225552334, 15.6701932268423, 16.105097342805, 23.2042308298874, 13.9047481053326, 15.8403269194233, 20.8419193653455, 20.2709051168846, 20.7263155519018, 12.1126599947022, 25.0124136514641, 15.3335043546509, 24.3342592235343, 8.24548656984644, 15.1049179317712, 21.2495728548461, 11.423789063886, 11.2365551589113, 10.3073064819655, 9.73743360330632, 12.814202334098, 16.6558958647949, 4.9289475103964, 16.0599021668615, 21.1833752320829, 25.1878700912022, 21.5058799610029, 18.7838738189798, 6.36634800442835, 11.9924664599661, 13.2397677170869, 18.5176195137845, 14.4716433299811, 8.70675685969914, 23.4221785404706, 19.5569564589798, 16.1871513624551, 21.0905430516291, 8.30612856382514, 18.304101488949, 12.3854381184329, 18.4187276092536, 14.6958902266996, 18.1648035651573, 21.677588075297, 15.0364504515845, 20.0877931847605, 9.05782982426011, 11.3919777978199, 22.5960885569409, 16.8869398651196, 4.73888589783135, 8.17981273958812, 13.9960949220544, 19.3288970216724, 14.4905837214239, 18.1209373601033, 19.7950268889391, 23.3552741443147, 15.2800836663748, 14.7400904690955, 6.23381320428865, 15.4966379704391, 12.1407497105222, 10.1299520859795, 14.1004688447623, 20.0747158637183, 5.03625755655709, 12.8636035639729, 15.5831864179135, 10.5339621497252, 16.6695147124961, 17.0571496030787, 14.83481920362, 2.67050903119986, 27.8572907293332, 13.973503712659, 18.2559664079384, 16.3688324551827, 20.1233661740917, 19.0882972318704 \n\nRolling Mean Temperature in Moving Windows:\n 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14.764481219841, 14.9289711978458, 14.918155532261, 14.8075584234103, 14.942162606095, 15.0575448305078, 14.8864740417106, 14.9019742829687, 15.1024992036403, 15.1659809016659, 15.1768460631805, 14.8570479337872, 14.762426082941, 14.4847314492473, 14.8277769908128, 15.121744179656, 14.636740559655, 14.4866180074002, 14.7366116412346, 14.7497131773634, 14.8146169009061, 15.034807270903, 15.066378630788, 15.2302342959573, 15.5798165481749, 15.6463612618732, 16.1802219144019, 15.7824652729477, 15.854338711581, 16.0646705750571, 15.8916916830028, 15.8838875598833, 15.8493455648638, 15.6446233907447, 15.328505245638, 15.0129431942858, 14.9487579255032, 14.9311399463518, 14.9502926025697, 15.1549976245063, 15.5600902389462, 15.5940362077569, 15.2438276081534, 15.6223500874918, 15.142657299505, 14.8266621970429, 15.1847755225263, 15.2044608272343, 15.0787837674624, 14.9790068276978, 14.9697531117033, 14.9284940570208, 14.9974652497667, 14.9428336540165, 14.8221293647672, 14.823010102074, 14.6255619953217, 15.0664939648849, 15.0415885884179, 14.9666242835219, 15.1221019584731, 15.2244126873401, 15.3995327561097, 15.4948559425732, 15.5599674937511, 15.9653747729187, 15.8147434014935, 16.104597270558, 16.3511983371338, 16.1582036993038, 15.6454527683148, 15.608890202041, 16.0365656328405, 15.8278272323531, 15.8881035928839, 15.8441686010843, 15.6657354186667, 15.5823801749323, 15.5078428042101, 15.4142544708706, 15.5905691327872, 15.4937172747025, 15.5308312615523, 15.3229608154695, 15.2063043964623, 15.3296200240893, 15.3245152571311, 15.1593214539644, 14.9800068707745, 14.8927114106908, 14.5305553425148, 14.3845791327045, 14.1352672037557, 14.013719007287, 14.0756879879272, 14.1562229140442, 14.1476078616195, 13.8222834581642, 13.5798418287273, 13.458815782889, 13.6180012519195, 13.9771608222733, 14.0095975462604, 14.057585013003, 14.045092640846, 13.8614878651334, 14.0575486881706, 13.944926087926, 14.3462409958369, 14.7277960107544, 14.3339810490823, 14.5468362654128, 14.4018092968083, 14.4094323883322, 14.166248106956, 13.8127575440363, 13.6740808966468, 13.4128422667893, 13.6342793970029, 14.1259149447628, 14.082097997191, 14.1937800389202, 14.4798661814512, 14.616992851893, 14.4916121158798, 14.1643930139027, 14.2263187749902, 14.2809192684795, 14.2058526675349, 14.5289910043375, 14.478445542397, 14.413138976923, 14.1630254669654, 13.9461274391736, 14.5567169003872, 14.8294485059506, 14.690596902305, 15.0401318597981, 14.8361771703671, 14.6041369162165, 14.906279864276, 14.7534190571066, 14.8080008109278, 15.064346961829, 15.671533553962, 15.8149002335123, 15.7207186093616, 15.9706426067227, 15.9077366028006, 15.630472359198, 15.7685832164187, 15.4600723861777, 15.5424295420258, 15.4287873808751, 15.4525980051458, 15.4331268447607, 15.4469967515643, 15.5381518776465, 15.6143478856967, 15.5771946187798, 15.5564094255456, 15.416698399546, 15.5374469154085, 15.7632367108971, 15.0738361620771, 14.9248455045492, 15.2080098034171, 15.2020329950235, 15.0741178723708, 14.8861132122051, 14.6270391102081, 15.0294042529623, 15.2372575767803, 15.1987446096207, 14.9345349696567, 14.9890343277157, 15.0923258897922, 14.9545940169553, 14.8037443474715, 15.006141678876, 15.0735312134802, 15.27062644188, 15.1007101507983, 15.3643978618714, 15.4226393671562, 15.2966014369144, 15.6087469080397, 15.3502329902699, 15.215599113247, 14.883877962569, 14.7529740117114, 14.6215708802315, 14.7786828350077, 14.9478936034671, 15.2150522221026, 15.3729095948081, 15.0499990202475, 14.8324481390221, 14.9214570056973, 15.1707597416401, 15.1990743259724, 15.1584882620739, 14.9246995003407, 15.0046468142283, 14.791026520233, 14.7643517299256, 14.624223523644, 14.7251697652236, 14.8941030439542, 14.673002523425, 14.5517511297635, 14.4004399285499, 14.2761255705595, 14.3200563691197, 14.2165164041662, 14.3520923301682, 14.3898698117141, 14.4465678669769, 14.3404387446633, 14.4356893585982, 14.7307186421353, 14.7637804100733, 14.5671367132873, 14.3248424844442, 14.2219938666615, 14.5490215447017, 14.5243023697811, 14.8218683630864, 15.0465304092905, 14.8801486768766, 14.7939304394954, 14.3811952256546, 14.3146034157925, 14.2867060168146, 14.6264937472156, 15.1044050244814, 15.5818378896962, 15.5901336270368, 15.2275486602898, 15.108701292251, 15.253825243939, 15.5260977397592, 15.7702188542304, 15.6349269808254, 15.5919318593031, 15.763380965972, 15.371894459621, 15.417857390536, 15.6569227040352, 15.8390787374093, 15.7622812380629, 15.5469692004102, 15.7633645884641, 15.8850218888771, 15.8256638434742, 15.578936466791, 15.6160370930422, 15.4727404217771, 15.6410890968316, 15.6235206829271, 15.9070152473896, 16.1885987791599, 16.4459406919125, 16.6289588048489, 16.3160143583945, 16.2675816511381, 16.0207682864626, 16.3541020879685, 16.421706206223, 16.4900001146273, 16.6834513392767, 16.423408807395, 16.1375393179446, 15.8670646291074, 15.9241914741842, 15.7097243777152, 15.8393471092796, 15.4745115762659, 15.497416452299, 15.63216783254, 15.9676510051091, 16.4624261867401, 16.4658059865772, 16.1136798253332, 16.0577039782118, 15.9793388169402, 16.074253026505, 16.0198045594108, 15.5365554270712, 15.8538031082425, 15.977690759561, 15.8225318261314, 15.8498530906228, 15.435846857687, 15.6422282408285, 15.2213290563941, 15.3241698315476, 15.0028908649864, 15.3335347648301, 15.5526237696143, 15.3455196895056, 15.6343198268681, 15.5616956490464, 15.5978513595748, 16.0264731913627, 16.1622311090634, 15.7649974434979, 15.8733596178043, 15.8045660429774, 15.7427501026304, 15.3861738903045, 15.2733424702745, 15.3070475726064, 15.8733451106027, 15.9829323508163, 16.0329431092166, 15.6234828989, 15.6576493869153, 15.7721158152761, 15.3290416001264, 15.1471586796525, 15.2767441630279, 14.7416013131922, 14.8935171465304, 14.8028199774959, 14.7411041118723, 14.6827970153137, 14.761505661193, 14.6505061824751, 14.0169368810052, 14.4442982235968, 14.2404885745268, 14.5470931273161, 14.7129882825615, 14.6305642031332, 14.7039427820249, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 \n" + } + ], + "source": "# consume climwin\n\ncat(\"Original Temperature Data:\\n\", head(temperature_data_str), \"\\n\\n\")\ncat(\"Rolling Mean Temperature in Moving Windows:\\n\", head(coredata(rolling_mean_temp_str)), \"\\n\")\n" + } + ], + "metadata": { + "kernelspec": { + "display_name": "R", + "language": "R", + "name": "R" + }, + "language_info": { + "codemirror_mode": "r", + "file_extension": ".r", + "mimetype": "text/x-r-source", + "name": "R", + "pygments_lexer": "r", + "version": "4.3.2" + } + }, + "nbformat": 4, + "nbformat_minor": 5 + }, + "original_source": "\ncat(\"Original Temperature Data:\\n\", head(temperature_data_str), \"\\n\\n\")\ncat(\"Rolling Mean Temperature in Moving Windows:\\n\", head(coredata(rolling_mean_temp_str)), \"\\n\")", + "outputs": [], + "param_values": {}, + "params": [], + "task_name": "consume-climwin-dev-user-name-at-domain-com", + "title": "consume climwin-dev-user-name-at-domain-com", + "types": { + "rolling_mean_temp_str": "str", + "temperature_data_str": "str" + }, + "skip_exec": false +} diff --git a/vreapis/tests/resources/cells/create-file-dev-user-name-at-domain-com.json b/vreapis/tests/resources/cells/create-file-dev-user-name-at-domain-com.json new file mode 100644 index 00000000..f8241ad8 --- /dev/null +++ b/vreapis/tests/resources/cells/create-file-dev-user-name-at-domain-com.json @@ -0,0 +1,93 @@ +{ + "all_inputs": [], + "base_image": {"build":"ghcr.io/qcdis/naavre/naavre-cell-build-python:latest","runtime":"ghcr.io/qcdis/naavre/naavre-cell-runtime-python:latest"}, + "chart_obj": { + "hovered": {}, + "links": {}, + "nodes": { + "5a918c8": { + "id": "5a918c8", + "ports": { + "file_path": { + "id": "file_path", + "properties": { + "color": "#663a78" + }, + "type": "right" + } + }, + "position": { + "x": 35, + "y": 15 + }, + "properties": { + "inputs": [], + "og_node_id": "5a918c8", + "outputs": [ + "file_path" + ], + "params": [], + "title": "Create file-dev-user-name-at-domain-com", + "vars": [ + { + "color": "#663a78", + "direction": "output", + "name": "file_path", + "type": "datatype" + } + ] + }, + "type": "input-output" + } + }, + "offset": { + "x": 0, + "y": 0 + }, + "scale": 1, + "selected": {} + }, + "confs": { + "conf_data_folder": "conf_data_folder = os.path.join('/tmp','data')" + }, + "container_source": "", + "dependencies": [ + { + "asname": null, + "module": "os.path", + "name": "isfile" + }, + { + "asname": null, + "module": "os.path", + "name": "join" + }, + { + "asname": null, + "module": "os", + "name": "listdir" + }, + { + "asname": null, + "module": "", + "name": "os" + } + ], + "inputs": [], + "kernel": "ipython", + "node_id": "5a918c8", + "original_source": "conf_data_folder = os.path.join('/tmp','data')\n\nL = [\"a\\n\", \"b\\n\", \"c\\n\"]\nfile_path = os.path.join(conf_data_folder,'hello.txt')\nfp = open(file_path, 'w')\nfp.writelines(L)\nfp.close()\n\nonlyfiles = [f for f in listdir(conf_data_folder) if isfile(join(conf_data_folder, f))]\n\nprint(onlyfiles)", + "outputs": [ + "file_path" + ], + "params": [], + "task_name": "create-file-dev-user-name-at-domain-com", + "title": "Create file-dev-user-name-at-domain-com", + "types": { + "file_path": "str" + }, + "example_inputs" : [ + "--id", + "0" + ] +} diff --git a/vreapis/tests/resources/cells/create-file-r-dev-user-name-at-domain-com.json b/vreapis/tests/resources/cells/create-file-r-dev-user-name-at-domain-com.json new file mode 100644 index 00000000..0873cf67 --- /dev/null +++ b/vreapis/tests/resources/cells/create-file-r-dev-user-name-at-domain-com.json @@ -0,0 +1,70 @@ +{ + "all_inputs": [], + "base_image": {"build":"ghcr.io/qcdis/naavre/naavre-cell-build-r:latest","runtime":"ghcr.io/qcdis/naavre/naavre-cell-runtime-r:latest"}, + "chart_obj": { + "hovered": {}, + "links": {}, + "nodes": { + "c45fb67": { + "id": "c45fb67", + "ports": { + "file_path": { + "id": "file_path", + "properties": { + "color": "#663a78" + }, + "type": "right" + } + }, + "position": { + "x": 35, + "y": 15 + }, + "properties": { + "inputs": [], + "og_node_id": "c45fb67", + "outputs": [ + "file_path" + ], + "params": [], + "title": "Create file R-dev-user-name-at-domain-com", + "vars": [ + { + "color": "#663a78", + "direction": "output", + "name": "file_path", + "type": "datatype" + } + ] + }, + "type": "input-output" + } + }, + "offset": { + "x": 0, + "y": 0 + }, + "scale": 1, + "selected": {} + }, + "confs": {}, + "container_source": "", + "dependencies": [], + "inputs": [], + "kernel": "IRkernel", + "node_id": "c45fb67", + "original_source": "\nL <- c(\"a\", \"b\", \"c\")\n\nconf_data_folder <- \"/tmp/data\"\nfile_path <- file.path(conf_data_folder, \"hello.txt\")\n\nwriteLines(L, file_path)\n\nonlyfiles <- list.files(conf_data_folder, full.names = TRUE)\n\nprint(onlyfiles)", + "outputs": [ + "file_path" + ], + "params": [], + "task_name": "create-file-r-dev-user-name-at-domain-com", + "title": "Create file R-dev-user-name-at-domain-com", + "types": { + "file_path": "str" + }, + "example_inputs" : [ + "--id", + "0" + ] +} diff --git a/vreapis/tests/resources/cells/input-lists-dev-user-name-at-domain-com.json b/vreapis/tests/resources/cells/input-lists-dev-user-name-at-domain-com.json new file mode 100644 index 00000000..684c6e2a --- /dev/null +++ b/vreapis/tests/resources/cells/input-lists-dev-user-name-at-domain-com.json @@ -0,0 +1,103 @@ +{ + "all_inputs": [ + "msg" + ], + "base_image": {"build":"ghcr.io/qcdis/naavre/naavre-cell-build-python:latest","runtime":"ghcr.io/qcdis/naavre/naavre-cell-runtime-python:latest"}, + "chart_obj": { + "hovered": {}, + "links": {}, + "nodes": { + "3e4f548": { + "id": "3e4f548", + "ports": { + "list_of_ints": { + "id": "list_of_ints", + "properties": { + "color": "#5f1f93" + }, + "type": "right" + }, + "list_of_paths": { + "id": "list_of_paths", + "properties": { + "color": "#79c5d2" + }, + "type": "right" + }, + "msg": { + "id": "msg", + "properties": { + "color": "#7bd279" + }, + "type": "left" + } + }, + "position": { + "x": 35, + "y": 15 + }, + "properties": { + "inputs": [ + "msg" + ], + "og_node_id": "3e4f548", + "outputs": [ + "list_of_ints", + "list_of_paths" + ], + "params": [], + "title": "input lists-dev-user-name-at-domain-com", + "vars": [ + { + "color": "#7bd279", + "direction": "input", + "name": "msg", + "type": "datatype" + }, + { + "color": "#5f1f93", + "direction": "output", + "name": "list_of_ints", + "type": "datatype" + }, + { + "color": "#79c5d2", + "direction": "output", + "name": "list_of_paths", + "type": "datatype" + } + ] + }, + "type": "input-output" + } + }, + "offset": { + "x": 0, + "y": 0 + }, + "scale": 1, + "selected": {} + }, + "confs": {}, + "container_source": "", + "dependencies": [], + "inputs": [ + "msg" + ], + "kernel": "ipython", + "node_id": "3e4f548", + "original_source": "\nlist_of_paths = [\"/webdav/LAZ/targets_myname\",\"/webdav/LAZ/targets_myname\",\"/webdav/LAZ/targets_myname\",\"/webdav/LAZ/targets_myname\",\"/webdav/LAZ/targets_myname\",\"/webdav/LAZ/targets_myname\",\"/webdav/LAZ/targets_myname\",\"/webdav/LAZ/targets_myname\",\"/webdav/LAZ/targets_myname\",\"/webdav/LAZ/targets_myname\"]\nlist_of_ints = [1,2,35,6,65]\nprint(msg)", + "outputs": [ + "list_of_paths", + "list_of_ints" + ], + "params": [], + "task_name": "input-lists-dev-user-name-at-domain-com", + "title": "input lists-dev-user-name-at-domain-com", + "types": { + "list_of_ints": "list", + "list_of_paths": "list", + "msg": "str" + }, + "skip_exec": true +} diff --git a/vreapis/tests/resources/cells/input-lists-r-dev-user-name-at-domain-com.json b/vreapis/tests/resources/cells/input-lists-r-dev-user-name-at-domain-com.json new file mode 100644 index 00000000..6ed403d6 --- /dev/null +++ b/vreapis/tests/resources/cells/input-lists-r-dev-user-name-at-domain-com.json @@ -0,0 +1,83 @@ +{ + "all_inputs": [], + "base_image": {"build":"ghcr.io/qcdis/naavre/naavre-cell-build-r:latest","runtime":"ghcr.io/qcdis/naavre/naavre-cell-runtime-r:latest"}, + "chart_obj": { + "hovered": {}, + "links": {}, + "nodes": { + "3cfdc6f": { + "id": "3cfdc6f", + "ports": { + "list_of_ints": { + "id": "list_of_ints", + "properties": { + "color": "#5f1f93" + }, + "type": "right" + }, + "list_of_paths": { + "id": "list_of_paths", + "properties": { + "color": "#79c5d2" + }, + "type": "right" + } + }, + "position": { + "x": 35, + "y": 15 + }, + "properties": { + "inputs": [], + "og_node_id": "3cfdc6f", + "outputs": [ + "list_of_ints", + "list_of_paths" + ], + "params": [], + "title": "input lists R-dev-user-name-at-domain-com", + "vars": [ + { + "color": "#5f1f93", + "direction": "output", + "name": "list_of_ints", + "type": "datatype" + }, + { + "color": "#79c5d2", + "direction": "output", + "name": "list_of_paths", + "type": "datatype" + } + ] + }, + "type": "input-output" + } + }, + "offset": { + "x": 0, + "y": 0 + }, + "scale": 1, + "selected": {} + }, + "confs": {}, + "container_source": "", + "dependencies": [], + "inputs": [], + "kernel": "IRkernel", + "node_id": "3cfdc6f", + "original_source": "\nlist_of_paths <- c(\n \"/webdav/LAZ/targets_myname\",\n \"/webdav/LAZ/targets_myname\",\n \"/webdav/LAZ/targets_myname\",\n \"/webdav/LAZ/targets_myname\"\n)\n\nlist_of_ints <- c(1, 2, 35, 6, 65)\n\nprint(list_of_paths)\nprint(list_of_ints)", + "outputs": [ + "list_of_paths", + "list_of_ints" + ], + "params": [], + "task_name": "input-lists-r-dev-user-name-at-domain-com", + "title": "input lists R-dev-user-name-at-domain-com", + "types": { + "list_of_ints": "list", + "list_of_paths": "list" + }, + "skip_exec": true +} diff --git a/vreapis/tests/resources/cells/install-and-load-the-climwinb-packages-dev-user-name-at-domain-com.json b/vreapis/tests/resources/cells/install-and-load-the-climwinb-packages-dev-user-name-at-domain-com.json new file mode 100644 index 00000000..5e5f8b1d --- /dev/null +++ b/vreapis/tests/resources/cells/install-and-load-the-climwinb-packages-dev-user-name-at-domain-com.json @@ -0,0 +1,130 @@ +{ + "all_inputs": [], + "base_image": {"build":"ghcr.io/qcdis/naavre/naavre-cell-build-lter-life-veluwe:latest","runtime":"ghcr.io/qcdis/naavre/naavre-cell-runtime-lter-life-veluwe:latest"}, + "chart_obj": { + "hovered": {}, + "links": {}, + "nodes": { + "7b4271d": { + "id": "7b4271d", + "ports": { + "rolling_mean_temp_str": { + "id": "rolling_mean_temp_str", + "properties": { + "color": "#c5a987" + }, + "type": "right" + }, + "temperature_data_str": { + "id": "temperature_data_str", + "properties": { + "color": "#aca953" + }, + "type": "right" + } + }, + "position": { + "x": 35, + "y": 15 + }, + "properties": { + "inputs": [], + "og_node_id": "7b4271d", + "outputs": [ + "rolling_mean_temp_str", + "temperature_data_str" + ], + "params": [], + "title": "Install and load the climwinb packages-dev-user-name-at-domain-com", + "vars": [ + { + "color": "#c5a987", + "direction": "output", + "name": "rolling_mean_temp_str", + "type": "datatype" + }, + { + "color": "#aca953", + "direction": "output", + "name": "temperature_data_str", + "type": "datatype" + } + ] + }, + "type": "input-output" + } + }, + "offset": { + "x": 0, + "y": 0 + }, + "scale": 1, + "selected": {} + }, + "confs": {}, + "container_source": "", + "dependencies": [ + { + "asname": "", + "module": "", + "name": "climwin" + }, + { + "asname": "", + "module": "", + "name": "zoo" + } + ], + "inputs": [], + "kernel": "IRkernel", + "node_id": "7b4271d", + "notebook_dict": { + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "4681c280-3623-47c4-a020-fe8f34166536", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": "Loading required package: ggplot2\n\nLoading required package: gridExtra\n\nLoading required package: Matrix\n\n\nAttaching package: \u2018zoo\u2019\n\n\nThe following objects are masked from \u2018package:base\u2019:\n\n as.Date, as.Date.numeric\n\n\n" + } + ], + "source": "# Install and load the climwinb packages\nif (!requireNamespace(\"climwin\", quietly = TRUE)) {\n install.packages(\"climwin\",repos = \"http://cran.us.r-project.org\")\n}\nif (!requireNamespace(\"zoo\", quietly = TRUE)) {\n install.packages(\"zoo\",repos = \"http://cran.us.r-project.org\")\n}\n\nzoo = ''\nclimwin = ''\nlibrary(climwin)\nlibrary(zoo)\n\n# Generate example temperature data\nset.seed(123)\ntemperature_data <- rnorm(365, mean = 15, sd = 5)\n\n# Define the window size (e.g., 30 days)\nwindow_size <- 30\n\n# Convert temperature_data to a zoo object for rollmean\ntemperature_zoo <- zoo::zoo(temperature_data)\n\n# Calculate the rolling mean temperature within a moving window\nrolling_mean_temp <- rollmean(temperature_zoo, k = window_size, fill = 0.0)\n\ntemperature_zoo_str <- toString(temperature_zoo)\nrolling_mean_temp_str <- toString(rolling_mean_temp)\ntemperature_data_str <- toString(temperature_data)\n\n\n" + } + ], + "metadata": { + "kernelspec": { + "display_name": "R", + "language": "R", + "name": "R" + }, + "language_info": { + "codemirror_mode": "r", + "file_extension": ".r", + "mimetype": "text/x-r-source", + "name": "R", + "pygments_lexer": "r", + "version": "4.3.2" + } + }, + "nbformat": 4, + "nbformat_minor": 5 + }, + "original_source": "if (!requireNamespace(\"climwin\", quietly = TRUE)) {\n install.packages(\"climwin\",repos = \"http://cran.us.r-project.org\")\n}\nif (!requireNamespace(\"zoo\", quietly = TRUE)) {\n install.packages(\"zoo\",repos = \"http://cran.us.r-project.org\")\n}\n\nzoo = ''\nclimwin = ''\nlibrary(climwin)\nlibrary(zoo)\n\nset.seed(123)\ntemperature_data <- rnorm(365, mean = 15, sd = 5)\n\nwindow_size <- 30\n\ntemperature_zoo <- zoo::zoo(temperature_data)\n\nrolling_mean_temp <- rollmean(temperature_zoo, k = window_size, fill = 0.0)\n\ntemperature_zoo_str <- toString(temperature_zoo)\nrolling_mean_temp_str <- toString(rolling_mean_temp)\ntemperature_data_str <- toString(temperature_data)\n", + "outputs": [ + "rolling_mean_temp_str", + "temperature_data_str" + ], + "param_values": {}, + "params": [], + "task_name": "install-and-load-the-climwinb-packages-dev-user-name-at-domain-com", + "title": "Install and load the climwinb packages-dev-user-name-at-domain-com", + "types": { + "rolling_mean_temp_str": "str", + "temperature_data_str": "str" + }, + "skip_exec": false +} diff --git a/vreapis/tests/resources/cells/knmi-vol-h5-to-odim-h5-dev-user-name-at-domain-com.json b/vreapis/tests/resources/cells/knmi-vol-h5-to-odim-h5-dev-user-name-at-domain-com.json new file mode 100644 index 00000000..8c98afa4 --- /dev/null +++ b/vreapis/tests/resources/cells/knmi-vol-h5-to-odim-h5-dev-user-name-at-domain-com.json @@ -0,0 +1,93 @@ +{ + "all_inputs": [ + "a" + ], + "base_image": {"build":"ghcr.io/qcdis/naavre/naavre-cell-build-vol2bird:latest","runtime":"ghcr.io/qcdis/naavre/naavre-cell-runtime-vol2bird:latest"}, + "chart_obj": { + "hovered": {}, + "links": {}, + "nodes": { + "e25ec34": { + "id": "e25ec34", + "ports": { + "a": { + "id": "a", + "properties": { + "color": "#77862d" + }, + "type": "left" + }, + "msg": { + "id": "msg", + "properties": { + "color": "#7bd279" + }, + "type": "right" + } + }, + "position": { + "x": 35, + "y": 15 + }, + "properties": { + "inputs": [ + "a" + ], + "og_node_id": "e25ec34", + "outputs": [ + "msg" + ], + "params": [], + "title": "KNMI-vol-h5-to-ODIM-h5-dev-user-name-at-domain-com", + "vars": [ + { + "color": "#77862d", + "direction": "input", + "name": "a", + "type": "datatype" + }, + { + "color": "#7bd279", + "direction": "output", + "name": "msg", + "type": "datatype" + } + ] + }, + "type": "input-output" + } + }, + "offset": { + "x": 0, + "y": 0 + }, + "scale": 1, + "selected": {} + }, + "confs": {}, + "container_source": "", + "dependencies": [ + { + "asname": null, + "module": "", + "name": "os" + } + ], + "inputs": [ + "a" + ], + "kernel": "ipython", + "node_id": "e25ec34", + "original_source": "print(a)\ncmd = \"KNMI_vol_h5_to_ODIM_h5 \"\n\nmsg = os.system(cmd) # returns the exit code in unix", + "outputs": [ + "msg" + ], + "params": [], + "task_name": "knmi-vol-h5-to-odim-h5-dev-user-name-at-domain-com", + "title": "KNMI-vol-h5-to-ODIM-h5-dev-user-name-at-domain-com", + "types": { + "a": "int", + "msg": "int" + }, + "skip_exec": true +} diff --git a/vreapis/tests/resources/cells/loop-file-lines-dev-user-name-at-domain-com.json b/vreapis/tests/resources/cells/loop-file-lines-dev-user-name-at-domain-com.json new file mode 100644 index 00000000..d9a47f78 --- /dev/null +++ b/vreapis/tests/resources/cells/loop-file-lines-dev-user-name-at-domain-com.json @@ -0,0 +1,87 @@ +{ + "all_inputs": [ + "lines" + ], + "base_image": {"build":"ghcr.io/qcdis/naavre/naavre-cell-build-python:latest","runtime":"ghcr.io/qcdis/naavre/naavre-cell-runtime-python:latest"}, + "chart_obj": { + "hovered": {}, + "links": {}, + "nodes": { + "5d9de73": { + "id": "5d9de73", + "ports": { + "count": { + "id": "count", + "properties": { + "color": "#783d3a" + }, + "type": "right" + }, + "lines": { + "id": "lines", + "properties": { + "color": "#ac535c" + }, + "type": "left" + } + }, + "position": { + "x": 35, + "y": 15 + }, + "properties": { + "inputs": [ + "lines" + ], + "og_node_id": "5d9de73", + "outputs": [ + "count" + ], + "params": [], + "title": "loop file lines-dev-user-name-at-domain-com", + "vars": [ + { + "color": "#ac535c", + "direction": "input", + "name": "lines", + "type": "datatype" + }, + { + "color": "#783d3a", + "direction": "output", + "name": "count", + "type": "datatype" + } + ] + }, + "type": "input-output" + } + }, + "offset": { + "x": 0, + "y": 0 + }, + "scale": 1, + "selected": {} + }, + "confs": {}, + "container_source": "", + "dependencies": [], + "inputs": [ + "lines" + ], + "kernel": "ipython", + "node_id": "5d9de73", + "original_source": "\ncount = 0\nfor l in lines:\n count += 1\n print(\"Line{}: {}\".format(count, l.strip()))", + "outputs": [ + "count" + ], + "params": [], + "task_name": "loop-file-lines-dev-user-name-at-domain-com", + "title": "loop file lines-dev-user-name-at-domain-com", + "types": { + "count": "int", + "lines": "list" + }, + "skip_exec": true +} diff --git a/vreapis/tests/resources/cells/loop-file-lines-r-dev-user-name-at-domain-com.json b/vreapis/tests/resources/cells/loop-file-lines-r-dev-user-name-at-domain-com.json new file mode 100644 index 00000000..a20cf0df --- /dev/null +++ b/vreapis/tests/resources/cells/loop-file-lines-r-dev-user-name-at-domain-com.json @@ -0,0 +1,87 @@ +{ + "all_inputs": [ + "lines" + ], + "base_image": {"build":"ghcr.io/qcdis/naavre/naavre-cell-build-r:latest","runtime":"ghcr.io/qcdis/naavre/naavre-cell-runtime-r:latest"}, + "chart_obj": { + "hovered": {}, + "links": {}, + "nodes": { + "f893a78": { + "id": "f893a78", + "ports": { + "count": { + "id": "count", + "properties": { + "color": "#783d3a" + }, + "type": "right" + }, + "lines": { + "id": "lines", + "properties": { + "color": "#ac535c" + }, + "type": "left" + } + }, + "position": { + "x": 35, + "y": 15 + }, + "properties": { + "inputs": [ + "lines" + ], + "og_node_id": "f893a78", + "outputs": [ + "count" + ], + "params": [], + "title": "loop file lines R-dev-user-name-at-domain-com", + "vars": [ + { + "color": "#ac535c", + "direction": "input", + "name": "lines", + "type": "datatype" + }, + { + "color": "#783d3a", + "direction": "output", + "name": "count", + "type": "datatype" + } + ] + }, + "type": "input-output" + } + }, + "offset": { + "x": 0, + "y": 0 + }, + "scale": 1, + "selected": {} + }, + "confs": {}, + "container_source": "", + "dependencies": [], + "inputs": [ + "lines" + ], + "kernel": "IRkernel", + "node_id": "f893a78", + "original_source": "count <- 0\nfor (l in lines) {\n count <- count + 1\n cat(sprintf(\"Line %d: %s\\n\", count, trimws(l)))\n}", + "outputs": [ + "count" + ], + "params": [], + "task_name": "loop-file-lines-r-dev-user-name-at-domain-com", + "title": "loop file lines R-dev-user-name-at-domain-com", + "types": { + "count": "int", + "lines": "list" + }, + "skip_exec": true +} diff --git a/vreapis/tests/resources/cells/loop-int-list-dev-user-name-at-domain-com.json b/vreapis/tests/resources/cells/loop-int-list-dev-user-name-at-domain-com.json new file mode 100644 index 00000000..f2b2d69f --- /dev/null +++ b/vreapis/tests/resources/cells/loop-int-list-dev-user-name-at-domain-com.json @@ -0,0 +1,87 @@ +{ + "all_inputs": [ + "list_of_ints" + ], + "base_image": {"build":"ghcr.io/qcdis/naavre/naavre-cell-build-python:latest","runtime":"ghcr.io/qcdis/naavre/naavre-cell-runtime-python:latest"}, + "chart_obj": { + "hovered": {}, + "links": {}, + "nodes": { + "e1578ef": { + "id": "e1578ef", + "ports": { + "a": { + "id": "a", + "properties": { + "color": "#77862d" + }, + "type": "right" + }, + "list_of_ints": { + "id": "list_of_ints", + "properties": { + "color": "#5f1f93" + }, + "type": "left" + } + }, + "position": { + "x": 35, + "y": 15 + }, + "properties": { + "inputs": [ + "list_of_ints" + ], + "og_node_id": "e1578ef", + "outputs": [ + "a" + ], + "params": [], + "title": "loop int list-dev-user-name-at-domain-com", + "vars": [ + { + "color": "#5f1f93", + "direction": "input", + "name": "list_of_ints", + "type": "datatype" + }, + { + "color": "#77862d", + "direction": "output", + "name": "a", + "type": "datatype" + } + ] + }, + "type": "input-output" + } + }, + "offset": { + "x": 0, + "y": 0 + }, + "scale": 1, + "selected": {} + }, + "confs": {}, + "container_source": "", + "dependencies": [], + "inputs": [ + "list_of_ints" + ], + "kernel": "ipython", + "node_id": "e1578ef", + "original_source": "\nfor i in list_of_ints:\n a = i -1\n print(a)", + "outputs": [ + "a" + ], + "params": [], + "task_name": "loop-int-list-dev-user-name-at-domain-com", + "title": "loop int list-dev-user-name-at-domain-com", + "types": { + "a": "int", + "list_of_ints": "list" + }, + "skip_exec": true +} diff --git a/vreapis/tests/resources/cells/loop-int-list-r-dev-user-name-at-domain-com.json b/vreapis/tests/resources/cells/loop-int-list-r-dev-user-name-at-domain-com.json new file mode 100644 index 00000000..85ecec25 --- /dev/null +++ b/vreapis/tests/resources/cells/loop-int-list-r-dev-user-name-at-domain-com.json @@ -0,0 +1,69 @@ +{ + "all_inputs": [ + "list_of_ints" + ], + "base_image": {"build":"ghcr.io/qcdis/naavre/naavre-cell-build-r:latest","runtime":"ghcr.io/qcdis/naavre/naavre-cell-runtime-r:latest"}, + "chart_obj": { + "hovered": {}, + "links": {}, + "nodes": { + "3285abd": { + "id": "3285abd", + "ports": { + "list_of_ints": { + "id": "list_of_ints", + "properties": { + "color": "#5f1f93" + }, + "type": "left" + } + }, + "position": { + "x": 35, + "y": 15 + }, + "properties": { + "inputs": [ + "list_of_ints" + ], + "og_node_id": "3285abd", + "outputs": [], + "params": [], + "title": "loop int list R-dev-user-name-at-domain-com", + "vars": [ + { + "color": "#5f1f93", + "direction": "input", + "name": "list_of_ints", + "type": "datatype" + } + ] + }, + "type": "input-output" + } + }, + "offset": { + "x": 0, + "y": 0 + }, + "scale": 1, + "selected": {} + }, + "confs": {}, + "container_source": "", + "dependencies": [], + "inputs": [ + "list_of_ints" + ], + "kernel": "IRkernel", + "node_id": "3285abd", + "original_source": "\nfor (l in list_of_ints) {\n print(l)\n}", + "outputs": [], + "params": [], + "task_name": "loop-int-list-r-dev-user-name-at-domain-com", + "title": "loop int list R-dev-user-name-at-domain-com", + "types": { + "list_of_ints": "list" + }, + "skip_exec": true +} diff --git a/vreapis/tests/resources/cells/loop-list-dev-user-name-at-domain-com.json b/vreapis/tests/resources/cells/loop-list-dev-user-name-at-domain-com.json new file mode 100644 index 00000000..eff08db5 --- /dev/null +++ b/vreapis/tests/resources/cells/loop-list-dev-user-name-at-domain-com.json @@ -0,0 +1,69 @@ +{ + "all_inputs": [ + "list_of_paths" + ], + "base_image": {"build":"ghcr.io/qcdis/naavre/naavre-cell-build-python:latest","runtime":"ghcr.io/qcdis/naavre/naavre-cell-runtime-python:latest"}, + "chart_obj": { + "hovered": {}, + "links": {}, + "nodes": { + "6ce40db": { + "id": "6ce40db", + "ports": { + "list_of_paths": { + "id": "list_of_paths", + "properties": { + "color": "#79c5d2" + }, + "type": "left" + } + }, + "position": { + "x": 35, + "y": 15 + }, + "properties": { + "inputs": [ + "list_of_paths" + ], + "og_node_id": "6ce40db", + "outputs": [], + "params": [], + "title": "loop list-dev-user-name-at-domain-com", + "vars": [ + { + "color": "#79c5d2", + "direction": "input", + "name": "list_of_paths", + "type": "datatype" + } + ] + }, + "type": "input-output" + } + }, + "offset": { + "x": 0, + "y": 0 + }, + "scale": 1, + "selected": {} + }, + "confs": {}, + "container_source": "", + "dependencies": [], + "inputs": [ + "list_of_paths" + ], + "kernel": "ipython", + "node_id": "6ce40db", + "original_source": "\nfor l in list_of_paths:\n print(l)", + "outputs": [], + "params": [], + "task_name": "loop-list-dev-user-name-at-domain-com", + "title": "loop list-dev-user-name-at-domain-com", + "types": { + "list_of_paths": "list" + }, + "skip_exec": true +} diff --git a/vreapis/tests/resources/cells/loop-list-r-dev-user-name-at-domain-com.json b/vreapis/tests/resources/cells/loop-list-r-dev-user-name-at-domain-com.json new file mode 100644 index 00000000..1f4796cb --- /dev/null +++ b/vreapis/tests/resources/cells/loop-list-r-dev-user-name-at-domain-com.json @@ -0,0 +1,69 @@ +{ + "all_inputs": [ + "list_of_paths" + ], + "base_image": {"build":"ghcr.io/qcdis/naavre/naavre-cell-build-r:latest","runtime":"ghcr.io/qcdis/naavre/naavre-cell-runtime-r:latest"}, + "chart_obj": { + "hovered": {}, + "links": {}, + "nodes": { + "943e143": { + "id": "943e143", + "ports": { + "list_of_paths": { + "id": "list_of_paths", + "properties": { + "color": "#79c5d2" + }, + "type": "left" + } + }, + "position": { + "x": 35, + "y": 15 + }, + "properties": { + "inputs": [ + "list_of_paths" + ], + "og_node_id": "943e143", + "outputs": [], + "params": [], + "title": "loop list R-dev-user-name-at-domain-com", + "vars": [ + { + "color": "#79c5d2", + "direction": "input", + "name": "list_of_paths", + "type": "datatype" + } + ] + }, + "type": "input-output" + } + }, + "offset": { + "x": 0, + "y": 0 + }, + "scale": 1, + "selected": {} + }, + "confs": {}, + "container_source": "", + "dependencies": [], + "inputs": [ + "list_of_paths" + ], + "kernel": "IRkernel", + "node_id": "943e143", + "original_source": "\nfor (l in list_of_paths) {\n print(l)\n}", + "outputs": [], + "params": [], + "task_name": "loop-list-r-dev-user-name-at-domain-com", + "title": "loop list R-dev-user-name-at-domain-com", + "types": { + "list_of_paths": "list" + }, + "skip_exec": true +} diff --git a/vreapis/tests/resources/cells/print-msg-dev-user-name-at-domain-com.json b/vreapis/tests/resources/cells/print-msg-dev-user-name-at-domain-com.json new file mode 100644 index 00000000..7741afc3 --- /dev/null +++ b/vreapis/tests/resources/cells/print-msg-dev-user-name-at-domain-com.json @@ -0,0 +1,74 @@ +{ + "all_inputs": [ + "msg" + ], + "base_image": {"build":"ghcr.io/qcdis/naavre/naavre-cell-build-python:latest","runtime":"ghcr.io/qcdis/naavre/naavre-cell-runtime-python:latest"}, + "chart_obj": { + "hovered": {}, + "links": {}, + "nodes": { + "a74dc70": { + "id": "a74dc70", + "ports": { + "msg": { + "id": "msg", + "properties": { + "color": "#7bd279" + }, + "type": "left" + } + }, + "position": { + "x": 35, + "y": 15 + }, + "properties": { + "inputs": [ + "msg" + ], + "og_node_id": "a74dc70", + "outputs": [], + "params": [], + "title": "Print msg-dev-user-name-at-domain-com", + "vars": [ + { + "color": "#7bd279", + "direction": "input", + "name": "msg", + "type": "datatype" + } + ] + }, + "type": "input-output" + } + }, + "offset": { + "x": 0, + "y": 0 + }, + "scale": 1, + "selected": {} + }, + "confs": {}, + "container_source": "", + "dependencies": [], + "inputs": [ + "msg" + ], + "kernel": "ipython", + "node_id": "a74dc70", + "original_source": "print('msg:', str(msg))", + "outputs": [], + "params": [], + "task_name": "print-msg-dev-user-name-at-domain-com", + "title": "Print msg-dev-user-name-at-domain-com", + "types": { + "msg": "int" + }, + "example_inputs" : [ + "--id", + "0", + "--msg", + "1" + ] +} diff --git a/vreapis/tests/resources/cells/read-file-lines-dev-user-name-at-domain-com.json b/vreapis/tests/resources/cells/read-file-lines-dev-user-name-at-domain-com.json new file mode 100644 index 00000000..d1313f66 --- /dev/null +++ b/vreapis/tests/resources/cells/read-file-lines-dev-user-name-at-domain-com.json @@ -0,0 +1,110 @@ +{ + "all_inputs": [ + "file_path" + ], + "base_image": {"build":"ghcr.io/qcdis/naavre/naavre-cell-build-python:latest","runtime":"ghcr.io/qcdis/naavre/naavre-cell-runtime-python:latest"}, + "chart_obj": { + "hovered": {}, + "links": {}, + "nodes": { + "4deac70": { + "id": "4deac70", + "ports": { + "file_path": { + "id": "file_path", + "properties": { + "color": "#663a78" + }, + "type": "left" + }, + "lines": { + "id": "lines", + "properties": { + "color": "#ac535c" + }, + "type": "right" + } + }, + "position": { + "x": 35, + "y": 15 + }, + "properties": { + "inputs": [ + "file_path" + ], + "og_node_id": "4deac70", + "outputs": [ + "lines" + ], + "params": [], + "title": "read file lines-dev-user-name-at-domain-com", + "vars": [ + { + "color": "#663a78", + "direction": "input", + "name": "file_path", + "type": "datatype" + }, + { + "color": "#ac535c", + "direction": "output", + "name": "lines", + "type": "datatype" + } + ] + }, + "type": "input-output" + } + }, + "offset": { + "x": 0, + "y": 0 + }, + "scale": 1, + "selected": {} + }, + "confs": { + "conf_data_folder": "conf_data_folder = os.path.join('/tmp','data')" + }, + "container_source": "", + "dependencies": [ + { + "asname": null, + "module": "os.path", + "name": "isfile" + }, + { + "asname": null, + "module": "os.path", + "name": "join" + }, + { + "asname": null, + "module": "os", + "name": "listdir" + }, + { + "asname": null, + "module": "", + "name": "os" + } + ], + "inputs": [ + "file_path" + ], + "kernel": "ipython", + "node_id": "4deac70", + "original_source": "conf_data_folder = os.path.join('/tmp','data')\n\nonlyfiles = [f for f in listdir(conf_data_folder) if isfile(join(conf_data_folder, f))]\n\nprint(onlyfiles)\n\nf = open(file_path, 'r')\nlines = f.readlines()\nf.close()", + "outputs": [ + "lines" + ], + "params": [], + "task_name": "read-file-lines-dev-user-name-at-domain-com", + "title": "read file lines-dev-user-name-at-domain-com", + "types": { + "file_path": "str", + "lines": "list" + }, + "skip_exec": true +} diff --git a/vreapis/tests/resources/cells/read-file-lines-r-dev-user-name-at-domain-com.json b/vreapis/tests/resources/cells/read-file-lines-r-dev-user-name-at-domain-com.json new file mode 100644 index 00000000..347c164b --- /dev/null +++ b/vreapis/tests/resources/cells/read-file-lines-r-dev-user-name-at-domain-com.json @@ -0,0 +1,89 @@ +{ + "all_inputs": [ + "file_path" + ], + "base_image": {"build":"ghcr.io/qcdis/naavre/naavre-cell-build-r:latest","runtime":"ghcr.io/qcdis/naavre/naavre-cell-runtime-r:latest"}, + "chart_obj": { + "hovered": {}, + "links": {}, + "nodes": { + "47b2d97": { + "id": "47b2d97", + "ports": { + "file_path": { + "id": "file_path", + "properties": { + "color": "#663a78" + }, + "type": "left" + }, + "lines": { + "id": "lines", + "properties": { + "color": "#ac535c" + }, + "type": "right" + } + }, + "position": { + "x": 35, + "y": 15 + }, + "properties": { + "inputs": [ + "file_path" + ], + "og_node_id": "47b2d97", + "outputs": [ + "lines" + ], + "params": [], + "title": "read file lines R-dev-user-name-at-domain-com", + "vars": [ + { + "color": "#663a78", + "direction": "input", + "name": "file_path", + "type": "datatype" + }, + { + "color": "#ac535c", + "direction": "output", + "name": "lines", + "type": "datatype" + } + ] + }, + "type": "input-output" + } + }, + "offset": { + "x": 0, + "y": 0 + }, + "scale": 1, + "selected": {} + }, + "confs": { + "conf_data_folder": "conf_data_folder <- file.path('/tmp', 'data')" + }, + "container_source": "", + "dependencies": [], + "inputs": [ + "file_path" + ], + "kernel": "IRkernel", + "node_id": "47b2d97", + "original_source": "conf_data_folder <- file.path('/tmp', 'data')\n\nonlyfiles <- list.files(conf_data_folder, full.names = TRUE)\n\nprint(onlyfiles)\n\nf <- file(file_path, \"r\")\nlines <- readLines(f)\nclose(f)", + "outputs": [ + "lines" + ], + "params": [], + "task_name": "read-file-lines-r-dev-user-name-at-domain-com", + "title": "read file lines R-dev-user-name-at-domain-com", + "types": { + "file_path": "str", + "lines": "list" + }, + "skip_exec": true +} diff --git a/vreapis/tests/resources/cells/vol2bird-dev-user-name-at-domain-com.json b/vreapis/tests/resources/cells/vol2bird-dev-user-name-at-domain-com.json new file mode 100644 index 00000000..4fef8818 --- /dev/null +++ b/vreapis/tests/resources/cells/vol2bird-dev-user-name-at-domain-com.json @@ -0,0 +1,93 @@ +{ + "all_inputs": [ + "a" + ], + "base_image": {"build":"ghcr.io/qcdis/naavre/naavre-cell-build-vol2bird:latest","runtime":"ghcr.io/qcdis/naavre/naavre-cell-runtime-vol2bird:latest"}, + "chart_obj": { + "hovered": {}, + "links": {}, + "nodes": { + "ded63d7": { + "id": "ded63d7", + "ports": { + "a": { + "id": "a", + "properties": { + "color": "#77862d" + }, + "type": "left" + }, + "msg": { + "id": "msg", + "properties": { + "color": "#7bd279" + }, + "type": "right" + } + }, + "position": { + "x": 35, + "y": 15 + }, + "properties": { + "inputs": [ + "a" + ], + "og_node_id": "ded63d7", + "outputs": [ + "msg" + ], + "params": [], + "title": "vol2bird-dev-user-name-at-domain-com", + "vars": [ + { + "color": "#77862d", + "direction": "input", + "name": "a", + "type": "datatype" + }, + { + "color": "#7bd279", + "direction": "output", + "name": "msg", + "type": "datatype" + } + ] + }, + "type": "input-output" + } + }, + "offset": { + "x": 0, + "y": 0 + }, + "scale": 1, + "selected": {} + }, + "confs": {}, + "container_source": "", + "dependencies": [ + { + "asname": null, + "module": "", + "name": "os" + } + ], + "inputs": [ + "a" + ], + "kernel": "ipython", + "node_id": "ded63d7", + "original_source": "print(a)\ncmd = \"vol2bird --version\"\n\nmsg = os.system(cmd) # returns the exit code in unix\n", + "outputs": [ + "msg" + ], + "params": [], + "task_name": "vol2bird-dev-user-name-at-domain-com", + "title": "vol2bird-dev-user-name-at-domain-com", + "types": { + "a": "int", + "msg": "int" + }, + "skip_exec": true +} diff --git a/vreapis/tests/resources/notebooks/R-notebook.ipynb b/vreapis/tests/resources/notebooks/R-notebook.ipynb new file mode 100644 index 00000000..36ff7417 --- /dev/null +++ b/vreapis/tests/resources/notebooks/R-notebook.ipynb @@ -0,0 +1,74 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "59b2ee0b-958d-431f-a396-b4b34c77d540", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "# input\n", + "a = 2 \n", + "numbers <- c(a, 4, 6, 8, 10)" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "0a83d716-23c8-4eb2-b246-9a8f4cdf9ac2", + "metadata": {}, + "outputs": [], + "source": [ + "#csddfv\n", + "average <- mean(numbers)" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "73a1f742-2a8e-4c38-a1c0-5ff8bb372579", + "metadata": { + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[1] 6\n" + ] + } + ], + "source": [ + "print(average)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "528b97bb-396a-4520-8be0-9aeaed7e4715", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "R [conda env:jupyterlab] *", + "language": "R", + "name": "conda-env-jupyterlab-r" + }, + "language_info": { + "codemirror_mode": "r", + "file_extension": ".r", + "mimetype": "text/x-r-source", + "name": "R", + "pygments_lexer": "r", + "version": "4.2.3" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/vreapis/tests/resources/notebooks/TraitsComputation_NaaVRE.json b/vreapis/tests/resources/notebooks/TraitsComputation_NaaVRE.json new file mode 100644 index 00000000..ad056c0d --- /dev/null +++ b/vreapis/tests/resources/notebooks/TraitsComputation_NaaVRE.json @@ -0,0 +1,155 @@ +{ + "save": false, + "kernel": "IRkernel", + "cell_index": 1, + "notebook": { + "metadata": { + "kernelspec": { + "display_name": "R", + "language": "R", + "name": "ir" + }, + "language_info": { + "codemirror_mode": "r", + "file_extension": ".r", + "mimetype": "text/x-r-source", + "name": "R", + "pygments_lexer": "r", + "version": "4.2.3" + } + }, + "nbformat_minor": 5, + "nbformat": 4, + "cells": [ + { + "cell_type": "markdown", + "source": "### DESCRIPTION\nScript to compute demographic and morphological traits such as biovolume (\u00b5m3), surface area (\u00b5m2), surface/volume ratio, density (cells/L), cell carbon content (pg), total biovolume (\u00b5m\u00b3/L), total carbon content (pg/L). \n### INPUT VARIABLES\n- datain: the input file\n- CalcType: the computation type simplified or advanced\n- CompTraits: one or more among biovolume,totalbiovolume,density,surfacearea,surfacevolumeratio,cellcarboncontent,totalcarboncontent\n- CountingStrategy: one among density0,density1,density2,density3 \n\n### OUTPUT \nfile in .csv format, containing the original input data and the new calculated traits", + "metadata": {}, + "id": "prepared-shield" + }, + { + "cell_type": "code", + "source": "# input parameters\ninstall.packages(\"dplyr\",repos = \"http://cran.us.r-project.org\")\ndplyr = ''\nlibrary(dplyr)\n\n\nparam_datain = 'input/Phytoplankton__Progetto_Strategico_2009_2012_Australia.csv'\n\nparam_CalcType = 'advanced'\n\nparam_biovolume = 1 # if 1 it is calculated, if 0 it is not calculated\nparam_totalbiovolume = 1\nparam_density = 1\nparam_surfacearea = 1\nparam_surfacevolumeratio = 1\nparam_cellcarboncontent = 1\nparam_totalcarboncontent = 1\nparam_CountingStrategy = ''\n\nif (param_density==1) {param_CountingStrategy <- 'density0'}\n\n# read input cvs\ndf.datain=read.csv(param_datain,stringsAsFactors=FALSE,sep = \";\", dec = \".\")\nmeasurementremarks = tolower(df.datain$measurementremarks) # eliminate capital letters\ndf.datain$measurementremarks <- tolower(df.datain$measurementremarks) # eliminate capital letters\nindex = c(1:nrow(df.datain))\ndf.datain$index <- c(1:nrow(df.datain)) # needed to restore rows order later\n\n# read support cvs file\ndf.operator<-read.csv('input/2_FILEinformativo_OPERATORE.csv',stringsAsFactors=FALSE,sep = \";\", dec = \".\") ## load internal database \ndf.operator[df.operator==('no')]<-NA\ndf.operator[df.operator==('see note')]<-NA\n\n# merge dataframes\ndf.merged <- merge(x = df.datain, y = df.operator, by = c(\"scientificname\",\"measurementremarks\"), all.x = TRUE)\n\n# check if mandatory fields are present\ndiameterofsedimentationchamber = 'diameterofsedimentationchamber'\ndiameteroffieldofview = 'diameteroffieldofview'\ntransectcounting = 'transectcounting'\nnumberofcountedfields = 'numberofcountedfields'\nnumberoftransects = 'numberoftransects'\nsettlingvolume = 'settlingvolume'\ndilutionfactor = 'dilutionfactor'\n\nif(!'diameterofsedimentationchamber'%in%names(df.merged))df.merged$diameterofsedimentationchamber=NA\nif(!'diameteroffieldofview'%in%names(df.merged))df.merged$diameteroffieldofview=NA\nif(!'transectcounting'%in%names(df.merged))df.merged$transectcounting=NA\nif(!'numberofcountedfields'%in%names(df.merged))df.merged$numberofcountedfields=df.merged$transectcounting\nif(!'numberoftransects'%in%names(df.merged))df.merged$numberoftransects==df.merged$transectcounting\nif(!'settlingvolume'%in%names(df.merged))df.merged$settlingvolume=NA\nif(!'dilutionfactor'%in%names(df.merged))df.merged$dilutionfactor=1\n\n# save merged dataframe as csv\noutput_dfmerged = 'output/dfmerged.csv'\noutput_dfdatain = 'output/dfdatain.csv'\nwrite.table(df.merged,paste(output_dfmerged,sep=''),row.names=FALSE,sep = \";\",dec = \".\",quote=FALSE)\nwrite.table(df.datain,paste(output_dfdatain,sep=''),row.names=FALSE,sep = \";\",dec = \".\",quote=FALSE) ", + "metadata": { + "tags": [] + }, + "execution_count": 2, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": "Updating HTML index of packages in '.Library'\n\nMaking 'packages.html' ...\n done\n\n" + }, + { + "ename": "ERROR", + "evalue": "Error in `$<-.data.frame`(`*tmp*`, measurementremarks, value = character(0)): replacement has 0 rows, data has 6\n", + "output_type": "error", + "traceback": [ + "Error in `$<-.data.frame`(`*tmp*`, measurementremarks, value = character(0)): replacement has 0 rows, data has 6\nTraceback:\n", + "1. `$<-`(`*tmp*`, measurementremarks, value = character(0))", + "2. `$<-.data.frame`(`*tmp*`, measurementremarks, value = character(0))", + "3. stop(sprintf(ngettext(N, \"replacement has %d row, data has %d\", \n . \"replacement has %d rows, data has %d\"), N, nrows), domain = NA)" + ] + } + ], + "id": "steady-metabolism" + }, + { + "cell_type": "code", + "source": "#Calculate \n\ndf.merged=read.csv(output_dfmerged,stringsAsFactors=FALSE,sep = \";\", dec = \".\")\nCalcType=param_CalcType\n\ndf.temp = '' # Because the Component cannot identify '<-' we need to initialize these variables so they are not showing up as inputs\nmd.form = ''\nmissingdimension = ''\ndf.temp2 = ''\nindex = ''\nformulaformissingdimensionsimplified = '' \nmd = ''\ndf.merged.concat = '' \nmd.formulas = ''\nmissingdimensionsimplified = ''\nformulaformissingdimension = '' \n\nif(CalcType=='advanced'){\n df.merged.concat <- df.merged[is.na(df.merged$formulaformissingdimension),]\n md.formulas = ''\n md.formulas <- unique(df.merged[!is.na(df.merged$formulaformissingdimension),]$formulaformissingdimension)\n for(md.form in md.formulas){\n df.temp <- subset(df.merged,formulaformissingdimension==md.form)\n for(md in unique(df.temp$missingdimension)){\n df.temp2 <- df.temp[df.temp$missingdimension==md,]\n df.temp2[[md]] <- round(with(df.temp2,eval(parse(text=md.form))),2)\n df.merged.concat <- rbind(df.merged.concat,df.temp2)\n }\n }\n df.merged.concat = ''\n df.merged.concat <- df.merged.concat[order(df.merged.concat$index),]\n df.merged = '' \n df.merged <- df.merged.concat\n} else if(CalcType=='simplified'){\n formulaformissingdimensionsimplified = ''\n df.merged.concat <- df.merged[is.na(df.merged$formulaformissingdimensionsimplified),]\n md.formulas <- unique(df.merged[!is.na(df.merged$formulaformissingdimensionsimplified),]$formulaformissingdimensionsimplified)\n for(md.form in md.formulas){\n df.temp <- subset(df.merged,formulaformissingdimensionsimplified==md.form)\n for(md in unique(df.temp$missingdimensionsimplified)){\n df.temp2 = '' \n df.temp2 <- df.temp[df.temp$missingdimensionsimplified==md,]\n df.temp2[[md]] <- round(with(df.temp2,eval(parse(text=md.form))),2)\n df.merged.concat <- rbind(df.merged.concat,df.temp2)\n }\n }\n df.merged.concat <- df.merged.concat[order(df.merged.concat$index),]\n df.merged <- df.merged.concat\n}\n \nwrite.table(df.merged,paste(output_dfmerged,sep=''),row.names=FALSE,sep = \";\",dec = \".\",quote=FALSE) \n", + "metadata": { + "tags": [] + }, + "execution_count": null, + "outputs": [], + "id": "reflected-cardiff" + }, + { + "cell_type": "code", + "source": "df.merged=read.csv('~/Unisalento/Lifewatch/Phyto_VRE/Script_R/Traits_Computation/dfmerged.csv',stringsAsFactors=FALSE,sep = \";\", dec = \".\")\nTraitBiovolume=param_biovolume\nCalcType=param_CalcType\n\nif(TraitBiovolume==1){\n \n if(CalcType=='advanced'){\n df.merged$biovolume <- rep(NA,length=nrow(df.merged))\n df.merged.concat <- df.merged[is.na(df.merged$formulaforbiovolume),]\n bv.formulas <- unique(df.merged[!is.na(df.merged$formulaforbiovolume),]$formulaforbiovolume)\n for(bv.form in bv.formulas){\n df.temp <- subset(df.merged,formulaforbiovolume==bv.form)\n df.temp$biovolume <- round(with(df.temp,eval(parse(text=bv.form))),2)\n df.merged.concat <- rbind(df.merged.concat,df.temp)\n }\n df.merged.concat <- df.merged.concat[order(df.merged.concat$index),]\n df.merged <- df.merged.concat\n BV_calc = df.merged$biovolume\n }\n else if(CalcType=='simplified'){\n df.merged$biovolume <- rep(NA,length=nrow(df.merged))\n df.merged.concat <- df.merged[is.na(df.merged$formulaforbiovolumesimplified),]\n bv.formulas <- unique(df.merged[!is.na(df.merged$formulaforbiovolumesimplified),]$formulaforbiovolumesimplified)\n for(bv.form in bv.formulas){\n df.temp <- subset(df.merged,formulaforbiovolumesimplified==bv.form)\n df.temp$biovolume <- round(with(df.temp,eval(parse(text=bv.form))),2)\n df.merged.concat <- rbind(df.merged.concat,df.temp)\n }\n df.merged.concat <- df.merged.concat[order(df.merged.concat$index),]\n df.merged <- df.merged.concat\n BV_calc = df.merged$biovolume\n }\n} \n\nwrite.table(df.merged,paste('~/Unisalento/Lifewatch/Phyto_VRE/Script_R/Traits_Computation/dfmerged.csv',sep=''),row.names=FALSE,sep = \";\",dec = \".\",quote=FALSE)", + "metadata": {}, + "execution_count": 16, + "outputs": [], + "id": "induced-trash" + }, + { + "cell_type": "code", + "source": "df.merged=read.csv('~/Unisalento/Lifewatch/Phyto_VRE/Script_R/Traits_Computation/dfmerged.csv',stringsAsFactors=FALSE,sep = \";\", dec = \".\")\n\nTraitCellcarboncontent=param_cellcarboncontent\nTraitBiovolume=param_biovolume\n\nif(TraitCellcarboncontent==1){\n \n df.merged$cellcarboncontent <- rep(NA,length=nrow(df.merged))\n if(TraitBiovolume==1){\n df.merged.concat <- df.merged[is.na(df.merged$biovolume),]\n df.cc <- df.merged[!is.na(df.merged$biovolume),]\n df.cc1 <- subset(df.cc,biovolume <= 3000)\n df.cc2 <- subset(df.cc,biovolume > 3000)\n cc.formulas1 <- unique(df.merged[!is.na(df.merged$formulaforweight1),]$formulaforweight1)\n for(cc.form in cc.formulas1){\n df.temp <- subset(df.cc1,formulaforweight1==cc.form)\n df.temp$cellcarboncontent <- round(with(df.temp,eval(parse(text=tolower(cc.form)))),2)\n df.merged.concat <- rbind(df.merged.concat,df.temp)\n }\n cc.formulas2 <- unique(df.merged[!is.na(df.merged$formulaforweight2),]$formulaforweight2)\n for(cc.form in cc.formulas2){\n df.temp <- subset(df.cc2,formulaforweight2==cc.form)\n df.temp$cellcarboncontent <- round(with(df.temp,eval(parse(text=tolower(cc.form)))),2)\n df.merged.concat <- rbind(df.merged.concat,df.temp)\n }\n df.merged.concat <- df.merged.concat[order(df.merged.concat$index),]\n df.merged <- df.merged.concat\n CC_calc = df.merged$cellcarboncontent\n }\n}\n \nwrite.table(df.merged,paste('~/Unisalento/Lifewatch/Phyto_VRE/Script_R/Traits_Computation/dfmerged.csv',sep=''),row.names=FALSE,sep = \";\",dec = \".\",quote=FALSE) ", + "metadata": {}, + "execution_count": 17, + "outputs": [], + "id": "peripheral-contest" + }, + { + "cell_type": "code", + "source": "df.merged=read.csv('~/Unisalento/Lifewatch/Phyto_VRE/Script_R/Traits_Computation/dfmerged.csv',stringsAsFactors=FALSE,sep = \";\", dec = \".\")\n\nTraitDensity=param_density\nCountingStrategy=param_CountingStrategy\n\nif(TraitDensity==1){\n df.merged$density <- rep(NA,length=nrow(df.merged))\n # default method to calculate the density\n if(CountingStrategy=='density0'){ \n df.merged.concat <- df.merged[(is.na(df.merged$volumeofsedimentationchamber)) & (is.na(df.merged$transectcounting)),]\n df.temp <- df.merged[!is.na(df.merged$volumeofsedimentationchamber) & !is.na(df.merged$transectcounting),]\n df.temp1 <- subset(df.temp,volumeofsedimentationchamber <= 5)\n df.temp1$density <- df.temp1$organismquantity/df.temp1$transectcounting*1000/0.001979\n df.merged.concat <- rbind(df.merged.concat,df.temp1)\n df.temp2 <- subset(df.temp,(volumeofsedimentationchamber > 5) & (volumeofsedimentationchamber <= 10))\n df.temp2$density <- df.temp2$organismquantity/df.temp2$transectcounting*1000/0.00365\n df.merged.concat <- rbind(df.merged.concat,df.temp2)\n df.temp3 <- subset(df.temp,(volumeofsedimentationchamber > 10) & (volumeofsedimentationchamber <= 25))\n df.temp3$density <- df.temp3$organismquantity/df.temp3$transectcounting*1000/0.010555\n df.merged.concat <- rbind(df.merged.concat,df.temp3)\n df.temp4 <- subset(df.temp,(volumeofsedimentationchamber > 25) & (volumeofsedimentationchamber <= 50))\n df.temp4$density <- df.temp4$organismquantity/df.temp4$transectcounting*1000/0.021703\n df.merged.concat <- rbind(df.merged.concat,df.temp4)\n df.temp5 <- subset(df.temp,volumeofsedimentationchamber > 50)\n df.temp5$density <- df.temp5$organismquantity/df.temp5$transectcounting*1000/0.041598\n df.merged.concat <- rbind(df.merged.concat,df.temp5)\n df.merged.concat <- df.merged.concat[order(df.merged.concat$index),]\n df.merged <- df.merged.concat\n D_calc <- round(df.merged$density,2)\n }\n # counts per random field\n else if(CountingStrategy=='density1'){\n df.merged$areaofsedimentationchamber <- ((df.merged$diameterofsedimentationchamber/2)^2)*pi\n df.merged$areaofcountingfield <- ((df.merged$diameteroffieldofview/2)^2)*pi\n df.merged$density <- round(df.merged$organismquantity*1000*df.merged$areaofsedimentationchamber/df.merged$numberofcountedfields*df.merged$areaofcountingfield*df.merged$settlingvolume,2)\n }\n # counts per diameter transects\n else if(CountingStrategy=='density2'){\n df.merged$density <- round(((df.merged$organismquantity/df.merged$numberoftransects)*(pi/4)*(df.merged$diameterofsedimentationchamber/df.merged$diameteroffieldofview))*1000/df.merged$settlingvolume,2)\n }\n # counting method for whole chamber\n else if(CountingStrategy=='density3'){\n df.merged$density <- round((df.merged$organismquantity*1000)/df.merged$settlingvolume,2)\n }\n D_calc = df.merged$density/df.merged$dilutionfactor\n}\n \n \nwrite.table(df.merged,paste('~/Unisalento/Lifewatch/Phyto_VRE/Script_R/Traits_Computation/dfmerged.csv',sep=''),row.names=FALSE,sep = \";\",dec = \".\",quote=FALSE)\n", + "metadata": {}, + "execution_count": 18, + "outputs": [], + "id": "cellular-constraint" + }, + { + "cell_type": "code", + "source": "df.merged=read.csv('~/Unisalento/Lifewatch/Phyto_VRE/Script_R/Traits_Computation/dfmerged.csv',stringsAsFactors=FALSE,sep = \";\", dec = \".\")\n\nTraitTotalbiovolume=param_totalbiovolume\nTraitDensity=param_density\nTraitBiovolume=param_biovolume\n\nif(TraitTotalbiovolume==1){\n if((TraitDensity==0) & (!'density'%in%names(df.merged))) df.merged$density<-NA\n if((TraitBiovolume==0) & (!'biovolume'%in%names(df.merged))) df.merged$biovolume<-NA\n TBV_calc = round(df.merged$density*df.merged$biovolume,2)\n}\n\nwrite.table(df.merged,paste('~/Unisalento/Lifewatch/Phyto_VRE/Script_R/Traits_Computation/dfmerged.csv',sep=''),row.names=FALSE,sep = \";\",dec = \".\",quote=FALSE)\n", + "metadata": {}, + "execution_count": 19, + "outputs": [], + "id": "intense-muscle" + }, + { + "cell_type": "code", + "source": "df.merged=read.csv('~/Unisalento/Lifewatch/Phyto_VRE/Script_R/Traits_Computation/dfmerged.csv',stringsAsFactors=FALSE,sep = \";\", dec = \".\")\n\nTraitSurfacearea=param_surfacearea\nCalcType=param_CalcType\n\nif(TraitSurfacearea==1){\n if(CalcType=='advanced'){\n df.merged$surfacearea <- rep(NA,length=nrow(df.merged))\n df.merged.concat <- df.merged[is.na(df.merged$formulaforsurface),]\n sa.formulas <- unique(df.merged[!is.na(df.merged$formulaforsurface),]$formulaforsurface)\n for(sa.form in sa.formulas){\n df.temp <- subset(df.merged,formulaforsurface==sa.form)\n df.temp$surfacearea <- round(with(df.temp,eval(parse(text=sa.form))),2)\n df.merged.concat <- rbind(df.merged.concat,df.temp)\n }\n df.merged.concat <- df.merged.concat[order(df.merged.concat$index),]\n df.merged <- df.merged.concat\n SA_calc <- df.merged$surfacearea\n }\n else if(CalcType=='simplified'){\n df.merged$surfacearea <- rep(NA,length=nrow(df.merged))\n df.merged.concat <- df.merged[is.na(df.merged$formulaforsurfacesimplified),]\n sa.formulas <- unique(df.merged[!is.na(df.merged$formulaforsurfacesimplified),]$formulaforsurfacesimplified)\n for(sa.form in sa.formulas){\n df.temp <- subset(df.merged,formulaforsurfacesimplified==sa.form)\n df.temp$surfacearea <- round(with(df.temp,eval(parse(text=sa.form))),2)\n df.merged.concat <- rbind(df.merged.concat,df.temp)\n }\n df.merged.concat <- df.merged.concat[order(df.merged.concat$index),]\n df.merged <- df.merged.concat\n SA_calc <- df.merged$surfacearea\n }\n}\n\nwrite.table(df.merged,paste('~/Unisalento/Lifewatch/Phyto_VRE/Script_R/Traits_Computation/dfmerged.csv',sep=''),row.names=FALSE,sep = \";\",dec = \".\",quote=FALSE)\n", + "metadata": {}, + "execution_count": 20, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": "Warning message in sqrt((h^2) - (d^2)):\n\"Si \u00e8 prodotto un NaN\"\nWarning message in sqrt((h^2) - (d^2)):\n\"Si \u00e8 prodotto un NaN\"\nWarning message in sqrt(4 * h^2 - ((b + c)^2)):\n\"Si \u00e8 prodotto un NaN\"\nWarning message in sqrt(4 * h^2 - ((b + c)^2)):\n\"Si \u00e8 prodotto un NaN\"\n" + } + ], + "id": "shared-explorer" + }, + { + "cell_type": "code", + "source": "df.merged=read.csv('~/Unisalento/Lifewatch/Phyto_VRE/Script_R/Traits_Computation/dfmerged.csv',stringsAsFactors=FALSE,sep = \";\", dec = \".\")\n\nTraitSurfacevolumeratio=param_surfacevolumeratio\nTraitSurfacearea=param_surfacearea\nTraitBiovolume=param_biovolume\n\nif(TraitSurfacevolumeratio==1){\n if((TraitSurfacearea==0) & (!'surfacearea'%in%names(df.merged))) df.merged$surfacearea<-NA\n if((TraitBiovolume==0) & (!'biovolume'%in%names(df.merged))) df.merged$biovolume<-NA\n SVR_calc<-round(df.merged$surfacearea/df.merged$biovolume,2)\n}\n\nwrite.table(df.merged,paste('~/Unisalento/Lifewatch/Phyto_VRE/Script_R/Traits_Computation/dfmerged.csv',sep=''),row.names=FALSE,sep = \";\",dec = \".\",quote=FALSE)\n", + "metadata": {}, + "execution_count": 21, + "outputs": [], + "id": "affected-movement" + }, + { + "cell_type": "code", + "source": "df.merged=read.csv('~/Unisalento/Lifewatch/Phyto_VRE/Script_R/Traits_Computation/dfmerged.csv',stringsAsFactors=FALSE,sep = \";\", dec = \".\")\n\nTraitTotalcarboncontent=param_totalcarboncontent\nTraitDensity=param_density\nTraitCellcarboncontent=param_cellcarboncontent\n\nif(TraitTotalcarboncontent==1){\n if((TraitDensity==0) & (!'density'%in%names(df.merged))) df.merged$density<-NA\n if((TraitCellcarboncontent==0) & (!'cellcarboncontent'%in%names(df.merged))) df.merged$cellcarboncontent<-NA\n TCC_calc<-round(df.merged$density*df.merged$cellcarboncontent,2)\n}\n\nwrite.table(df.merged,paste('~/Unisalento/Lifewatch/Phyto_VRE/Script_R/Traits_Computation/dfmerged.csv',sep=''),row.names=FALSE,sep = \";\",dec = \".\",quote=FALSE)\n", + "metadata": {}, + "execution_count": 22, + "outputs": [], + "id": "threaded-english" + }, + { + "cell_type": "code", + "source": "df.datain=read.csv('~/Unisalento/Lifewatch/Phyto_VRE/Script_R/Traits_Computation/dfdatain.csv',stringsAsFactors=FALSE,sep = \";\", dec = \".\")\n\nBV=TraitBiovolume \nTBV=TraitTotalbiovolume\nD=TraitDensity\nSA=TraitSurfacearea\nSVR=TraitSurfacevolumeratio\nCC=TraitCellcarboncontent\nTCC=TraitTotalcarboncontent\n\nif(BV==1) {\n BV_column=BV_calc\n if('biovolume'%in%names(df.datain)) df.datain<-subset(df.datain,select=-biovolume) # drop column if already present\n df.datain$biovolume <- BV_column # write column with the results at the end of the dataframe\n }\nif(CC==1) {\n CC_column=CC_calc\n if('cellcarboncontent'%in%names(df.datain)) df.datain<-subset(df.datain,select=-cellcarboncontent)\n df.datain$cellcarboncontent <- CC_column\n }\nif(D==1) {\n D_column=D_calc\n df.datain$density <- D_column\n }\nif(TBV==1) {\n TBV_column=TBV_calc\n df.datain$totalbiovolume <- TBV_column\n }\nif(SA==1) {\n SA_column=SA_calc\n if('surfacearea'%in%names(df.datain)) df.datain<-subset(df.datain,select=-surfacearea)\n df.datain$surfacearea <- SA_column\n }\nif(SVR==1) {\n SVR_column=SVR_calc\n df.datain$surfacevolumeratio <- SVR_column\n }\nif(TCC==1) {\n TCC_column=TCC_calc\n df.datain$totalcarboncontent <- TCC_column\n }\n\ndf.datain <- subset(df.datain,select = -index)\n\nwrite.table(df.datain,paste('~/Unisalento/Lifewatch/Phyto_VRE/Script_R/Traits_Computation/Output/TraitsOutput.csv',sep=''),row.names=FALSE,sep = \";\",dec = \".\",quote=FALSE) ", + "metadata": {}, + "execution_count": 23, + "outputs": [], + "id": "congressional-observation" + }, + { + "cell_type": "code", + "source": "", + "metadata": {}, + "execution_count": null, + "outputs": [], + "id": "refined-calibration" + }, + { + "cell_type": "code", + "source": "", + "metadata": {}, + "execution_count": null, + "outputs": [], + "id": "closing-seating" + } + ] + } +} diff --git a/vreapis/tests/resources/notebooks/dependency_with_submodule.notebook.json b/vreapis/tests/resources/notebooks/dependency_with_submodule.notebook.json new file mode 100644 index 00000000..252667a3 --- /dev/null +++ b/vreapis/tests/resources/notebooks/dependency_with_submodule.notebook.json @@ -0,0 +1,91 @@ +{ + "save": false, + "kernel": "ipython", + "cell_index": 1, + "notebook": { + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.7" + } + }, + "nbformat_minor": 5, + "nbformat": 4, + "cells": [ + { + "cell_type": "code", + "source": "!pip install numpy", + "metadata": { + "trusted": true + }, + "execution_count": 10, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": "Collecting torch\n Using cached torch-2.2.0-cp311-cp311-manylinux1_x86_64.whl.metadata (25 kB)\nCollecting filelock (from torch)\n Using cached filelock-3.13.1-py3-none-any.whl.metadata (2.8 kB)\nRequirement already satisfied: typing-extensions>=4.8.0 in /home/alogo/miniforge3/envs/jupyterlab/lib/python3.11/site-packages (from torch) (4.9.0)\nCollecting sympy (from torch)\n Using cached sympy-1.12-py3-none-any.whl.metadata (12 kB)\nRequirement already satisfied: networkx in /home/alogo/miniforge3/envs/jupyterlab/lib/python3.11/site-packages (from torch) (3.1)\nRequirement already satisfied: jinja2 in /home/alogo/miniforge3/envs/jupyterlab/lib/python3.11/site-packages (from torch) (3.1.3)\nCollecting fsspec (from torch)\n Using cached fsspec-2024.2.0-py3-none-any.whl.metadata (6.8 kB)\nCollecting nvidia-cuda-nvrtc-cu12==12.1.105 (from torch)\n Using cached nvidia_cuda_nvrtc_cu12-12.1.105-py3-none-manylinux1_x86_64.whl.metadata (1.5 kB)\nCollecting nvidia-cuda-runtime-cu12==12.1.105 (from torch)\n Using cached nvidia_cuda_runtime_cu12-12.1.105-py3-none-manylinux1_x86_64.whl.metadata (1.5 kB)\nCollecting nvidia-cuda-cupti-cu12==12.1.105 (from torch)\n Using cached nvidia_cuda_cupti_cu12-12.1.105-py3-none-manylinux1_x86_64.whl.metadata (1.6 kB)\nCollecting nvidia-cudnn-cu12==8.9.2.26 (from torch)\n Using cached nvidia_cudnn_cu12-8.9.2.26-py3-none-manylinux1_x86_64.whl.metadata (1.6 kB)\nCollecting nvidia-cublas-cu12==12.1.3.1 (from torch)\n Using cached nvidia_cublas_cu12-12.1.3.1-py3-none-manylinux1_x86_64.whl.metadata (1.5 kB)\nCollecting nvidia-cufft-cu12==11.0.2.54 (from torch)\n Using cached nvidia_cufft_cu12-11.0.2.54-py3-none-manylinux1_x86_64.whl.metadata (1.5 kB)\nCollecting nvidia-curand-cu12==10.3.2.106 (from torch)\n Using cached nvidia_curand_cu12-10.3.2.106-py3-none-manylinux1_x86_64.whl.metadata (1.5 kB)\nCollecting nvidia-cusolver-cu12==11.4.5.107 (from torch)\n Using cached nvidia_cusolver_cu12-11.4.5.107-py3-none-manylinux1_x86_64.whl.metadata (1.6 kB)\nCollecting nvidia-cusparse-cu12==12.1.0.106 (from torch)\n Using cached nvidia_cusparse_cu12-12.1.0.106-py3-none-manylinux1_x86_64.whl.metadata (1.6 kB)\nCollecting nvidia-nccl-cu12==2.19.3 (from torch)\n Using cached nvidia_nccl_cu12-2.19.3-py3-none-manylinux1_x86_64.whl.metadata (1.8 kB)\nCollecting nvidia-nvtx-cu12==12.1.105 (from torch)\n Using cached nvidia_nvtx_cu12-12.1.105-py3-none-manylinux1_x86_64.whl.metadata (1.7 kB)\nCollecting triton==2.2.0 (from torch)\n Using cached triton-2.2.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (1.4 kB)\nCollecting nvidia-nvjitlink-cu12 (from nvidia-cusolver-cu12==11.4.5.107->torch)\n Using cached nvidia_nvjitlink_cu12-12.3.101-py3-none-manylinux1_x86_64.whl.metadata (1.5 kB)\nRequirement already satisfied: MarkupSafe>=2.0 in /home/alogo/miniforge3/envs/jupyterlab/lib/python3.11/site-packages (from jinja2->torch) (2.1.4)\nCollecting mpmath>=0.19 (from sympy->torch)\n Using cached mpmath-1.3.0-py3-none-any.whl.metadata (8.6 kB)\nUsing cached torch-2.2.0-cp311-cp311-manylinux1_x86_64.whl (755.5 MB)\nUsing cached nvidia_cublas_cu12-12.1.3.1-py3-none-manylinux1_x86_64.whl (410.6 MB)\nUsing cached nvidia_cuda_cupti_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (14.1 MB)\nUsing cached nvidia_cuda_nvrtc_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (23.7 MB)\nUsing cached nvidia_cuda_runtime_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (823 kB)\nUsing cached nvidia_cudnn_cu12-8.9.2.26-py3-none-manylinux1_x86_64.whl (731.7 MB)\nUsing cached nvidia_cufft_cu12-11.0.2.54-py3-none-manylinux1_x86_64.whl (121.6 MB)\nUsing cached nvidia_curand_cu12-10.3.2.106-py3-none-manylinux1_x86_64.whl (56.5 MB)\nDownloading nvidia_cusolver_cu12-11.4.5.107-py3-none-manylinux1_x86_64.whl (124.2 MB)\n\u001b[2K \u001b[38;2;114;156;31m\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u001b[0m \u001b[32m124.2/124.2 MB\u001b[0m \u001b[31m4.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0mm eta \u001b[36m0:00:01\u001b[0m[36m0:00:01\u001b[0m\n\u001b[?25hDownloading nvidia_cusparse_cu12-12.1.0.106-py3-none-manylinux1_x86_64.whl (196.0 MB)\n\u001b[2K \u001b[38;2;114;156;31m\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u001b[0m \u001b[32m196.0/196.0 MB\u001b[0m \u001b[31m4.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0mm eta \u001b[36m0:00:01\u001b[0m[36m0:00:01\u001b[0mm\n\u001b[?25hDownloading nvidia_nccl_cu12-2.19.3-py3-none-manylinux1_x86_64.whl (166.0 MB)\n\u001b[2K \u001b[38;2;114;156;31m\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u001b[0m \u001b[32m166.0/166.0 MB\u001b[0m \u001b[31m3.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0mm eta \u001b[36m0:00:01\u001b[0m[36m0:00:02\u001b[0m\n\u001b[?25hDownloading nvidia_nvtx_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (99 kB)\n\u001b[2K \u001b[38;2;114;156;31m\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u001b[0m \u001b[32m99.1/99.1 kB\u001b[0m \u001b[31m2.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n\u001b[?25hDownloading triton-2.2.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (167.9 MB)\n\u001b[2K \u001b[38;2;114;156;31m\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u001b[0m \u001b[32m167.9/167.9 MB\u001b[0m \u001b[31m2.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0mm eta \u001b[36m0:00:01\u001b[0m[36m0:00:02\u001b[0m\n\u001b[?25hDownloading filelock-3.13.1-py3-none-any.whl (11 kB)\nDownloading fsspec-2024.2.0-py3-none-any.whl (170 kB)\n\u001b[2K \u001b[38;2;114;156;31m\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u001b[0m \u001b[32m170.9/170.9 kB\u001b[0m \u001b[31m1.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m MB/s\u001b[0m eta \u001b[36m0:00:01\u001b[0m:01\u001b[0m\n\u001b[?25hDownloading sympy-1.12-py3-none-any.whl (5.7 MB)\n\u001b[2K \u001b[38;2;114;156;31m\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u001b[0m \u001b[32m5.7/5.7 MB\u001b[0m \u001b[31m2.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0mm eta \u001b[36m0:00:01\u001b[0m[36m0:00:01\u001b[0m\n\u001b[?25hDownloading mpmath-1.3.0-py3-none-any.whl (536 kB)\n\u001b[2K \u001b[38;2;114;156;31m\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u001b[0m \u001b[32m536.2/536.2 kB\u001b[0m \u001b[31m2.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0mm eta \u001b[36m0:00:01\u001b[0m0:01\u001b[0m:01\u001b[0m\n\u001b[?25hDownloading nvidia_nvjitlink_cu12-12.3.101-py3-none-manylinux1_x86_64.whl (20.5 MB)\n\u001b[2K \u001b[38;2;114;156;31m\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u001b[0m \u001b[32m20.5/20.5 MB\u001b[0m \u001b[31m3.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0mm eta \u001b[36m0:00:01\u001b[0m[36m0:00:01\u001b[0m\n\u001b[?25hInstalling collected packages: mpmath, sympy, nvidia-nvtx-cu12, nvidia-nvjitlink-cu12, nvidia-nccl-cu12, nvidia-curand-cu12, nvidia-cufft-cu12, nvidia-cuda-runtime-cu12, nvidia-cuda-nvrtc-cu12, nvidia-cuda-cupti-cu12, nvidia-cublas-cu12, fsspec, filelock, triton, nvidia-cusparse-cu12, nvidia-cudnn-cu12, nvidia-cusolver-cu12, torch\nSuccessfully installed filelock-3.13.1 fsspec-2024.2.0 mpmath-1.3.0 nvidia-cublas-cu12-12.1.3.1 nvidia-cuda-cupti-cu12-12.1.105 nvidia-cuda-nvrtc-cu12-12.1.105 nvidia-cuda-runtime-cu12-12.1.105 nvidia-cudnn-cu12-8.9.2.26 nvidia-cufft-cu12-11.0.2.54 nvidia-curand-cu12-10.3.2.106 nvidia-cusolver-cu12-11.4.5.107 nvidia-cusparse-cu12-12.1.0.106 nvidia-nccl-cu12-2.19.3 nvidia-nvjitlink-cu12-12.3.101 nvidia-nvtx-cu12-12.1.105 sympy-1.12 torch-2.2.0 triton-2.2.0\n" + } + ], + "id": "4f10f166-b8cf-4f9f-b90f-270bf4522d2d" + }, + { + "cell_type": "code", + "source": "#Dependency resolution submodule\nimport matplotlib.pyplot as plt\n\n# Create some sample data\nx_values = [1, 2, 3, 4, 5]\ny_values = [2, 4, 6, 8, 10]\n\n# Plot the data\nplt.plot(x_values, y_values, label='Linear Function')\n\n# Add labels and a title\nplt.xlabel('X-axis')\nplt.ylabel('Y-axis')\nplt.title('Simple Linear Plot')\n\n# Add a legend\nplt.legend()\n\n# Display the plot\nplt.show()", + "metadata": { + "trusted": true + }, + "execution_count": 30, + "outputs": [ + { + "output_type": "display_data", + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjIAAAHHCAYAAACle7JuAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/H5lhTAAAACXBIWXMAAA9hAAAPYQGoP6dpAABX20lEQVR4nO3dd3gU5eL+//em90AglEASIBBKaKGINFHpIopKxyPWDypVQBQLRVQ4giIQxHIUPEe6AjYQRQSlSUkCobfQQ4ckJKTtzu8Pv+RngECCSWY3uV/Xtdflzs7s3k8msnfmmZ21GIZhICIiIuKAnMwOICIiInKnVGRERETEYanIiIiIiMNSkRERERGHpSIjIiIiDktFRkRERByWioyIiIg4LBUZERERcVgqMiIiIuKwVGREioEqVarw5JNPmvLa48aNw2KxFOlrHjlyBIvFwpw5c4r0dYvanDlzsFgsHDlyxOwoInZLRUbEjsXFxdG9e3dCQ0Px8PCgUqVKtG/fnhkzZpgdrdBce/PeunWr2VEKzbXyd+3m5eVFnTp1eOONN0hKSiqQ15g3bx4ffvhhgTyXiD1zMTuAiNzchg0buO+++wgJCeG5556jQoUKHD9+nE2bNjFt2jQGDx6cve6+fftwcio5f5eEhoZy9epVXF1dzY7yj8yaNQsfHx+uXLnCzz//zDvvvMPq1atZv379Pz7KNW/ePHbu3MmwYcMKJqyInVKREbFT77zzDv7+/mzZsoVSpUrleOzs2bM57ru7uxdhMvNZLBY8PDzMjnFLqampeHl53XKd7t27U7ZsWQCef/55HnvsMZYsWcKmTZto3rx5UcQUcXgl5084EQdz6NAhIiIibigxAOXKlctx//pzZK5Nz6xbt44hQ4YQGBhIqVKlGDBgABkZGVy+fJknnniC0qVLU7p0aUaNGoVhGNnbXzsHZcqUKUydOpXQ0FA8PT1p06YNO3fuzFP+r776isaNG+Pp6UlAQAC9e/fm+PHjd/SzuN7NzpF58skn8fHx4eTJk3Tr1g0fHx8CAwMZOXIkVqs1x/Y2m40PP/yQiIgIPDw8KF++PAMGDODSpUs51vv222/p0qULQUFBuLu7ExYWxoQJE254vnvvvZe6deuybds27rnnHry8vHjttdfyPa77778fgPj4+Fuu99FHHxEREYG7uztBQUEMHDiQy5cv58jz448/cvTo0ezpqypVquQ7j4gj0BEZETsVGhrKxo0b2blzJ3Xr1r2j5xg8eDAVKlRg/PjxbNq0iU8//ZRSpUqxYcMGQkJCePfdd1m+fDmTJ0+mbt26PPHEEzm2/+9//0tycjIDBw4kLS2NadOmcf/99xMXF0f58uVzfd133nmHN998k549e/Lss89y7tw5ZsyYwT333ENMTMxNy1lBsFqtdOzYkWbNmjFlyhRWrVrF+++/T1hYGC+88EL2egMGDGDOnDk89dRTDBkyhPj4eKKiooiJiWH9+vXZU1Zz5szBx8eH4cOH4+Pjw+rVqxkzZgxJSUlMnjw5x2tfuHCBzp0707t3bx5//PFb/nxyc+jQIQDKlCmT6zrjxo1j/PjxtGvXjhdeeIF9+/Yxa9YstmzZkp399ddfJzExkRMnTjB16lQAfHx88p1HxCEYImKXfv75Z8PZ2dlwdnY2mjdvbowaNcpYuXKlkZGRccO6oaGhRv/+/bPvz5492wCMjh07GjabLXt58+bNDYvFYjz//PPZy7KysozKlSsbbdq0yV4WHx9vAIanp6dx4sSJ7OV//vmnARgvvfRS9rKxY8caf/+n5MiRI4azs7Pxzjvv5MgYFxdnuLi43LD8eteyb9myJdd1ruWbPXt29rL+/fsbgPHWW2/lWDcyMtJo3Lhx9v0//vjDAIy5c+fmWO+nn366YXlqauoNrz1gwADDy8vLSEtLy17Wpk0bAzA+/vjjW47tmms/s3379hnnzp0z4uPjjU8++cRwd3c3ypcvb6SkpOT4WcTHxxuGYRhnz5413NzcjA4dOhhWqzX7+aKiogzA+OKLL7KXdenSxQgNDc1THhFHpqklETvVvn17Nm7cyEMPPcT27dt577336NixI5UqVeK7777L03M888wzOU4abdasGYZh8Mwzz2Qvc3Z2pkmTJhw+fPiG7bt160alSpWy79911100a9aM5cuX5/qaS5YswWaz0bNnT86fP599q1ChAjVq1OC3337LU/Y79fzzz+e437p16xxjW7x4Mf7+/rRv3z5HvsaNG+Pj45Mjn6enZ/Z/Jycnc/78eVq3bk1qaip79+7N8Tru7u489dRT+cpas2ZNAgMDqVq1KgMGDKB69er8+OOPuZ5bs2rVKjIyMhg2bFiOk7ufe+45/Pz8+PHHH/P1+iLFgaaWROxY06ZNWbJkCRkZGWzfvp2lS5cydepUunfvTmxsLHXq1Lnl9iEhITnu+/v7AxAcHHzD8uvPDwGoUaPGDcvCw8NZtGhRrq954MABDMO46bZAoX7SyMPDg8DAwBzLSpcunWNsBw4cIDEx8YbzjK75+4nUu3bt4o033mD16tU3fCw6MTExx/1KlSrh5uaWr7zffPMNfn5+uLq6UrlyZcLCwm65/tGjR4G/CtDfubm5Ua1atezHRUoSFRkRB+Dm5kbTpk1p2rQp4eHhPPXUUyxevJixY8fecjtnZ+c8Lzf+drLvP2Gz2bBYLKxYseKmr1OY52rkNt6/s9lslCtXjrlz59708WtF6PLly7Rp0wY/Pz/eeustwsLC8PDwIDo6mldeeQWbzZZju78fvcmre+65J/tTSyJyZ1RkRBxMkyZNAEhISCj01zpw4MANy/bv33/LT8CEhYVhGAZVq1YlPDy8ENPdmbCwMFatWkXLli1vWT7WrFnDhQsXWLJkCffcc0/28tt9oqgwhYaGAn9dN6hatWrZyzMyMoiPj6ddu3bZy4r6assiZtE5MiJ26rfffrvpUZJr56dcP71QGJYtW8bJkyez72/evJk///yTzp0757rNo48+irOzM+PHj78hv2EYXLhwodDy5kXPnj2xWq1MmDDhhseysrKyP8Z87ejO38eQkZHBRx99VCQ5b6Zdu3a4ubkxffr0HLk+//xzEhMT6dKlS/Yyb2/vG6a/RIojHZERsVODBw8mNTWVRx55hFq1apGRkcGGDRtYuHAhVapUyfeJpXeievXqtGrVihdeeIH09HQ+/PBDypQpw6hRo3LdJiwsjLfffpvRo0dz5MgRunXrhq+vL/Hx8SxdupT/+7//Y+TIkbd97S+++IKffvrphuVDhw79R2Nq06YNAwYMYOLEicTGxtKhQwdcXV05cOAAixcvZtq0aXTv3p0WLVpQunRp+vfvz5AhQ7BYLPzvf/8rsCm4OxEYGMjo0aMZP348nTp14qGHHmLfvn189NFHNG3alMcffzx73caNG7Nw4UKGDx9O06ZN8fHxoWvXrqZlFyksKjIidmrKlCksXryY5cuX8+mnn5KRkUFISAgvvvgib7zxRqFdi+XvnnjiCZycnPjwww85e/Ysd911F1FRUVSsWPGW27366quEh4czdepUxo8fD/x1gnGHDh146KGH8vTas2bNuunygvhyzI8//pjGjRvzySef8Nprr+Hi4kKVKlV4/PHHadmyJfDXtVx++OEHRowYwRtvvEHp0qV5/PHHadu2LR07dvzHGe7UuHHjCAwMJCoqipdeeomAgAD+7//+j3fffTfHidQvvvgisbGxzJ49O/uihioyUhxZDDP/vBARu3TkyBGqVq3K5MmT83T0RETELDpHRkRERByWioyIiIg4LBUZERERcVg6R0ZEREQclo7IiIiIiMNSkRERERGHVeyvI2Oz2Th16hS+vr66ZLeIiIiDMAyD5ORkgoKCcnzb+/WKfZE5derUDd/0KyIiIo7h+PHjVK5cOdfHi32R8fX1Bf76Qfj5+ZmcRkRERPIiKSmJ4ODg7Pfx3BT7InNtOsnPz09FRkRExMHc7rQQnewrIiIiDktFRkRERByWioyIiIg4rGJ/jkxeWa1WMjMzzY4hxZibm9stP0IoIiL5V+KLjGEYnD59msuXL5sdRYo5Jycnqlatipubm9lRRESKjRJfZK6VmHLlyuHl5aWL5kmhuHZhxoSEBEJCQvR7JiJSQEp0kbFardklpkyZMmbHkWIuMDCQU6dOkZWVhaurq9lxRESKhRI9YX/tnBgvLy+Tk0hJcG1KyWq1mpxERKT4KNFF5hod5peioN8zEZGCpyIjIiIiDsvUIvP777/TtWtXgoKCsFgsLFu2LMfjhmEwZswYKlasiKenJ+3atePAgQPmhHUgN/tZyq2tWbMGi8WiT6+JiDgYU4tMSkoKDRo0YObMmTd9/L333mP69Ol8/PHH/Pnnn3h7e9OxY0fS0tKKOKl9efLJJ+nWrVuujyckJNC5c+eiC5RPFovlhlurVq2K7PXvvfdehg0blmNZixYtSEhIwN/fv8hyiIjIP2fqp5Y6d+6c6xuuYRh8+OGHvPHGGzz88MMA/Pe//6V8+fIsW7aM3r17F2VUh1KhQgWzI2AYBlarFReXm/+KzZ49m06dOmXfN/vaKm5ubnbxcxMRcSRWm8Ha/We5v1Z50zLY7Tky8fHxnD59mnbt2mUv8/f3p1mzZmzcuDHX7dLT00lKSspxK2n+PrV05MgRLBYLS5Ys4b777sPLy4sGDRrc8DNct24drVu3xtPTk+DgYIYMGUJKSkr24//73/9o0qQJvr6+VKhQgb59+3L27Nnsx69NzaxYsYLGjRvj7u7OunXrcs1YqlQpKlSokH0LCAi4Ifvf150zZ06+xrN+/XruvfdevLy8KF26NB07duTSpUs8+eSTrF27lmnTpmUfDTpy5MhNp5a++eYbIiIicHd3p0qVKrz//vs5XqNKlSq8++67PP300/j6+hISEsKnn356y30jIlJcnE1O44kv/uTpOVv5bvsp03LYbZE5ffo0AOXL52x55cuXz37sZiZOnIi/v3/2LTg4OM+vaRgGqRlZptwMw7izH1Qevf7664wcOZLY2FjCw8Pp06cPWVlZABw6dIhOnTrx2GOPsWPHDhYuXMi6desYNGhQ9vaZmZlMmDCB7du3s2zZMo4cOcKTTz55w+u8+uqrTJo0iT179lC/fn1TxhMbG0vbtm2pU6cOGzduZN26dXTt2hWr1cq0adNo3rw5zz33HAkJCSQkJNz0d2Tbtm307NmT3r17ExcXx7hx43jzzTezC9U177//Pk2aNCEmJoYXX3yRF154gX379hXauEVE7MH6g+d5YNo61h+8gKerc6G/h91Ksbsg3ujRoxk+fHj2/aSkpDyXmauZVuqMWVlY0W5p91sd8XIrvN0xcuRIunTpAsD48eOJiIjg4MGD1KpVi4kTJ9KvX7/s80Zq1KjB9OnTadOmDbNmzcLDw4Onn346+7mqVavG9OnTadq0KVeuXMHHxyf7sbfeeov27dvfNk+fPn1wdnbOvv/VV1/d8ryf/Iznvffeo0mTJnz00UfZ60dERGT/t5ubG15eXrecSvrggw9o27Ytb775JgDh4eHs3r2byZMn5yhwDzzwAC+++CIAr7zyClOnTuW3336jZs2aeR6LiIijsNoMpv16gBmrD2AYULO8L1F9I6lR3te0THZ7RObam8yZM2dyLD9z5swt34Dc3d3x8/PLcRNyHB2pWLEiQPbU0Pbt25kzZw4+Pj7Zt44dO2Kz2YiPjwf+OkLRtWtXQkJC8PX1pU2bNgAcO3Ysx+s0adIkT3mmTp1KbGxs9i0v5Sev47l2ROaf2LNnDy1btsyxrGXLlhw4cCDHBe3+nsNisVChQoUcU24iIsXFmaQ0+n62iem//lViejcNZtnAlqaWGLDjIzJVq1alQoUK/PrrrzRs2BD46+jKn3/+yQsvvFAor+np6szutzoWynPn5bUL098viX/twmw2mw2AK1euMGDAAIYMGXLDdiEhIaSkpNCxY0c6duzI3LlzCQwM5NixY3Ts2JGMjIwc63t7e+cpT4UKFahevfoNyy0Wyw2HKG/2reS3Go+np2eeMhSE679qwGKxZOcQESku1u4/x0sLY7mYkoG3mzPvPlqPhxtWMjsWYHKRuXLlCgcPHsy+Hx8fT2xsLAEBAYSEhDBs2DDefvttatSoQdWqVXnzzTcJCgrK1xREflgslkKd3rFXjRo1Yvfu3TctFgBxcXFcuHCBSZMmZU/Tbd26tVCyBAYGkpCQkH3/wIEDpKam5us56tevz6+//sr48eNv+ribm9ttvyagdu3arF+/Psey9evXEx4enmNKTESkOMuy2nj/l/3MWnMIgNoV/ZjZN5JqgT632bLomPquvXXrVu67777s+9fObenfvz9z5sxh1KhRpKSk8H//939cvnyZVq1a8dNPP+Hh4WFWZLuRmJhIbGxsjmVlypTJ18nN17zyyivcfffdDBo0iGeffRZvb292797NL7/8QlRUFCEhIbi5uTFjxgyef/55du7cyYQJEwpoJDndf//9REVF0bx5c6xWK6+88kq+v2Bx9OjR1KtXjxdffJHnn38eNzc3fvvtN3r06EHZsmWpUqUKf/75J0eOHMHHxyf7E1N/N2LECJo2bcqECRPo1asXGzduJCoqKsd5NyIixdmpy1cZMj+GrUcvAfD43SG80aUOHoU8g5Bfpp4jc++992IYxg23a58MsVgsvPXWW5w+fZq0tDRWrVpFeHi4mZHtxpo1a4iMjMxxy+0IxO3Ur1+ftWvXsn//flq3bk1kZCRjxowhKCgI+OsoyZw5c1i8eDF16tRh0qRJTJkypSCHk+39998nODiY1q1b07dvX0aOHJnvL/UMDw/n559/Zvv27dx11100b96cb7/9NvuaNiNHjsTZ2Zk6depkT5Ndr1GjRixatIgFCxZQt25dxowZw1tvvXXTT2qJiBQ3q/ee4YHpf7D16CV83F2I6hvJ293q2V2JAbAYZn5mqggkJSXh7+9PYmLiDSf+pqWlER8fT9WqVXWURwqdft9ExN5lWm2899NePvvjrw961KvkT1TfSELL5O38x4J0q/fvvyt5J4SIiIjIDY5fTGXw/Bhij18G4MkWVRj9QC3cXezvKMzfqciIiIiUcCt3neblxdtJSsvCz8OF97o3oFNdx/jaFhUZERGREio9y8qkFXuZvf4IAA2CSxHVJ5LggPydm2gmFRkREZES6OiFFAbNiyHuZCIAz7Wuyssda+HmYrfXyr0pFRkw9TsipOTQ75mI2IsfdyTw6jc7SE7PopSXK1O6N6BdHfO+wfqfKNFF5tr1SVJTU4v0arBSMl27CrIuqCciZknLtPL2j7v5atNfl51oHFqaGX0iCSrluO+BJbrIODs7U6pUqezvxvHy8sq+3L1IQbLZbJw7dw4vL6/s69mIiBSl+PMpDJwbze6EJABeuDeM4e3DcXV2rKmk65X4f1GvfQGlvuhPCpuTkxMhISEqyyJS5L6NPclrS+JIybAS4O3GBz0bcG/NcmbHKhAlvshYLBYqVqxIuXLlbvrlhCIFxc3NDScnx/7LR0QcS1qmlXHf7WLBluMA3FU1gOm9I6ngX3wuylnii8w1zs7OOndBRESKjYNnkxk4N4Z9Z5KxWGDwfdUZ0rYGLg4+lXQ9FRkREZFi5pttJ3hj2U6uZlop6+POh70a0qpGWbNjFQoVGRERkWIiNSOLMd/u4uttJwBoEVaGD3s3pJxv8ZlKup6KjIiISDGw73QyA+dFc/DsFZwsMKxdOAPvq46zU/H+gIGKjIiIiAMzDINFW48z5ttdpGfZKOfrzrTekTQPK2N2tCKhIiMiIuKgrqRn8cbSOJbFngKgdY2yTO3VkLI+7iYnKzoqMiIiIg5o96kkBs2L5vD5FJydLIzoEM7z94ThVMynkq6nIiMiIuJADMNg7p/HeOuH3WRk2ajo78H0PpE0rRJgdjRTqMiIiIg4iKS0TEYviePHHQkA3F+rHFN6NCDA283kZOZRkREREXEAcScSGTQ/mqMXUnFxsvBKp1o806pqiZtKup6KjIiIiB0zDIMvNxzh3eV7ybDaqFTKkxl9I2kUUtrsaHZBRUZERMROJaZmMuqb7azcdQaADnXKM7l7A/y9XE1OZj9UZEREROxQzLFLDJ4fw4lLV3F1tvDaA7V5skUVLJaSPZV0PRUZERERO2IYBp+vi2fSir1k2QxCAryI6htJ/cqlzI5ml1RkRERE7MSllAxGLt7Or3vPAvBAvQpMeqw+fh6aSsqNioyIiIgd2Hb0IoPnxXAqMQ03FyfefLAOjzcL0VTSbajIiIiImMhmM/jk98NM+XkfVptB1bLeRPWNJCLI3+xoDkFFRkRExCQXrqQzfNF21u4/B8BDDYJ499F6+Ljr7Tmv9JMSERExwZ+HLzBkQQxnktJxd3Fi/EMR9GoarKmkfFKRERERKUJWm8FHvx1k6qr92AwIC/RmZr9G1KrgZ3Y0h6QiIyIiUkTOJafz0sJY1h08D8CjjSox4eG6eGsq6Y45mR3gdpKTkxk2bBihoaF4enrSokULtmzZYnYsERGRfFl/8Dydp/3BuoPn8XR1ZnL3+nzQs6FKzD9k9z+9Z599lp07d/K///2PoKAgvvrqK9q1a8fu3bupVKmS2fFERERuyWozmPbrAWasPoBhQHh5H2b2bUSN8r5mRysWLIZhGGaHyM3Vq1fx9fXl22+/pUuXLtnLGzduTOfOnXn77bdv+xxJSUn4+/uTmJiIn5/mH0VEpOicSUpj6IIYNh2+CECvJsGMeygCTzdnk5PZv7y+f9v1EZmsrCysViseHh45lnt6erJu3bqbbpOenk56enr2/aSkpELNKCIicjNr959j+MJYLqRk4OXmzLuP1KNbpGYSCppdnyPj6+tL8+bNmTBhAqdOncJqtfLVV1+xceNGEhISbrrNxIkT8ff3z74FBwcXcWoRESnJsqw23vtpL/2/2MyFlAxqV/Tjh8GtVGIKiV1PLQEcOnSIp59+mt9//x1nZ2caNWpEeHg427ZtY8+ePTesf7MjMsHBwZpaEhGRQpeQeJUh82PYcuQSAP2ahfDmg3XwcNVUUn4Vi6klgLCwMNauXUtKSgpJSUlUrFiRXr16Ua1atZuu7+7ujru7exGnFBGRkm713jOMWLSdS6mZ+Li7MOmxejxYP8jsWMWe3ReZa7y9vfH29ubSpUusXLmS9957z+xIIiIiZFptTF65j09/PwxA3Up+zOzbiNAy3iYnKxnsvsisXLkSwzCoWbMmBw8e5OWXX6ZWrVo89dRTZkcTEZES7sSlVAbPjyHm2GUAnmxRhdEP1MLdRVNJRcXui0xiYiKjR4/mxIkTBAQE8Nhjj/HOO+/g6upqdjQRESnBVu46zcuLt5OUloWvhwuTu9enU92KZscqcez+ZN9/SteRERGRgpSRZWPiij3MXn8EgAbBpYjqE0lwgJe5wYqZYnOyr4iIiL04diGVQfOj2XEiEYBnW1VlVKdauLnY9dVMijUVGRERkTxYHpfAK1/vIDk9C39PV97v0YB2dcqbHavEU5ERERG5hbRMK+/8uIf/bToKQOPQ0kzvE0mlUp4mJxNQkREREclV/PkUBs6NZnfCX19383ybMEZ0CMfVWVNJ9kJFRkRE5Ca+jT3Ja0viSMmwEuDtxvs9G3BfzXJmx5LrqMiIiIj8TVqmlfHf72L+5uMA3FUlgOl9Iqng73GbLcUMKjIiIiL/z8GzVxg0L5q9p5OxWGDQfdUZ2rYGLppKslsqMiIiIsA3207wxrKdXM20UtbHjQ97RdKqRlmzY8ltqMiIiEiJlpqRxZhvd/H1thMAtAgrw4e9GlLOT1NJjkBFRkRESqz9Z5IZODeaA2ev4GSBoW3DGXR/dZydLGZHkzxSkRERkRLHMAwWbT3O2O92kZZpo5yvO9N6R9I8rIzZ0SSfVGRERKREuZKexRtL41gWewqA1jXKMrVXQ8r6uJucTO6EioyIiJQYu08lMWheNIfPp+DsZGF4+3BeaBOGk6aSHJaKjIiIFHuGYTBv8zHGf7+bjCwbFfw8mNE3kqZVAsyOJv+QioyIiBRryWmZvLokjh93JABwX81A3u/ZkABvN5OTSUFQkRERkWJr58lEBs6L5uiFVFycLIzqVJNnW1XTVFIxoiIjIiLFjmEYfLnhCO8u30uG1UalUp7M6BtJo5DSZkeTAqYiIyIixUri1Uxe+XoHP+06DUD7OuWZ3L0+pbw0lVQcqciIiEixEXv8MoPmRXPi0lVcnS2M7lybp1pWwWLRVFJxpSIjIiIOzzAMPl8Xz6QVe8myGQQHeBLVpxENgkuZHU0KmYqMiIg4tMupGYxcvJ1Ve84C0LluBSY9Vh9/T1eTk0lRUJERERGHte3oRQbPi+FUYhpuzk68+WBtHr87VFNJJYiKjIiIOBybzeDTPw4zeeU+rDaDKmW8iOrbiLqV/M2OJkVMRUZERBzKhSvpjFi8nTX7zgHQtUEQ7z5SF18PTSWVRCoyIiLiMP48fIEhC2I4k5SOu4sT4x6KoHfTYE0llWAqMiIiYvesNoOPfjvI1FX7sRlQLdCbmX0bUbuin9nRxGQqMiIiYtfOJafz0sJY1h08D8CjkZWY0K0u3u56CxMVGRERsWMbDp5n6MJYziWn4+HqxISH69KjSbDZscSOqMiIiIjdsdoMpv16gBmrD2AYEF7eh5l9G1GjvK/Z0cTOqMiIiIhdOZOUxtAFMWw6fBGAnk0qM/6huni6OZucTOyRioyIiNiN3/ef46WFsVxIycDLzZl3HqnLI5GVzY4ldszJ7AC3YrVaefPNN6latSqenp6EhYUxYcIEDMMwO5qIiBSgLKuNySv30n/2Zi6kZFCrgi/fD26lEiO3ZddHZP79738za9YsvvzySyIiIti6dStPPfUU/v7+DBkyxOx4IiJSABISrzJkfgxbjlwCoG+zEMY8WAcPV00lye3ZdZHZsGEDDz/8MF26dAGgSpUqzJ8/n82bN5ucTERECsJve88yfFEsl1Iz8XF3YeKj9ejaIMjsWOJA7HpqqUWLFvz666/s378fgO3bt7Nu3To6d+6c6zbp6ekkJSXluImIiH3JtNqYuHwPT83ZwqXUTOpW8uOHwa1UYiTf7PqIzKuvvkpSUhK1atXC2dkZq9XKO++8Q79+/XLdZuLEiYwfP74IU4qISH6cuJTK4PkxxBy7DED/5qG81qU27i6aSpL8s+sis2jRIubOncu8efOIiIggNjaWYcOGERQURP/+/W+6zejRoxk+fHj2/aSkJIKDdfEkERF78POu07z89Q4Sr2bi6+HC5O716VS3otmxxIFZDDv+CFBwcDCvvvoqAwcOzF729ttv89VXX7F37948PUdSUhL+/v4kJibi56fv5BARMUNGlo2JK/Ywe/0RABpU9ieqbyOCA7zMDSZ2K6/v33Z9RCY1NRUnp5yn8Tg7O2Oz2UxKJCIi+XXsQiqD5kez40QiAM+0qsornWrh5mLXp2mKg7DrItO1a1feeecdQkJCiIiIICYmhg8++ICnn37a7GgiIpIHK+ISGPX1DpLTs/D3dGVKjwa0r1Pe7FhSjNj11FJycjJvvvkmS5cu5ezZswQFBdGnTx/GjBmDm5tbnp5DU0siIkUvLdPKu8v38N+NRwFoFFKKGX0bUamUp8nJxFHk9f3brotMQVCREREpWvHnUxg0L5pdp/66/MWANtUY2aEmrs6aSpK8KxbnyIiIiGP5bvspXlsSx5X0LAK83Xi/ZwPuq1nO7FhSjKnIiIjIP5aWaWX897uZv/kYAHdVCWB6n0gq+HuYnEyKOxUZERH5Rw6evcKgedHsPZ2MxQKD7qvO0LY1cNFUkhQBFRkREbljS6JP8MaynaRmWCnr48bUXg1pXSPQ7FhSgqjIiIhIvqVmZDH2210s3nYCgObVyjCtd0PK+WkqSYqWioyIiOTL/jPJDJwbzYGzV7BYYGjbGgy+vwbOThazo0kJpCIjIiJ5YhgGi7eeYMx3O0nLtBHo68603g1pEVbW7GhSgqnIiIjIbaWkZ/HGsp0sjTkJQOsaZZnaqyFlfdxNTiYlnYqMiIjc0p6EJAbOjebw+RScLDCiQ01eaBOGk6aSxA6oyIiIyE0ZhsG8zccY//1uMrJsVPDzYHqfSO6qGmB2NJFsKjIiInKD5LRMRi+J44cdCQDcVzOQ93s2JMA7b99zJ1JUVGRERCSHnScTGTQvmiMXUnFxsvByx5o817qappLELqnIiIgI8NdU0n83HuWdH/eQYbVRqZQn0/tE0ji0tNnRRHKlIiMiIiRezeTVb3awYudpANrVLs+UHvUp5aWpJLFvKjIiIiVc7PHLDJoXzYlLV3F1tvBq59o83bIKFoumksT+qciIiJRQhmHw+bp4/v3TXjKtBsEBnkT1aUSD4FJmRxPJMxUZEZES6HJqBiMX72DVnjMAdK5bgUmP1cff09XkZCL5oyIjIlLCbDt6icHzojmVmIabsxNvPFibf90dqqkkcUgqMiIiJYTNZvDpH4eZvHIfVptBlTJeRPVtRN1K/mZHE7ljKjIiIiXAxZQMhi+KZc2+cwB0bRDEu4/UxddDU0ni2FRkRESKuc3xFxkyP4bTSWm4uzgxtmsEfe4K1lSSFAsqMiIixZTNZvDRmoN88Mt+bAZUC/RmZt9G1K7oZ3Y0kQKjIiMiUgydS05n+KJY/jhwHoBHIysxoVtdvN31z74UL/qNFhEpZjYcPM/QhbGcS07Hw9WJtx6uS4/GlTWVJMWSioyISDFhtRlM//UA01cfwDCgRjkfZvZrRHh5X7OjiRQaFRkRkWLgbFIaQxfEsvHwBQB6NqnM+Ifq4unmbHIykcKlIiMi4uD+OHCOlxbGcv5KBl5uzrzzSF0eiaxsdiyRIqEiIyLioLKsNj5cdYCZaw5iGFCrgi9RfRtRvZyP2dFEioyKjIiIA0pIvMrQ+bFsPnIRgL7NQhjzYB08XDWVJCWLioyIiIP5be9Zhi+K5VJqJj7uLrz7aD0eahBkdiwRU6jIiIg4iEyrjSkr9/HJ74cBiAjyY2bfRlQp621yMhHzOJkd4HaqVKmCxWK54TZw4ECzo4mIFJmTl6/S65ON2SWmf/NQvnmhhUqMlHh2f0Rmy5YtWK3W7Ps7d+6kffv29OjRw8RUIiJF55fdZxi5eDuJVzPx9XDhvcfq07leRbNjidgFuy8ygYGBOe5PmjSJsLAw2rRpY1IiEZGikZFl498/7eXzdfEANKjsT1TfRgQHeJmcTMR+2H2R+buMjAy++uorhg8fnuulttPT00lPT8++n5SUVFTxREQKzPGLqQyaF832E4kAPNOqKq90qoWbi92fESBSpByqyCxbtozLly/z5JNP5rrOxIkTGT9+fNGFEhEpYD/tTODlr3eQnJaFv6crU3o0oH2d8mbHErFLFsMwDLND5FXHjh1xc3Pj+++/z3Wdmx2RCQ4OJjExET8/fXW9iNivtEwrE5fv4cuNRwFoFFKK6X0iqVxaU0lS8iQlJeHv73/b92+HOSJz9OhRVq1axZIlS265nru7O+7u7kWUSkSkYBw5n8LAedHsOvXXdPiANtUY2aEmrs6aShK5FYcpMrNnz6ZcuXJ06dLF7CgiIgXq++2nGL0kjivpWZT2cuWDng25r1Y5s2OJOASHKDI2m43Zs2fTv39/XFwcIrKIyG2lZVp564fdzPvzGABNq5Rmep9IKvp7mpxMxHE4RCtYtWoVx44d4+mnnzY7iohIgTh07goD50az93QyFgsMvLc6w9rVwEVTSSL54hBFpkOHDjjQOckiIre0NOYEry/dSWqGlbI+bkzt1ZDWNQJvv6GI3MAhioyISHFwNcPK2O92smjrCQCaVyvDtN4NKefnYXIyEcelIiMiUgQOnEnmxbnRHDh7BYsFhratweD7a+DsdPOLe4pI3qjIiIgUIsMwWLztBGO+3Ulapo1AX3em9W5Ii7CyZkcTKRZUZERECklKehZvLtvJkpiTALSuUZYPejYk0FfXuhIpKCoyIiKFYE9CEgPnRXP4XApOFhjRoSYvtAnDSVNJIgVKRUZEpAAZhsH8zccZ//0u0rNsVPDzYHqfSO6qGmB2NJFiSUVGRKSAJKdl8trSnXy//RQA99YM5IOeDQnwdjM5mUjxpSIjIlIAdp5MZNC8aI5cSMXZycKojjV5rnU1TSWJFDIVGRGRf8AwDP636Shv/7CHDKuNSqU8md4nksahpc2OJlIiqMiIiNyhxKuZjF6yg+VxpwFoV7s8U3rUp5SXppJEioqKjIjIHdh+/DKD5kdz/OJVXJ0tvNq5Nk+3rILFoqkkkaKkIiMikg+GYfDF+iNMWrGHTKtB5dKezOzbiAbBpcyOJlIiqciIiOTR5dQMXv56B7/sPgNAp4gK/Lt7ffw9XU1OJlJyqciIiORB9LFLDJ4Xw8nLV3FzduKNB2vzr7tDNZUkYjIVGRGRW7DZDD774zCTV+4jy2YQWsaLmX0bUbeSv9nRRAQVGRGRXF1MyWDEolh+23cOgAfrV2Tio/Xw9dBUkoi9UJEREbmJzfEXGTI/htNJabi5ODGuawR97grWVJKInXHK7wY//fQT69aty74/c+ZMGjZsSN++fbl06VKBhhMRKWo2m8HM3w7S57NNnE5Ko1qgN98ObEnfZiEqMSJ2KN9F5uWXXyYpKQmAuLg4RowYwQMPPEB8fDzDhw8v8IAiIkXl/JV0+s/ezOSV+7DaDB6JrMT3g1pRu6Kf2dFEJBf5nlqKj4+nTp06AHzzzTc8+OCDvPvuu0RHR/PAAw8UeEARkaKw8dAFhi6I4WxyOh6uTrz1cF16NK6sozAidi7fRcbNzY3U1FQAVq1axRNPPAFAQEBA9pEaERFHYbUZzFh9gOm/HsBmQI1yPszs14jw8r5mRxORPMh3kWnVqhXDhw+nZcuWbN68mYULFwKwf/9+KleuXOABRUQKy9nkNIYtiGXDoQsA9GhcmfEPR+Dlps9BiDiKfJ8jExUVhYuLC19//TWzZs2iUqVKAKxYsYJOnToVeEARkcKw7sB5Hpj2BxsOXcDLzZkPejZgco8GKjEiDsZiGIZhdojClJSUhL+/P4mJifj56YQ9kZIuy2rjw1UHmLnmIIYBtSr4EtW3EdXL+ZgdTUT+Jq/v33n60yMpKSn7SW53HozKgojYq9OJaQyZH8PmIxcB6NsshDEP1sHD1dnkZCJyp/JUZEqXLk1CQgLlypWjVKlSNz2L3zAMLBYLVqu1wEOKiPxTv+07y4hF27mYkoGPuwvvPlqPhxoEmR1LRP6hPBWZ1atXExAQkP3f+jiiiDiKTKuNKT/v45O1hwGICPIjqm8jqpb1NjmZiBQEnSMjIsXWyctXGTI/hm1H/7rq+BPNQ3ntgdqaShJxAHl9/873p5bGjRuHzWa7YXliYiJ9+vTJ79OJiBSKVbvP8MC0P9h29BK+Hi7M6teItx6uqxIjUszku8h8/vnntGrVisOHD2cvW7NmDfXq1ePQoUMFGk5EJL8ysmy8/cNunv3vVhKvZtKgsj8/Dm5N53oVzY4mIoUg30Vmx44dVK5cmYYNG/LZZ5/x8ssv06FDB/71r3+xYcOGwsgoIpInxy+m0uOTjfxnXTwAT7esyuLnWxBSxsvkZCJSWPJ95afSpUuzaNEiXnvtNQYMGICLiwsrVqygbdu2hZGPkydP8sorr7BixQpSU1OpXr06s2fPpkmTJoXyeiLimH7amcDLX+8gOS0LPw8XpvRoQIeICmbHEpFClu8jMgAzZsxg2rRp9OnTh2rVqjFkyBC2b99e0Nm4dOkSLVu2xNXVlRUrVrB7927ef/99SpcuXeCvJSKOKT3Lythvd/L8V9Ekp2URGVKK5UNbq8SIlBD5PiLTqVMntm7dypdffkn37t25evUqw4cP5+6772b8+PGMGjWqwML9+9//Jjg4mNmzZ2cvq1q1aoE9v4g4tiPnUxg0P5qdJ/+6UOeANtUY2aEmrs539DeaiDigfP/fbrVa2bFjB927dwfA09OTWbNm8fXXXzN16tQCDffdd9/RpEkTevToQbly5YiMjOSzzz675Tbp6ekkJSXluIlI8fPDjlM8OGMdO08mUdrLldlPNmV059oqMSIlTIFeR+b8+fOULVu2oJ4ODw8PAIYPH06PHj3YsmULQ4cO5eOPP6Z///433WbcuHGMHz/+huW6joxI8ZCWaeWtH3Yz789jADStUprpfSKp6O9pcjIRKUh5vY6MXV8Qz83NjSZNmuT4NNSQIUPYsmULGzduvOk26enppKenZ99PSkoiODhYRUakGDh07goD50az93QyFgu8eG8YL7ULx0VHYUSKnQL90si/s1qtTJ06lUWLFnHs2DEyMjJyPH7x4sX8p81FxYoVqVOnTo5ltWvX5ptvvsl1G3d3d9zd3Qssg4jYh2UxJ3ltaRypGVbKeLsxtVdD7gkPNDuWiJgs33/GjB8/ng8++IBevXqRmJjI8OHDefTRR3FycmLcuHEFGq5ly5bs27cvx7L9+/cTGhpaoK8jIvbraoaVV77ewbCFsaRmWLm7WgArhrZWiRER4A6mlsLCwpg+fTpdunTB19eX2NjY7GWbNm1i3rx5BRZuy5YttGjRgvHjx9OzZ082b97Mc889x6effkq/fv3y9Bz6riURx3XgTDID50Wz/8wVLBYYcn8NhrStgbOTvrhWpLgrtO9aOn36NPXq1QPAx8eHxMREAB588EF+/PHHO4x7c02bNmXp0qXMnz+funXrMmHCBD788MM8lxgRcVyLtx7noaj17D9zhUBfd+Y+04yX2oerxIhIDvk+R6Zy5cokJCQQEhJCWFgYP//8M40aNWLLli2Fcm7Kgw8+yIMPPljgzysi9iklPYs3v93JkuiTALSqXpapvRoS6Ktz30TkRvkuMo888gi//vorzZo1Y/DgwTz++ON8/vnnHDt2jJdeeqkwMopICbH3dBID50Zz6FwKThYY3j6cF++tjpOOwohILv7xx683btzIxo0bqVGjBl27di2oXAVG58iI2D/DMFiw5TjjvttFepaN8n7uTO8dSbNqZcyOJiImKbSPX1+vefPmNG/e/J8+jYiUUFfSs3htSRzfbT8FwL01A3m/RwPK+GgqSURu7x8VGT8/P2JjY6lWrVpB5RGREmTnyUQGzYvmyIVUnJ0svNyxJv/XupqmkkQkz/JcZE6dOkVQUFCOZXZ8UWARsWOGYfDVpqNM+GEPGVYbQf4ezOgbSePQALOjiYiDyfPHryMiIgr0GjEiUjIlpWUycF40b367iwyrjXa1y7F8aGuVGBG5I3kuMu+88w4DBgygR48e2V9D8Pjjj+sEWhHJsx0nLtNl+h8sjzuNq7OFN7rU5rMnmlDKy83saCLioPL1qaX4+HieeeYZdu/ezWeffWaXn1K6nj61JGI+wzCYvf4IE1fsIdNqULm0J1F9G9EwuJTZ0UTEThXKp5aqVq3K6tWriYqK4tFHH6V27dq4uOR8iujo6DtLLCLFUmJqJi9/vZ2fd58BoFNEBf7dvT7+nq4mJxOR4iDfn1o6evQoS5YsoXTp0jz88MM3FBkRkWuij11i8LwYTl6+ipuzE693qc0TzUOxWPSpJBEpGPlqIZ999hkjRoygXbt27Nq1i8BAffusiNzIZjP4z7rDvPfTPrJsBqFlvIjq04h6lf3NjiYixUyei0ynTp3YvHkzUVFRPPHEE4WZSUQc2KWUDEYs3s7qvWcBeLB+RSY+Wg9fD00liUjBy3ORsVqt7Nixg8qVKxdmHhFxYFuOXGTI/BgSEtNwc3FibNc69L0rRFNJIlJo8lxkfvnll8LMISIOzGYzmLX2EB/8sh+rzaBaWW+i+jaiTpA+KSgihUtn6orIP3L+SjovLYzljwPnAXgkshJvd6uLt7v+eRGRwqd/aUTkjm08dIGhC2I4m5yOh6sTbz1Ulx5NKmsqSUSKjIqMiOSb1WYQtfog037dj82A6uV8+KhfI8LL+5odTURKGBUZEcmXs8lpDFsQy4ZDFwDo0bgy4x+OwMtN/5yISNHTvzwikmfrDpxn2MJYzl9Jx9PVmXceqcujjfRJRhExj4qMiNxWltXGtF8PEPXbQQwDalXwJapvI6qX8zE7moiUcCoyInJLpxPTGLIghs3xf33rfZ+7ghnbNQIPV2eTk4mIqMiIyC2s2XeW4Yu2czElA283Z959tB4PN6xkdiwRkWwqMiJyg0yrjQ9+2c+sNYcAqFPRj5n9GlG1rLfJyUREclKREZEcTl2+yuD5MWw7egmAJ5qH8toDtTWVJCJ2SUVGRLKt2n2GkV9v53JqJr7uLvy7e30eqFfR7FgiIrlSkRERMrJsvPfTXv6zLh6A+pX9ierTiJAyXiYnExG5NRUZkRLu+MVUBs2PYfvxywA83bIqr3SuibuLppJExP6pyIiUYD/tPM2or7eTlJaFn4cLU3o0oENEBbNjiYjkmYqMSAmUnmVl4vK9zNlwBIDIkFLM6BNJ5dKaShIRx6IiI1LCHL2QwqB5McSdTATg/+6pxssda+Lq7GRyMhGR/FORESlBftyRwKvf7CA5PYvSXq6837MB99cqb3YsEZE7Ztd/go0bNw6LxZLjVqtWLbNjiTictEwrbyyLY+C8aJLTs2hapTTLh7ZWiRERh2f3R2QiIiJYtWpV9n0XF7uPLGJXDp+7wsB5MexJSALgxXvDGN4+HBdNJYlIMWD3rcDFxYUKFfQpCpE78W3sSV5bEkdKhpUy3m580KshbcIDzY4lIlJg7L7IHDhwgKCgIDw8PGjevDkTJ04kJCQk1/XT09NJT0/Pvp+UlFQUMUXsytUMK+O+28XCrccBuLtaANN6R1Lez8PkZCIiBcuujy03a9aMOXPm8NNPPzFr1izi4+Np3bo1ycnJuW4zceJE/P39s2/BwcFFmFjEfAfPJtNt5noWbj2OxQJD2tZg7rN3q8SISLFkMQzDMDtEXl2+fJnQ0FA++OADnnnmmZuuc7MjMsHBwSQmJuLn51dUUUVM8fW2E7y5bCdXM62U9XFneu+GtKhe1uxYIiL5lpSUhL+//23fv+1+aunvSpUqRXh4OAcPHsx1HXd3d9zd3YswlYj5UjOyeGPZTpZEnwSgVfWyTO3VkEBf/b8gIsWbXU8tXe/KlSscOnSIihX1bbwi1+w9nUTXGetYEn0SJwuMaB/Ol0/fpRIjIiWCXR+RGTlyJF27diU0NJRTp04xduxYnJ2d6dOnj9nRRExnGAYLtxxn7He7SM+yUd7PnWm9I7m7Whmzo4mIFBm7LjInTpygT58+XLhwgcDAQFq1asWmTZsIDNTHR6Vku5KexetL4/g29hQAbcID+aBnA8r46CiMiJQsdl1kFixYYHYEEbuz61Qig+bFEH8+BWcnCyM71GTAPdVwcrKYHU1EpMjZdZERkf+fYRh89ecxJvywm4wsG0H+HszoG0nj0ACzo4mImEZFRsQBJKVlMvqbOH6MSwCgXe1yTO7egNLebiYnExExl4qMiJ3bceIyg+bFcOxiKi5OFl7tXItnWlXFYtFUkoiIioyInTIMgzkbjvDu8j1kWg0qlfIkqm8kkSGlzY4mImI3VGRE7FBiaiYvf72dn3efAaBjRHnee6wB/l6uJicTEbEvKjIidibm2CUGzYvh5OWruDk78doDtejfooqmkkREbkJFRsROGIbBf/6I598/7SXLZhAS4MXMvo2oV9nf7GgiInZLRUbEDlxKyWDk4u38uvcsAF3qV2Tio/Xw89BUkojIrajIiJhs65GLDJ4fQ0JiGm4uTox5sA79moVoKklEJA9UZERMYrMZfPz7Id7/eT9Wm0HVst5E9Y0kIkhTSSIieaUiI2KCC1fSGb5oO2v3nwPg4YZBvPNIPXzc9b+kiEh+6F9NkSK26fAFhi6I4UxSOh6uTox/KIKeTYI1lSQicgdUZESKiNVmMPO3g3y4aj82A6qX82Fm30bUrOBrdjQREYelIiNSBM4mp/HSwljWH7wAQPfGlXnr4Qi83PS/oIjIP6F/RUUK2fqD5xm6IJbzV9LxdHXm7W51eaxxZbNjiYgUCyoyIoXEajOYtmo/M347iGFAzfK+zOwXSfVymkoSESkoKjIiheBMUhpD5sfwZ/xFAPrcFczYrhF4uDqbnExEpHhRkREpYGv3n+OlhbFcTMnA282Zdx+tx8MNK5kdS0SkWFKRESkgWVYb7/+yn1lrDgFQu6IfM/tGUi3Qx+RkIiLFl4qMSAE4dfkqQ+bHsPXoJQD+dXcor3eprakkEZFCpiIj8g+t3nuG4Yu2czk1E193FyY9Vp8u9SuaHUtEpERQkRG5Q5lWG+/9tJfP/ogHoF4lf6L6RhJaxtvkZCIiJYeKjMgdOH4xlcHzY4g9fhmAp1pW4dXOtXB30VSSiEhRUpERyaeVu07z8uLtJKVl4efhwuQeDegYUcHsWCIiJZKKjEgepWdZmbh8L3M2HAGgYXApZvSJJDjAy9xgIiIlmIqMSB4cvZDCoHkxxJ1MBOC51lV5uWMt3FycTE4mIlKyqciI3MaPOxJ49ZsdJKdnUcrLlfd7NKBt7fJmxxIREVRkRHKVlmnl7R9389WmYwA0CS3N9D6RBJXyNDmZiIhcoyIjchPx51MYODea3QlJALx4bxgvtQ/H1VlTSSIi9kRFRuQ638ae5LUlcaRkWAnwdmNqr4a0CQ80O5aIiNyEiozI/5OWaWXcd7tYsOU4AM2qBjC9TyTl/TxMTiYiIrlxqOPkkyZNwmKxMGzYMLOjSDFz8GwyD0etZ8GW41gsMKRtDeY+20wlRkTEzjnMEZktW7bwySefUL9+fbOjSDHzzbYTvLFsJ1czrZT1cWda74a0rF7W7FgiIpIHDnFE5sqVK/Tr14/PPvuM0qVLmx1HionUjCxGLt7OiMXbuZpppWX1Miwf2kolRkTEgThEkRk4cCBdunShXbt2t103PT2dpKSkHDeR6+07ncxDUev5etsJnCwwvH04/326GeV8NZUkIuJI7H5qacGCBURHR7Nly5Y8rT9x4kTGjx9fyKnEURmGwaKtxxnz7S7Ss2yU83Vnep9I7q5WxuxoIiJyB+z6iMzx48cZOnQoc+fOxcMjb38pjx49msTExOzb8ePHCzmlOIor6Vm8tDCWV76JIz3Lxj3hgawY2lolRkTEgVkMwzDMDpGbZcuW8cgjj+Ds7Jy9zGq1YrFYcHJyIj09PcdjN5OUlIS/vz+JiYn4+fkVdmSxU7tPJTFoXjSHz6fg7GRhRIdwnr8nDCcni9nRRETkJvL6/m3XU0tt27YlLi4ux7KnnnqKWrVq8corr9y2xIgYhsHcP4/x1g+7yciyUdHfgxl9ImlSJcDsaCIiUgDsusj4+vpSt27dHMu8vb0pU6bMDctFrpeUlsnoJXH8uCMBgLa1yjGlRwNKe7uZnExERAqKXRcZkTsVdyKRQfOjOXohFRcnC690qsWzratisWgqSUSkOHG4IrNmzRqzI4gdMwyDLzcc4d3le8mw2qhUypMZfSNpFKLrD4mIFEcOV2REcpOYmsmob7azctcZADrUKc/k7g3w93I1OZmIiBQWFRkpFmKPX2bQvGhOXLqKq7OF1x6ozZMtqmgqSUSkmFOREYdmGAafr4tn0oq9ZNkMQgK8iOobSf3KpcyOJiIiRUBFRhzW5dQMRi7ezqo9ZwHoUq8iEx+rh5+HppJEREoKFRlxSNuOXmTwvBhOJabh5uLEmw/W4fFmIZpKEhEpYVRkxKHYbAaf/H6YKT/vw2ozqFrWm6i+kUQE+ZsdTURETKAiIw7jwpV0hi/aztr95wB4uGEQ7zxSDx93/RqLiJRUegcQh/Dn4QsMWRDDmaR03F2cGP9QBL2aBmsqSUSkhFOREbtmtRl89NtBpq7aj82AsEBvZvZrRK0K+gJQERFRkRE7di45nZcWxrLu4HkAHmtUmQndIvBy06+tiIj8Re8IYpc2HDzPkAWxnL+SjqerMxO61aV748pmxxIRETujIiN2xWozmPbrAWasPoBhQM3yvkT1jaRGeV+zo4mIiB1SkRG7cSYpjaELYth0+CIAvZsGM7ZrBJ5uziYnExERe6UiI3bh9/3neGlhLBdSMvB2c+bdR+vxcMNKZscSERE7pyIjpsqy2vjgl/18tOYQALUr+jGzbyTVAn1MTiYiIo5ARUZMk5B4lSHzY9hy5BIAj98dwhtd6uDhqqkkERHJGxUZMcVve88yfFEsl1Iz8XF3YdJj9XiwfpDZsURExMGoyEiRyrTamLJyH5/8fhiAepX8ieobSWgZb5OTiYiII1KRkSJz4lIqg+fHEHPsMgBPtqjC6Adq4e6iqSQREbkzKjJSJH7edZqRi7eTlJaFn4cL73VvQKe6FcyOJSIiDk5FRgpVRpaNiSv2MHv9EQAaBJciqk8kwQFe5gYTEZFiQUVGCs2xC6kMmh/NjhOJADzXuiovd6yFm4uTyclERKS4UJGRQrE8LoFXvt5BcnoWpbxcmdK9Ae3qlDc7loiIFDMqMlKg0jKtvPPjHv636SgAjUNLM6NPJEGlPE1OJiIixZGKjBSY+PMpDJoXza5TSQC8cG8Yw9uH4+qsqSQRESkcKjJSIL7bforR3+wgJcNKgLcbH/RswL01y5kdS0REijkVGflH0jKtjP9+N/M3HwPgrqoBTO8dSQV/D5OTiYhISaAiI3fs4NkrDJoXzd7TyVgsMPi+6gxpWwMXTSWJiEgRUZGRO7Ik+gRvLNtJaoaVsj7ufNirIa1qlDU7loiIlDAqMpIvqRlZjP12F4u3nQCgRVgZPuzdkHK+mkoSEZGipyIjebb/TDID50Zz4OwVnCwwrF04A++rjrOTxexoIiJSQtn1yQyzZs2ifv36+Pn54efnR/PmzVmxYoXZsUocwzBYtOU4D0Wt48DZK5TzdWfus3czpG0NlRgRETGVXR+RqVy5MpMmTaJGjRoYhsGXX37Jww8/TExMDBEREWbHKxFS0rN4fWkcy2JPAdC6Rlmm9mpIWR93k5OJiIiAxTAMw+wQ+REQEMDkyZN55pln8rR+UlIS/v7+JCYm4ufnV8jpipfdp5IYNC+aw+dTcHayMKJDOM/fE4aTjsKIiEghy+v7t10fkfk7q9XK4sWLSUlJoXnz5rmul56eTnp6evb9pKSkoohXrBiGwbzNxxj//W4ysmxU9Pdgep9ImlYJMDuaiIhIDnZfZOLi4mjevDlpaWn4+PiwdOlS6tSpk+v6EydOZPz48UWYsHhJTstk9JI4ftiRAMD9tcoxpUcDArzdTE4mIiJyI7ufWsrIyODYsWMkJiby9ddf85///Ie1a9fmWmZudkQmODhYU0t5sPNkIgPnRXP0QiouThZe6VSLZ1pV1VSSiIgUubxOLdl9kbleu3btCAsL45NPPsnT+jpH5vYMw+C/G4/yzo97yLDaqFTKkxl9I2kUUtrsaCIiUkIVu3NkrrHZbDmOuMg/k3g1k1e+3sFPu04D0KFOeSZ3b4C/l6vJyURERG7ProvM6NGj6dy5MyEhISQnJzNv3jzWrFnDypUrzY5WLMQev8ygedGcuHQVV2cLrz1QmydbVMFi0VSSiIg4BrsuMmfPnuWJJ54gISEBf39/6tevz8qVK2nfvr3Z0RyaYRh8vi6ef/+0l0yrQUiAF1F9I6lfuZTZ0URERPLFrovM559/bnaEYudyagYjF+9g1Z4zADxQrwKTHquPn4emkkRExPHYdZGRgrXt6CUGz4vmVGIabi5OvPlgHR5vFqKpJBERcVgqMiWAzWbw6R+HmbxyH1abQdWy3kT1jSQiyN/saCIiIv+Iikwxd+FKOiMWb2fNvnMAPNQgiHcfrYePu3a9iIg4Pr2bFWOb4y8yeH40Z5LScXdxYvxDEfRqGqypJBERKTZUZIohm83gozUH+eCX/dgMCAv0Zma/RtSqoAsCiohI8aIiU8ycS05n+KJY/jhwHoBHG1ViwsN18dZUkoiIFEN6dytGNhw8z9CFsZxLTsfT1Zm3Ho6gR5Ngs2OJiIgUGhWZYsBqM5j+6wGmrz6AYUB4eR9m9m1EjfK+ZkcTEREpVCoyDu5sUhpDF8Sy8fAFAHo1CWbcQxF4ujmbnExERKTwqcg4sD8OnOOlhbGcv5KBl5sz7z5Sj26RlcyOJSIiUmRUZBxQltXGh6sOMHPNQQwDalf0Y2bfSKoF+pgdTUREpEipyDiYhMSrDJ0fy+YjFwHo1yyENx+sg4erppJERKTkUZFxIL/tPcvwRbFcSs3Ex92FSY/V48H6QWbHEhERMY2KjAPItNqYsnIfn/x+GIC6lfyY2bcRoWW8TU4mIiJiLhUZO3fy8lUGz4sm+thlAJ5sUYXRD9TC3UVTSSIiIioyduyX3WcYuXg7iVcz8fVwYXL3+nSqW9HsWCIiInZDRcYOZWTZmLRiL1+sjwegQXApovpEEhzgZXIyERER+6IiY2eOX0xl0Lxotp9IBODZVlUZ1akWbi5OJicTERGxPyoyduSnnQm8/PUOktOy8Pd05f0eDWhXp7zZsUREROyWiowdSMu0MnH5Hr7ceBSAxqGlmd4nkkqlPE1OJiIiYt9UZEx25HwKA+dFs+tUEgDPtwljRIdwXJ01lSQiInI7KjIm+n77KUYvieNKehYB3m580LMB99YsZ3YsERERh6EiY4K0TCtv/bCbeX8eA+CuqgFM7x1JBX8Pk5OJiIg4FhWZInbo3BUGzo1m7+lkLBYYdF91hratgYumkkRERPJNRaYILY05wetLd5KaYaWsjxsf9oqkVY2yZscSERFxWCoyReBqhpWx3+1k0dYTALQIK8OHvRpSzk9TSSIiIv+EikwhO3AmmRfnRnPg7BWcLDC0bTiD7q+Os5PF7GgiIiIOT0WmkBiGweJtJxjz7U7SMm2U83VnWu9ImoeVMTuaiIhIsaEiUwhS0rN4c9lOlsScBKB1jbJM7dWQsj7uJicTEREpXlRkCtiehCQGzovm8LkUnJ0sDG8fzgttwnDSVJKIiEiBU5EpIIZhMH/zccZ/v4v0LBsV/DyY0TeSplUCzI4mIiJSbNn1xUsmTpxI06ZN8fX1pVy5cnTr1o19+/aZHesGyWmZDFkQy2tL40jPsnF/rXIsH9paJUZERKSQ2XWRWbt2LQMHDmTTpk388ssvZGZm0qFDB1JSUsyOlm3nyUS6zljH99tP4eJk4bUHavGfJ5oQ4O1mdjQREZFiz2IYhmF2iLw6d+4c5cqVY+3atdxzzz152iYpKQl/f38SExPx8/MrsCyGYfC/TUd5+4c9ZFhtVCrlyYy+kTQKKV1gryEiIlJS5fX926HOkUlMTAQgICD3KZv09HTS09Oz7yclJRV4DsMweGlhLMtiTwHQvk55JnevTykvHYUREREpSnY9tfR3NpuNYcOG0bJlS+rWrZvrehMnTsTf3z/7FhwcXOBZLBYLkSGlcXW2MObBOnz6r8YqMSIiIiZwmKmlF154gRUrVrBu3ToqV66c63o3OyITHBxcKFNL8edTqBboU2DPKSIiIn8pVlNLgwYN4ocffuD333+/ZYkBcHd3x9298C88Z7FYVGJERERMZtdFxjAMBg8ezNKlS1mzZg1Vq1Y1O5KIiIjYEbsuMgMHDmTevHl8++23+Pr6cvr0aQD8/f3x9PQ0OZ2IiIiYza7PkbFYbn5Z/9mzZ/Pkk0/m6TkK6+PXIiIiUniKxTkydtyxRERExA44zMevRURERK6nIiMiIiIOS0VGREREHJaKjIiIiDgsFRkRERFxWCoyIiIi4rBUZERERMRhqciIiIiIw1KREREREYdl11f2LQjXrg6clJRkchIRERHJq2vv27e7yn+xLzLJyckABAcHm5xERERE8is5ORl/f/9cH7frL40sCDabjVOnTuHr65vrl1DeiaSkJIKDgzl+/Hix/TLK4j7G4j4+KP5j1PgcX3Efo8Z35wzDIDk5maCgIJyccj8TptgfkXFycqJy5cqF9vx+fn7F8pfz74r7GIv7+KD4j1Hjc3zFfYwa35251ZGYa3Syr4iIiDgsFRkRERFxWCoyd8jd3Z2xY8fi7u5udpRCU9zHWNzHB8V/jBqf4yvuY9T4Cl+xP9lXREREii8dkRERERGHpSIjIiIiDktFRkRERByWioyIiIg4LBWZXPz+++907dqVoKAgLBYLy5Ytu+02a9asoVGjRri7u1O9enXmzJlT6DnvVH7Ht2bNGiwWyw2306dPF03gfJo4cSJNmzbF19eXcuXK0a1bN/bt23fb7RYvXkytWrXw8PCgXr16LF++vAjS3pk7GeOcOXNu2IceHh5FlDh/Zs2aRf369bMvtNW8eXNWrFhxy20caf/ld3yOtO9uZtKkSVgsFoYNG3bL9RxpH14vL2N0pP04bty4G7LWqlXrltuYsf9UZHKRkpJCgwYNmDlzZp7Wj4+Pp0uXLtx3333ExsYybNgwnn32WVauXFnISe9Mfsd3zb59+0hISMi+lStXrpAS/jNr165l4MCBbNq0iV9++YXMzEw6dOhASkpKrtts2LCBPn368MwzzxATE0O3bt3o1q0bO3fuLMLkeXcnY4S/rsD593149OjRIkqcP5UrV2bSpEls27aNrVu3cv/99/Pwww+za9eum67vaPsvv+MDx9l319uyZQuffPIJ9evXv+V6jrYP/y6vYwTH2o8RERE5sq5bty7XdU3bf4bcFmAsXbr0luuMGjXKiIiIyLGsV69eRseOHQsxWcHIy/h+++03AzAuXbpUJJkK2tmzZw3AWLt2ba7r9OzZ0+jSpUuOZc2aNTMGDBhQ2PEKRF7GOHv2bMPf37/oQhWw0qVLG//5z39u+pij7z/DuPX4HHXfJScnGzVq1DB++eUXo02bNsbQoUNzXddR92F+xuhI+3Hs2LFGgwYN8ry+WftPR2QKyMaNG2nXrl2OZR07dmTjxo0mJSocDRs2pGLFirRv357169ebHSfPEhMTAQgICMh1HUffh3kZI8CVK1cIDQ0lODj4tkcA7IXVamXBggWkpKTQvHnzm67jyPsvL+MDx9x3AwcOpEuXLjfsm5tx1H2YnzGCY+3HAwcOEBQURLVq1ejXrx/Hjh3LdV2z9l+x/9LIonL69GnKly+fY1n58uVJSkri6tWreHp6mpSsYFSsWJGPP/6YJk2akJ6ezn/+8x/uvfde/vzzTxo1amR2vFuy2WwMGzaMli1bUrdu3VzXy20f2ut5QH+X1zHWrFmTL774gvr165OYmMiUKVNo0aIFu3btKtQvV71TcXFxNG/enLS0NHx8fFi6dCl16tS56bqOuP/yMz5H23cACxYsIDo6mi1btuRpfUfch/kdoyPtx2bNmjFnzhxq1qxJQkIC48ePp3Xr1uzcuRNfX98b1jdr/6nISJ7UrFmTmjVrZt9v0aIFhw4dYurUqfzvf/8zMdntDRw4kJ07d95ybtfR5XWMzZs3z/EXf4sWLahduzaffPIJEyZMKOyY+VazZk1iY2NJTEzk66+/pn///qxduzbXN3tHk5/xOdq+O378OEOHDuWXX36x25NZ/6k7GaMj7cfOnTtn/3f9+vVp1qwZoaGhLFq0iGeeecbEZDmpyBSQChUqcObMmRzLzpw5g5+fn8MfjcnNXXfdZfflYNCgQfzwww/8/vvvt/1rJ7d9WKFChcKM+I/lZ4zXc3V1JTIykoMHDxZSun/Gzc2N6tWrA9C4cWO2bNnCtGnT+OSTT25Y1xH3X37Gdz1733fbtm3j7NmzOY7YWq1Wfv/9d6KiokhPT8fZ2TnHNo62D+9kjNez9/34d6VKlSI8PDzXrGbtP50jU0CaN2/Or7/+mmPZL7/8csv5bkcXGxtLxYoVzY5xU4ZhMGjQIJYuXcrq1aupWrXqbbdxtH14J2O8ntVqJS4uzm734/VsNhvp6ek3fczR9t/N3Gp817P3fde2bVvi4uKIjY3NvjVp0oR+/foRGxt70zd4R9uHdzLG69n7fvy7K1eucOjQoVyzmrb/CvVUYgeWnJxsxMTEGDExMQZgfPDBB0ZMTIxx9OhRwzAM49VXXzX+9a9/Za9/+PBhw8vLy3j55ZeNPXv2GDNnzjScnZ2Nn376yawh3FJ+xzd16lRj2bJlxoEDB4y4uDhj6NChhpOTk7Fq1SqzhnBLL7zwguHv72+sWbPGSEhIyL6lpqZmr/Ovf/3LePXVV7Pvr1+/3nBxcTGmTJli7Nmzxxg7dqzh6upqxMXFmTGE27qTMY4fP95YuXKlcejQIWPbtm1G7969DQ8PD2PXrl1mDOGWXn31VWPt2rVGfHy8sWPHDuPVV181LBaL8fPPPxuG4fj7L7/jc6R9l5vrP9Hj6PvwZm43RkfajyNGjDDWrFljxMfHG+vXrzfatWtnlC1b1jh79qxhGPaz/1RkcnHt48bX3/r3728YhmH079/faNOmzQ3bNGzY0HBzczOqVatmzJ49u8hz51V+x/fvf//bCAsLMzw8PIyAgADj3nvvNVavXm1O+Dy42diAHPukTZs22eO9ZtGiRUZ4eLjh5uZmREREGD/++GPRBs+HOxnjsGHDjJCQEMPNzc0oX7688cADDxjR0dFFHz4Pnn76aSM0NNRwc3MzAgMDjbZt22a/yRuG4++//I7PkfZdbq5/k3f0fXgztxujI+3HXr16GRUrVjTc3NyMSpUqGb169TIOHjyY/bi97D+LYRhG4R7zERERESkcOkdGREREHJaKjIiIiDgsFRkRERFxWCoyIiIi4rBUZERERMRhqciIiIiIw1KREREREYelIiMiJcKaNWuwWCxcvnzZ7CgiUoBUZESkSFmtVlq0aMGjjz6aY3liYiLBwcG8/vrrhfK6LVq0ICEhAX9//0J5fhExh67sKyJFbv/+/TRs2JDPPvuMfv36AfDEE0+wfft2tmzZgpubm8kJRcRR6IiMiBS58PBwJk2axODBg0lISODbb79lwYIF/Pe//821xLzyyiuEh4fj5eVFtWrVePPNN8nMzAT++ibwdu3a0bFjR679bXbx4kUqV67MmDFjgBunlo4ePUrXrl0pXbo03t7eREREsHz58sIfvIgUKBezA4hIyTR48GCWLl3Kv/71L+Li4hgzZgwNGjTIdX1fX1/mzJlDUFAQcXFxPPfcc/j6+jJq1CgsFgtffvkl9erVY/r06QwdOpTnn3+eSpUqZReZ6w0cOJCMjAx+//13vL292b17Nz4+PoU1XBEpJJpaEhHT7N27l9q1a1OvXj2io6Nxccn731ZTpkxhwYIFbN26NXvZ4sWLeeKJJxg2bBgzZswgJiaGGjVqAH8dkbnvvvu4dOkSpUqVon79+jz22GOMHTu2wMclIkVHU0siYpovvvgCLy8v4uPjOXHiBADPP/88Pj4+2bdrFi5cSMuWLalQoQI+Pj688cYbHDt2LMfz9ejRg0ceeYRJkyYxZcqU7BJzM0OGDOHtt9+mZcuWjB07lh07dhTOIEWkUKnIiIgpNmzYwNSpU/nhhx+46667eOaZZzAMg7feeovY2NjsG8DGjRvp168fDzzwAD/88AMxMTG8/vrrZGRk5HjO1NRUtm3bhrOzMwcOHLjl6z/77LMcPnw4e2qrSZMmzJgxo7CGKyKFREVGRIpcamoqTz75JC+88AL33Xcfn3/+OZs3b+bjjz+mXLlyVK9ePfsGf5We0NBQXn/9dZo0aUKNGjU4evToDc87YsQInJycWLFiBdOnT2f16tW3zBEcHMzzzz/PkiVLGDFiBJ999lmhjFdECo+KjIgUudGjR2MYBpMmTQKgSpUqTJkyhVGjRnHkyJEb1q9RowbHjh1jwYIFHDp0iOnTp7N06dIc6/z444988cUXzJ07l/bt2/Pyyy/Tv39/Ll26dNMMw4YNY+XKlcTHxxMdHc1vv/1G7dq1C3ysIlK4dLKviBSptWvX0rZtW9asWUOrVq1yPNaxY0eysrJYtWoVFoslx2OjRo3iiy++ID09nS5dunD33Xczbtw4Ll++zLlz56hXrx5Dhw5l9OjRAGRmZtK8eXPCwsJYuHDhDSf7Dh48mBUrVnDixAn8/Pzo1KkTU6dOpUyZMkX2sxCRf05FRkRERByWppZERETEYanIiIiIiMNSkRERERGHpSIjIiIiDktFRkRERByWioyIiIg4LBUZERERcVgqMiIiIuKwVGRERETEYanIiIiIiMNSkRERERGHpSIjIiIiDuv/A9e35220Xd/CAAAAAElFTkSuQmCC", + "text/plain": "
" + }, + "metadata": {} + } + ], + "id": "591678dd-b904-4a8d-855a-37b5b1829df9" + }, + { + "cell_type": "code", + "source": "# Dependency resolution submodule consumer\nprint(x_values)", + "metadata": { + "trusted": true + }, + "execution_count": null, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": "Output:\ntensor([[-2.1823]], grad_fn=)\n" + } + ], + "id": "d8b17425-8ff3-4997-bc4f-dee428909dda" + }, + { + "cell_type": "code", + "source": "", + "metadata": { + "trusted": true + }, + "execution_count": null, + "outputs": [], + "id": "35a36bb3-eef9-44a7-8276-2f4d0b4f8c6c" + } + ] + } +} \ No newline at end of file diff --git a/vreapis/tests/resources/notebooks/laserfarm.json b/vreapis/tests/resources/notebooks/laserfarm.json new file mode 100644 index 00000000..9eff9c87 --- /dev/null +++ b/vreapis/tests/resources/notebooks/laserfarm.json @@ -0,0 +1,178 @@ +{ + "save": false, + "kernel": "ipython", + "cell_index": 4, + "notebook": { + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.6" + } + }, + "nbformat_minor": 5, + "nbformat": 4, + "cells": [ + { + "cell_type": "code", + "source": "import fnmatch\nimport json\nimport getpass\nimport os\nimport pathlib\nimport datetime\nimport laspy\n\n\nimport time\nimport requests\n \nfrom dask.distributed import LocalCluster, SSHCluster \nfrom laserfarm import Retiler, DataProcessing, GeotiffWriter, MacroPipeline\nfrom laserfarm.remote_utils import get_wdclient, get_info_remote, list_remote", + "metadata": { + "trusted": true + }, + "execution_count": 1, + "outputs": [ + { + "ename": "ModuleNotFoundError", + "evalue": "No module named 'laspy'", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mModuleNotFoundError\u001b[0m Traceback (most recent call last)", + "Cell \u001b[0;32mIn[1], line 7\u001b[0m\n\u001b[1;32m 5\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mpathlib\u001b[39;00m\n\u001b[1;32m 6\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mdatetime\u001b[39;00m\n\u001b[0;32m----> 7\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mlaspy\u001b[39;00m\n\u001b[1;32m 10\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mtime\u001b[39;00m\n\u001b[1;32m 11\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mrequests\u001b[39;00m\n", + "\u001b[0;31mModuleNotFoundError\u001b[0m: No module named 'laspy'" + ] + } + ], + "id": "8f0e44d1" + }, + { + "cell_type": "markdown", + "source": "## Global Configuration", + "metadata": {}, + "id": "7404fb96" + }, + { + "cell_type": "code", + "source": "import fnmatch\nimport json\nimport getpass\nimport os\nimport pathlib\nimport datetime\n \nfrom dask.distributed import LocalCluster, SSHCluster \nfrom laserfarm import Retiler, DataProcessing, GeotiffWriter, MacroPipeline\nfrom laserfarm.remote_utils import get_wdclient, get_info_remote, list_remote\n\nparam_username = ''\nif 'JUPYTERHUB_USER' in os.environ:\n param_username = os.environ['JUPYTERHUB_USER']\n \nconf_remote_path_root = '/webdav/LAZ'\nconf_remote_path_split = pathlib.Path(conf_remote_path_root + '/split_'+param_username)\nconf_remote_path_retiled = pathlib.Path(conf_remote_path_root + '/retiled_'+param_username)\nconf_remote_path_norm = pathlib.Path(conf_remote_path_root + '/norm_'+param_username)\nconf_remote_path_targets = pathlib.Path(conf_remote_path_root + '/targets_'+param_username)\nconf_local_tmp = pathlib.Path('/tmp')\nconf_remote_path_ahn = conf_remote_path_root\n\n\nparam_hostname = 'https://lfw-ds001-i022.lifewatch.dev:32443/'\nparam_login = '20BNXDdL8mg24OaD'\nparam_password = 'zDoy0hNKkcnsdsQ@OYAVd'\n\nconf_feature_name = 'perc_95_normalized_height'\nconf_validate_precision = '0.001'\nconf_tile_mesh_size = '10.'\nconf_filter_type= 'select_equal'\nconf_attribute = 'raw_classification'\nconf_min_x = '-113107.81'\nconf_max_x = '398892.19'\nconf_min_y = '214783.87'\nconf_max_y = '726783.87'\nconf_n_tiles_side = '512'\nconf_apply_filter_value = '1'\nconf_laz_compression_factor = '7'\nconf_max_filesize = '262144000' # desired max file size (in bytes)\n\nconf_wd_opts = { 'webdav_hostname': param_hostname, 'webdav_login': param_login, 'webdav_password': param_password}", + "metadata": { + "trusted": true + }, + "execution_count": null, + "outputs": [], + "id": "18041536" + }, + { + "cell_type": "markdown", + "source": "## Fetching Laz Files from remote WebDAV", + "metadata": {}, + "id": "d03d1b76" + }, + { + "cell_type": "code", + "source": "# Fetch Laz Files\nprint(conf_remote_path_ahn)\nlaz_files = [f for f in list_remote(get_wdclient(conf_wd_opts), pathlib.Path(conf_remote_path_ahn).as_posix())\n if f.lower().endswith('.laz')]\nprint(laz_files)", + "metadata": { + "trusted": true + }, + "execution_count": null, + "outputs": [ + { + "ename": "NameError", + "evalue": "name 'conf_remote_path_ahn' is not defined", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)", + "Cell \u001b[0;32mIn [2], line 2\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[38;5;66;03m# Fetch Laz Files\u001b[39;00m\n\u001b[0;32m----> 2\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[43mconf_remote_path_ahn\u001b[49m)\n\u001b[1;32m 3\u001b[0m laz_files \u001b[38;5;241m=\u001b[39m [f \u001b[38;5;28;01mfor\u001b[39;00m f \u001b[38;5;129;01min\u001b[39;00m list_remote(get_wdclient(conf_wd_opts), pathlib\u001b[38;5;241m.\u001b[39mPath(conf_remote_path_ahn)\u001b[38;5;241m.\u001b[39mas_posix())\n\u001b[1;32m 4\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m f\u001b[38;5;241m.\u001b[39mlower()\u001b[38;5;241m.\u001b[39mendswith(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m.laz\u001b[39m\u001b[38;5;124m'\u001b[39m)]\n\u001b[1;32m 5\u001b[0m \u001b[38;5;28mprint\u001b[39m(laz_files)\n", + "\u001b[0;31mNameError\u001b[0m: name 'conf_remote_path_ahn' is not defined" + ] + } + ], + "id": "573679af" + }, + { + "cell_type": "markdown", + "source": "## Splitting big files into smaller files before retiling\nThis step can be added if the original files are too large for normal VMs to process", + "metadata": {}, + "id": "bc0773b1" + }, + { + "cell_type": "code", + "source": "# split big files\n\nimport numpy as np\n\ndef save_chunk_to_laz_file(in_filename, \n out_filename, \n offset, \n n_points):\n \"\"\"Read points from a LAS/LAZ file and write them to a new file.\"\"\"\n \n points = np.array([])\n \n with laspy.open(in_filename) as in_file:\n with laspy.open(out_filename, \n mode=\"w\", \n header=in_file.header) as out_file:\n in_file.seek(offset)\n points = in_file.read_points(n_points)\n out_file.write_points(points)\n return len(points)\n\ndef split_strategy(filename, max_filesize):\n \"\"\"Set up splitting strategy for a LAS/LAZ file.\"\"\"\n with laspy.open(filename) as f:\n bytes_per_point = (\n f.header.point_format.num_standard_bytes +\n f.header.point_format.num_extra_bytes\n )\n n_points = f.header.point_count\n n_points_target = int(\n max_filesize * int(conf_laz_compression_factor) / bytes_per_point\n )\n stem, ext = os.path.splitext(filename)\n return [\n (filename, f\"{stem}-{n}{ext}\", offset, n_points_target)\n for n, offset in enumerate(range(0, n_points, n_points_target))\n ]\n\nfrom webdav3.client import Client\n\nclient = Client(conf_wd_opts)\nclient.mkdir(conf_remote_path_split.as_posix())\n\n\nremote_path_split = conf_remote_path_split\n\n\nfor file in laz_files:\n print('Splitting: '+file )\n client.download_sync(remote_path=os.path.join(conf_remote_path_ahn,file), local_path=file)\n inps = split_strategy(file, int(conf_max_filesize))\n for inp in inps:\n save_chunk_to_laz_file(*inp)\n client.upload_sync(remote_path=os.path.join(conf_remote_path_split,file), local_path=file)\n\n for f in os.listdir('.'):\n if not f.endswith('.LAZ'):\n continue\n os.remove(os.path.join('.', f))\n \nsplit_laz_files = laz_files", + "metadata": { + "trusted": true + }, + "execution_count": null, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": "Splitting: C_18HZ2.LAZ\n" + }, + { + "ename": "NameError", + "evalue": "name 'laspy' is not defined", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)", + "Input \u001b[0;32mIn [9]\u001b[0m, in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[1;32m 49\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mSplitting: \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;241m+\u001b[39mfile )\n\u001b[1;32m 50\u001b[0m client\u001b[38;5;241m.\u001b[39mdownload_sync(remote_path\u001b[38;5;241m=\u001b[39mos\u001b[38;5;241m.\u001b[39mpath\u001b[38;5;241m.\u001b[39mjoin(conf_remote_path_ahn,file), local_path\u001b[38;5;241m=\u001b[39mfile)\n\u001b[0;32m---> 51\u001b[0m inps \u001b[38;5;241m=\u001b[39m \u001b[43msplit_strategy\u001b[49m\u001b[43m(\u001b[49m\u001b[43mfile\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mint\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43mconf_max_filesize\u001b[49m\u001b[43m)\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 52\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m inp \u001b[38;5;129;01min\u001b[39;00m inps:\n\u001b[1;32m 53\u001b[0m save_chunk_to_laz_file(\u001b[38;5;241m*\u001b[39minp)\n", + "Input \u001b[0;32mIn [9]\u001b[0m, in \u001b[0;36msplit_strategy\u001b[0;34m(filename, max_filesize)\u001b[0m\n\u001b[1;32m 22\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21msplit_strategy\u001b[39m(filename, max_filesize):\n\u001b[1;32m 23\u001b[0m \u001b[38;5;124;03m\"\"\"Set up splitting strategy for a LAS/LAZ file.\"\"\"\u001b[39;00m\n\u001b[0;32m---> 24\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m \u001b[43mlaspy\u001b[49m\u001b[38;5;241m.\u001b[39mopen(filename) \u001b[38;5;28;01mas\u001b[39;00m f:\n\u001b[1;32m 25\u001b[0m bytes_per_point \u001b[38;5;241m=\u001b[39m (\n\u001b[1;32m 26\u001b[0m f\u001b[38;5;241m.\u001b[39mheader\u001b[38;5;241m.\u001b[39mpoint_format\u001b[38;5;241m.\u001b[39mnum_standard_bytes \u001b[38;5;241m+\u001b[39m\n\u001b[1;32m 27\u001b[0m f\u001b[38;5;241m.\u001b[39mheader\u001b[38;5;241m.\u001b[39mpoint_format\u001b[38;5;241m.\u001b[39mnum_extra_bytes\n\u001b[1;32m 28\u001b[0m )\n\u001b[1;32m 29\u001b[0m n_points \u001b[38;5;241m=\u001b[39m f\u001b[38;5;241m.\u001b[39mheader\u001b[38;5;241m.\u001b[39mpoint_count\n", + "\u001b[0;31mNameError\u001b[0m: name 'laspy' is not defined" + ] + } + ], + "id": "c074eba8" + }, + { + "cell_type": "markdown", + "source": "## Retiling of big files into smaller tiles", + "metadata": {}, + "id": "f416e842" + }, + { + "cell_type": "code", + "source": "# Retiling\nsplit_laz_files\nremote_path_retiled = str(conf_remote_path_retiled)\n\ngrid_retile = {\n 'min_x': float(conf_min_x),\n 'max_x': float(conf_max_x),\n 'min_y': float(conf_min_y),\n 'max_y': float(conf_max_y),\n 'n_tiles_side': int(conf_n_tiles_side)\n}\n\nretiling_input = {\n 'setup_local_fs': {'tmp_folder': conf_local_tmp.as_posix()},\n 'pullremote': conf_remote_path_split.as_posix(),\n 'set_grid': grid_retile,\n 'split_and_redistribute': {},\n 'validate': {},\n 'pushremote': conf_remote_path_retiled.as_posix(),\n 'cleanlocalfs': {}\n}\n\nfor file in split_laz_files:\n retiler = Retiler(file.replace('\"',''),label=file).config(retiling_input).setup_webdav_client(conf_wd_opts)\n retiler_output = retiler.run()", + "metadata": { + "trusted": true + }, + "execution_count": null, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": "2022-07-10 21:10:04,119 - laserfarm.pipeline_remote_data - INFO - Input dir set to /tmp/C_18HZ2.LAZ_input\n2022-07-10 21:10:04,121 - laserfarm.pipeline_remote_data - INFO - Output dir set to /tmp/C_18HZ2.LAZ_output\n2022-07-10 21:10:04,125 - laserfarm.pipeline_remote_data - INFO - Pulling from WebDAV /webdav/pointcloud/split_skoulouzis/C_18HZ2.LAZ ...\n2022-07-10 21:10:05,596 - laserfarm.pipeline_remote_data - INFO - ... pulling completed.\n2022-07-10 21:10:05,597 - laserfarm.retiler - INFO - Setting up the target grid\n2022-07-10 21:10:05,600 - laserfarm.retiler - INFO - Splitting file /tmp/C_18HZ2.LAZ_input/C_18HZ2.LAZ with PDAL ...\n2022-07-10 21:10:06,079 - laserfarm.retiler - INFO - ... splitting completed.\n2022-07-10 21:10:06,083 - laserfarm.retiler - INFO - Redistributing files to tiles ...\n2022-07-10 21:10:06,084 - laserfarm.retiler - INFO - ... file C_18HZ2_1.LAZ to tile_213_285\n2022-07-10 21:10:06,086 - laserfarm.retiler - INFO - ... file C_18HZ2_2.LAZ to tile_212_285\n2022-07-10 21:10:06,089 - laserfarm.retiler - INFO - ... redistributing completed.\n2022-07-10 21:10:06,095 - laserfarm.retiler - INFO - Validating split ...\n2022-07-10 21:10:06,099 - laserfarm.retiler - INFO - ... 257128 points in parent file\n2022-07-10 21:10:06,104 - laserfarm.retiler - INFO - ... 25414 points in C_18HZ2_2.LAZ\n2022-07-10 21:10:06,105 - laserfarm.retiler - INFO - ... 231714 points in C_18HZ2_1.LAZ\n2022-07-10 21:10:06,107 - laserfarm.retiler - INFO - ... split validation completed.\n2022-07-10 21:10:06,109 - laserfarm.pipeline_remote_data - INFO - Pushing to WebDAV /webdav/pointcloud/retiled_skoulouzis ...\n2022-07-10 21:10:12,031 - laserfarm.pipeline_remote_data - INFO - ... pushing completed.\n2022-07-10 21:10:12,032 - laserfarm.pipeline_remote_data - INFO - Removing input and output folders\n2022-07-10 21:10:12,036 - laserfarm.pipeline_remote_data - INFO - Input dir set to /tmp/C_19HZ2.LAZ_input\n2022-07-10 21:10:12,037 - laserfarm.pipeline_remote_data - INFO - Output dir set to /tmp/C_19HZ2.LAZ_output\n2022-07-10 21:10:12,038 - laserfarm.pipeline_remote_data - INFO - Pulling from WebDAV /webdav/pointcloud/split_skoulouzis/C_19HZ2.LAZ ...\n2022-07-10 21:10:13,474 - laserfarm.pipeline_remote_data - INFO - ... pulling completed.\n2022-07-10 21:10:13,475 - laserfarm.retiler - INFO - Setting up the target grid\n2022-07-10 21:10:13,476 - laserfarm.retiler - INFO - Splitting file /tmp/C_19HZ2.LAZ_input/C_19HZ2.LAZ with PDAL ...\n2022-07-10 21:10:13,615 - laserfarm.retiler - INFO - ... splitting completed.\n2022-07-10 21:10:13,616 - laserfarm.retiler - INFO - Redistributing files to tiles ...\n2022-07-10 21:10:13,618 - laserfarm.retiler - INFO - ... file C_19HZ2_1.LAZ to tile_248_285\n2022-07-10 21:10:13,618 - laserfarm.retiler - INFO - ... redistributing completed.\n2022-07-10 21:10:13,619 - laserfarm.retiler - INFO - Validating split ...\n2022-07-10 21:10:13,620 - laserfarm.retiler - INFO - ... 66663 points in parent file\n2022-07-10 21:10:13,621 - laserfarm.retiler - INFO - ... 66663 points in C_19HZ2_1.LAZ\n2022-07-10 21:10:13,622 - laserfarm.retiler - INFO - ... split validation completed.\n2022-07-10 21:10:13,623 - laserfarm.pipeline_remote_data - INFO - Pushing to WebDAV /webdav/pointcloud/retiled_skoulouzis ...\n2022-07-10 21:10:17,817 - laserfarm.pipeline_remote_data - INFO - ... pushing completed.\n2022-07-10 21:10:17,818 - laserfarm.pipeline_remote_data - INFO - Removing input and output folders\n2022-07-10 21:10:17,821 - laserfarm.pipeline_remote_data - INFO - Input dir set to /tmp/C_01GN2.LAZ_input\n2022-07-10 21:10:17,822 - laserfarm.pipeline_remote_data - INFO - Output dir set to /tmp/C_01GN2.LAZ_output\n2022-07-10 21:10:17,824 - laserfarm.pipeline_remote_data - INFO - Pulling from WebDAV /webdav/pointcloud/split_skoulouzis/C_01GN2.LAZ ...\n2022-07-10 21:10:20,467 - laserfarm.pipeline_remote_data - INFO - ... pulling completed.\n2022-07-10 21:10:20,468 - laserfarm.retiler - INFO - Setting up the target grid\n2022-07-10 21:10:20,470 - laserfarm.retiler - INFO - Splitting file /tmp/C_01GN2.LAZ_input/C_01GN2.LAZ with PDAL ...\n2022-07-10 21:10:26,047 - laserfarm.retiler - INFO - ... splitting completed.\n2022-07-10 21:10:26,049 - laserfarm.retiler - INFO - Redistributing files to tiles ...\n2022-07-10 21:10:26,050 - laserfarm.retiler - INFO - ... file C_01GN2_6.LAZ to tile_282_392\n2022-07-10 21:10:26,052 - laserfarm.retiler - INFO - ... file C_01GN2_2.LAZ to tile_278_392\n2022-07-10 21:10:26,053 - laserfarm.retiler - INFO - ... file C_01GN2_4.LAZ to tile_282_391\n2022-07-10 21:10:26,054 - laserfarm.retiler - INFO - ... file C_01GN2_1.LAZ to tile_278_391\n2022-07-10 21:10:26,056 - laserfarm.retiler - INFO - ... file C_01GN2_3.LAZ to tile_279_391\n2022-07-10 21:10:26,057 - laserfarm.retiler - INFO - ... file C_01GN2_7.LAZ to tile_283_392\n2022-07-10 21:10:26,058 - laserfarm.retiler - INFO - ... file C_01GN2_5.LAZ to tile_283_391\n2022-07-10 21:10:26,067 - laserfarm.retiler - INFO - ... redistributing completed.\n2022-07-10 21:10:26,068 - laserfarm.retiler - INFO - Validating split ...\n2022-07-10 21:10:26,069 - laserfarm.retiler - INFO - ... 3516323 points in parent file\n2022-07-10 21:10:26,070 - laserfarm.retiler - INFO - ... 2286839 points in C_01GN2_1.LAZ\n2022-07-10 21:10:26,071 - laserfarm.retiler - INFO - ... 105908 points in C_01GN2_4.LAZ\n2022-07-10 21:10:26,072 - laserfarm.retiler - INFO - ... 193772 points in C_01GN2_6.LAZ\n2022-07-10 21:10:26,073 - laserfarm.retiler - INFO - ... 219926 points in C_01GN2_3.LAZ\n2022-07-10 21:10:26,074 - laserfarm.retiler - INFO - ... 266823 points in C_01GN2_5.LAZ\n2022-07-10 21:10:26,079 - laserfarm.retiler - INFO - ... 407361 points in C_01GN2_7.LAZ\n2022-07-10 21:10:26,080 - laserfarm.retiler - INFO - ... 35694 points in C_01GN2_2.LAZ\n2022-07-10 21:10:26,080 - laserfarm.retiler - INFO - ... split validation completed.\n2022-07-10 21:10:26,081 - laserfarm.pipeline_remote_data - INFO - Pushing to WebDAV /webdav/pointcloud/retiled_skoulouzis ...\n2022-07-10 21:10:41,777 - laserfarm.pipeline_remote_data - INFO - ... pushing completed.\n2022-07-10 21:10:41,778 - laserfarm.pipeline_remote_data - INFO - Removing input and output folders\n2022-07-10 21:10:41,792 - laserfarm.pipeline_remote_data - INFO - Input dir set to /tmp/C_05FN1.LAZ_input\n2022-07-10 21:10:41,793 - laserfarm.pipeline_remote_data - INFO - Output dir set to /tmp/C_05FN1.LAZ_output\n2022-07-10 21:10:41,795 - laserfarm.pipeline_remote_data - INFO - Pulling from WebDAV /webdav/pointcloud/split_skoulouzis/C_05FN1.LAZ ...\n2022-07-10 21:10:43,375 - laserfarm.pipeline_remote_data - INFO - ... pulling completed.\n2022-07-10 21:10:43,377 - laserfarm.retiler - INFO - Setting up the target grid\n2022-07-10 21:10:43,378 - laserfarm.retiler - INFO - Splitting file /tmp/C_05FN1.LAZ_input/C_05FN1.LAZ with PDAL ...\n2022-07-10 21:10:44,596 - laserfarm.retiler - INFO - ... splitting completed.\n2022-07-10 21:10:44,597 - laserfarm.retiler - INFO - Redistributing files to tiles ...\n2022-07-10 21:10:44,599 - laserfarm.retiler - INFO - ... file C_05FN1_4.LAZ to tile_288_379\n2022-07-10 21:10:44,600 - laserfarm.retiler - INFO - ... file C_05FN1_2.LAZ to tile_287_379\n2022-07-10 21:10:44,601 - laserfarm.retiler - INFO - ... file C_05FN1_3.LAZ to tile_288_378\n2022-07-10 21:10:44,603 - laserfarm.retiler - INFO - ... file C_05FN1_1.LAZ to tile_287_378\n2022-07-10 21:10:44,603 - laserfarm.retiler - INFO - ... redistributing completed.\n2022-07-10 21:10:44,606 - laserfarm.retiler - INFO - Validating split ...\n2022-07-10 21:10:44,606 - laserfarm.retiler - INFO - ... 771576 points in parent file\n2022-07-10 21:10:44,607 - laserfarm.retiler - INFO - ... 232882 points in C_05FN1_4.LAZ\n2022-07-10 21:10:44,609 - laserfarm.retiler - INFO - ... 294136 points in C_05FN1_2.LAZ\n2022-07-10 21:10:44,610 - laserfarm.retiler - INFO - ... 55494 points in C_05FN1_3.LAZ\n2022-07-10 21:10:44,614 - laserfarm.retiler - INFO - ... 189064 points in C_05FN1_1.LAZ\n2022-07-10 21:10:44,615 - laserfarm.retiler - INFO - ... split validation completed.\n2022-07-10 21:10:44,616 - laserfarm.pipeline_remote_data - INFO - Pushing to WebDAV /webdav/pointcloud/retiled_skoulouzis ...\n2022-07-10 21:10:54,348 - laserfarm.pipeline_remote_data - INFO - ... pushing completed.\n2022-07-10 21:10:54,350 - laserfarm.pipeline_remote_data - INFO - Removing input and output folders\n2022-07-10 21:10:54,354 - laserfarm.pipeline_remote_data - INFO - Input dir set to /tmp/C_50GZ2.LAZ_input\n2022-07-10 21:10:54,355 - laserfarm.pipeline_remote_data - INFO - Output dir set to /tmp/C_50GZ2.LAZ_output\n2022-07-10 21:10:54,357 - laserfarm.pipeline_remote_data - INFO - Pulling from WebDAV /webdav/pointcloud/split_skoulouzis/C_50GZ2.LAZ ...\n2022-07-10 21:10:55,925 - laserfarm.pipeline_remote_data - INFO - ... pulling completed.\n2022-07-10 21:10:55,927 - laserfarm.retiler - INFO - Setting up the target grid\n2022-07-10 21:10:55,934 - laserfarm.retiler - INFO - Splitting file /tmp/C_50GZ2.LAZ_input/C_50GZ2.LAZ with PDAL ...\n2022-07-10 21:10:56,748 - laserfarm.retiler - INFO - ... splitting completed.\n2022-07-10 21:10:56,749 - laserfarm.retiler - INFO - Redistributing files to tiles ...\n2022-07-10 21:10:56,751 - laserfarm.retiler - INFO - ... file C_50GZ2_1.LAZ to tile_238_166\n2022-07-10 21:10:56,753 - laserfarm.retiler - INFO - ... redistributing completed.\n2022-07-10 21:10:56,755 - laserfarm.retiler - INFO - Validating split ...\n2022-07-10 21:10:56,758 - laserfarm.retiler - INFO - ... 499572 points in parent file\n2022-07-10 21:10:56,760 - laserfarm.retiler - INFO - ... 499572 points in C_50GZ2_1.LAZ\n2022-07-10 21:10:56,763 - laserfarm.retiler - INFO - ... split validation completed.\n2022-07-10 21:10:56,766 - laserfarm.pipeline_remote_data - INFO - Pushing to WebDAV /webdav/pointcloud/retiled_skoulouzis ...\n2022-07-10 21:11:01,220 - laserfarm.pipeline_remote_data - INFO - ... pushing completed.\n2022-07-10 21:11:01,221 - laserfarm.pipeline_remote_data - INFO - Removing input and output folders\n" + } + ], + "id": "8b489f30" + }, + { + "cell_type": "code", + "source": "print(retiler_output)", + "metadata": { + "trusted": true + }, + "execution_count": null, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": "None\n" + } + ], + "id": "986c70f3" + }, + { + "cell_type": "code", + "source": "", + "metadata": { + "trusted": true + }, + "execution_count": null, + "outputs": [], + "id": "c8814681" + } + ] + } +} diff --git a/vreapis/tests/resources/notebooks/r-notebook.json b/vreapis/tests/resources/notebooks/r-notebook.json new file mode 100644 index 00000000..3ff1dea3 --- /dev/null +++ b/vreapis/tests/resources/notebooks/r-notebook.json @@ -0,0 +1,63 @@ +{ + "save": false, + "kernel": "IRkernel", + "cell_index": 2, + "notebook": { + "metadata": { + "kernelspec": { + "display_name": "R", + "language": "R", + "name": "ir" + }, + "language_info": { + "codemirror_mode": "r", + "file_extension": ".r", + "mimetype": "text/x-r-source", + "name": "R", + "pygments_lexer": "r", + "version": "4.3.2" + } + }, + "nbformat_minor": 5, + "nbformat": 4, + "cells": [ + { + "cell_type": "code", + "source": "# inputR\na = 2 \nnumbers <- c(a, 4, 6, 8, 10)\n", + "metadata": { + "tags": [], + "trusted": true + }, + "execution_count": 1, + "outputs": [], + "id": "59b2ee0b-958d-431f-a396-b4b34c77d540" + }, + { + "cell_type": "code", + "source": "#averageR\n\n\naverage <- mean(numbers)", + "metadata": { + "trusted": true + }, + "execution_count": 2, + "outputs": [], + "id": "0a83d716-23c8-4eb2-b246-9a8f4cdf9ac2" + }, + { + "cell_type": "code", + "source": "#print \n\nprint(average)", + "metadata": { + "trusted": true + }, + "execution_count": 3, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": "[1] 6\n" + } + ], + "id": "bc5ace1d-809a-4dc1-b5f7-075a93e8474a" + } + ] + } +} \ No newline at end of file diff --git a/vreapis/tests/resources/notebooks/test_!.json b/vreapis/tests/resources/notebooks/test_!.json new file mode 100644 index 00000000..c7735f45 --- /dev/null +++ b/vreapis/tests/resources/notebooks/test_!.json @@ -0,0 +1,158 @@ +{ + "save": false, + "kernel": "ipython", + "cell_index": 1, + "notebook": { + "metadata": { + "kernelspec": { + "display_name": "Python [conda env:jupyterlab] *", + "language": "python", + "name": "conda-env-jupyterlab-py" + }, + "language_info": { + "name": "python", + "version": "3.10.12", + "mimetype": "text/x-python", + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "pygments_lexer": "ipython3", + "nbconvert_exporter": "python", + "file_extension": ".py" + } + }, + "nbformat_minor": 5, + "nbformat": 4, + "cells": [ + { + "cell_type": "code", + "source": "!conda install --yes -c conda-forge pdal python-pdal gdal dask\n!conda activate --yes pdalpy\n!pip install dask distributed laserfarm\n!conda upgrade --yes numpy", + "metadata": { + "trusted": true + }, + "execution_count": null, + "outputs": [], + "id": "af2a436e-6682-40cb-bd4c-fc7aa0a719ba" + }, + { + "cell_type": "code", + "source": "# conf \nimport os\n\nconf_data_folder = os.path.join('/tmp','data')\n\nconf_feature_name = 'perc_95_normalized_height'\nconf_validate_precision = '0.001'\nconf_tile_mesh_size = '10.'\nconf_filter_type= 'select_equal'\nconf_attribute = 'raw_classification'\nconf_min_x = '-113107.81'\nconf_max_x = '398892.19'\nconf_min_y = '214783.87'\nconf_max_y = '726783.87'\nconf_n_tiles_side = '512'\nconf_apply_filter_value = '1'\nconf_laz_compression_factor = '7'\nconf_max_filesize = '262144000' # desired max file size (in bytes)\n\n", + "metadata": { + "trusted": true + }, + "execution_count": 1, + "outputs": [], + "id": "bca08f46-0546-47a6-b0eb-d9fa3a5c9265" + }, + { + "cell_type": "code", + "source": "# Create file \n\nL = [\"a\\n\", \"b\\n\", \"c\\n\"]\nfile_path = os.path.join(conf_data_folder,'hello.txt')\nfp = open(file_path, 'w')\nfp.writelines(L)\nfp.close()", + "metadata": { + "trusted": true + }, + "execution_count": 2, + "outputs": [], + "id": "00dd23e8-531e-4dd8-90aa-43573c240a48" + }, + { + "cell_type": "code", + "source": "#read file lines\n\nf = open(file_path, 'r')\nlines = f.readlines()\nf.close()", + "metadata": { + "trusted": true + }, + "execution_count": 3, + "outputs": [], + "id": "452c7037-2686-46bc-90d2-1d681b2b4a72" + }, + { + "cell_type": "code", + "source": "#loop file lines\n\ncount = 0\n# Strips the newline character\nfor l in lines:\n count += 1\n print(\"Line{}: {}\".format(count, l.strip()))", + "metadata": { + "trusted": true + }, + "execution_count": 4, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": "Line1: a\nLine2: b\nLine3: c\n" + } + ], + "id": "918da96c-0e5d-4c49-8604-80dbea97f034" + }, + { + "cell_type": "code", + "source": "# Add1\n\na = count + 1", + "metadata": { + "trusted": true + }, + "execution_count": null, + "outputs": [], + "id": "25bdee30-184c-4fb9-9f60-6cb2bf3830a4" + }, + { + "cell_type": "code", + "source": "#Anti-pattern \nsome_list = range(count, a+1)\n\nmsg = '1'\n\n", + "metadata": { + "trusted": true + }, + "execution_count": 11, + "outputs": [], + "id": "c1ee0030-e097-49e9-b67c-69f21a8a6da3" + }, + { + "cell_type": "code", + "source": "# input lists\n\nlist_of_paths = [\"/webdav/LAZ/targets_myname\",\"/webdav/LAZ/targets_myname\",\"/webdav/LAZ/targets_myname\",\"/webdav/LAZ/targets_myname\",\"/webdav/LAZ/targets_myname\",\"/webdav/LAZ/targets_myname\",\"/webdav/LAZ/targets_myname\",\"/webdav/LAZ/targets_myname\",\"/webdav/LAZ/targets_myname\",\"/webdav/LAZ/targets_myname\"]\nlist_of_ints = [1,2,35,6,65]\nprint(msg)", + "metadata": { + "trusted": true + }, + "execution_count": 6, + "outputs": [], + "id": "7b8027f6-7be6-43a9-958f-2337a0ed0733" + }, + { + "cell_type": "code", + "source": "#loop list \n\nfor l in list_of_paths:\n print(l)", + "metadata": { + "trusted": true + }, + "execution_count": 6, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": "/webdav/LAZ/targets_myname\n/webdav/LAZ/targets_myname\n/webdav/LAZ/targets_myname\n/webdav/LAZ/targets_myname\n/webdav/LAZ/targets_myname\n/webdav/LAZ/targets_myname\n/webdav/LAZ/targets_myname\n/webdav/LAZ/targets_myname\n/webdav/LAZ/targets_myname\n/webdav/LAZ/targets_myname\n" + } + ], + "id": "ffdc9192-db4e-44f4-8a13-62a450ac3aa1" + }, + { + "cell_type": "code", + "source": "#loop int list \n\nfor i in list_of_ints:\n a = i -1\n print(a)", + "metadata": { + "trusted": true + }, + "execution_count": 8, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": "2\n3\n36\n7\n66\n" + } + ], + "id": "2899e58e-2c8b-46f4-a7a5-ed02156371a4" + }, + { + "cell_type": "code", + "source": "", + "metadata": { + "trusted": true + }, + "execution_count": null, + "outputs": [], + "id": "3ea24b6d-f13c-42d9-a340-42cd0aa7ae4c" + } + ] + } +} \ No newline at end of file diff --git a/vreapis/tests/resources/notebooks/test_R_param_in_cell.json b/vreapis/tests/resources/notebooks/test_R_param_in_cell.json new file mode 100644 index 00000000..b4b3112c --- /dev/null +++ b/vreapis/tests/resources/notebooks/test_R_param_in_cell.json @@ -0,0 +1,36 @@ +{ + "save": false, + "kernel": "IRkernel", + "cell_index": 0, + "notebook": { + "metadata": { + "kernelspec": { + "display_name": "R", + "language": "R", + "name": "ir" + }, + "language_info": { + "codemirror_mode": "r", + "file_extension": ".r", + "mimetype": "text/x-r-source", + "name": "R", + "pygments_lexer": "r", + "version": "4.3.1" + } + }, + "nbformat_minor": 5, + "nbformat": 4, + "cells": [ + { + "cell_type": "code", + "source": "# print param R in cell\nparam_b = 'value b'\nprint(param_b)", + "metadata": { + "trusted": true + }, + "execution_count": null, + "outputs": [], + "id": "f1a6b009-966b-4c7d-bb2a-e2381ead781f" + } + ] + } +} \ No newline at end of file diff --git a/vreapis/tests/resources/notebooks/test_R_param_outside_cell.json b/vreapis/tests/resources/notebooks/test_R_param_outside_cell.json new file mode 100644 index 00000000..b0cf19b9 --- /dev/null +++ b/vreapis/tests/resources/notebooks/test_R_param_outside_cell.json @@ -0,0 +1,46 @@ +{ + "save": false, + "kernel": "IRkernel", + "cell_index": 1, + "notebook": { + "metadata": { + "kernelspec": { + "display_name": "R", + "language": "R", + "name": "ir" + }, + "language_info": { + "codemirror_mode": "r", + "file_extension": ".r", + "mimetype": "text/x-r-source", + "name": "R", + "pygments_lexer": "r", + "version": "4.3.1" + } + }, + "nbformat_minor": 5, + "nbformat": 4, + "cells": [ + { + "cell_type": "code", + "source": "param_a = 'value a'", + "metadata": { + "trusted": true + }, + "execution_count": 1, + "outputs": [], + "id": "1b784005-98ec-48d0-8d6d-339def7e42cb" + }, + { + "cell_type": "code", + "source": "# print param R outside cell\nprint(param_a)", + "metadata": { + "trusted": true + }, + "execution_count": null, + "outputs": [], + "id": "e7657fee-d6ef-4748-83d9-997c713afb52" + } + ] + } +} \ No newline at end of file diff --git a/vreapis/tests/resources/notebooks/test_cell_header_R.json b/vreapis/tests/resources/notebooks/test_cell_header_R.json new file mode 100644 index 00000000..fa4bd032 --- /dev/null +++ b/vreapis/tests/resources/notebooks/test_cell_header_R.json @@ -0,0 +1,87 @@ +{ + "save": false, + "kernel": "IRkernel", + "cell_index": 1, + "notebook": { + "metadata": { + "kernelspec": { + "display_name": "R", + "language": "R", + "name": "ir" + }, + "language_info": { + "codemirror_mode": "r", + "file_extension": ".r", + "mimetype": "text/x-r-source", + "name": "R", + "pygments_lexer": "r", + "version": "4.3.1" + } + }, + "nbformat_minor": 5, + "nbformat": 4, + "cells": [ + { + "cell_type": "code", + "source": [ + "# params\n", + "param_string = 'param_string value'\n", + "param_string_with_comment = 'param_string value' # comment\n", + "param_int = 1\n", + "param_float = 1.1" + ], + "metadata": { + "trusted": true + }, + "execution_count": null, + "outputs": [], + "id": "de7cc7cf-56c9-4538-92be-1a258e5ee812" + }, + { + "cell_type": "code", + "source": [ + "# Param check\n", + "# ---\n", + "# NaaVRE:\n", + "# cell:\n", + "# inputs:\n", + "# - a\n", + "# - b: String\n", + "# - c:\n", + "# type: Integer\n", + "# outputs:\n", + "# - e\n", + "# - f: Float\n", + "# - g:\n", + "# type: List\n", + "# params:\n", + "# - param_h\n", + "# - param_i: Float\n", + "# - param_j:\n", + "# type: List\n", + "# - param_k:\n", + "# type: List\n", + "# default_value: 'param_k value'\n", + "# confs:\n", + "# - conf_l:\n", + "# assignation: 'conf_l = 1'\n", + "# dependencies:\n", + "# - name: numpy\n", + "# asname: np\n", + "# ...\n", + "print(param_string)\n", + "print(param_string_with_comment)\n", + "print(param_int)\n", + "print(param_float)" + ], + "metadata": { + "trusted": true, + "tags": [] + }, + "execution_count": null, + "outputs": [], + "id": "9824a80c-b00b-441c-8152-7b2e671e081b" + } + ] + } +} diff --git a/vreapis/tests/resources/notebooks/test_conf_nesting.json b/vreapis/tests/resources/notebooks/test_conf_nesting.json new file mode 100644 index 00000000..ec314a3e --- /dev/null +++ b/vreapis/tests/resources/notebooks/test_conf_nesting.json @@ -0,0 +1,53 @@ +{ + "cell_index": 1, + "kernel": "ipython", + "notebook": { + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "8684f3d5-5269-4caf-a594-1b6549e75732", + "metadata": { + "tags": [], + "trusted": true + }, + "outputs": [], + "source": "conf_a = \"a\"\nconf_b = conf_a + \"b\"\nconf_c = conf_b + \"c\"" + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6f9efd5a-a9ee-45d1-8f24-29701890e98d", + "metadata": { + "tags": [], + "trusted": true + }, + "outputs": [], + "source": "conf_c" + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.8" + }, + "toc-showcode": true + }, + "nbformat": 4, + "nbformat_minor": 5 + }, + "save": false +} \ No newline at end of file diff --git a/vreapis/tests/resources/notebooks/test_new_line.json b/vreapis/tests/resources/notebooks/test_new_line.json new file mode 100644 index 00000000..f9ddc618 --- /dev/null +++ b/vreapis/tests/resources/notebooks/test_new_line.json @@ -0,0 +1,48 @@ +{ + "save": false, + "kernel": "ipython", + "cell_index": 1, + "notebook": { + "metadata": { + "language_info": { + "name": "python", + "version": "3.11.6", + "mimetype": "text/x-python", + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "pygments_lexer": "ipython3", + "nbconvert_exporter": "python", + "file_extension": ".py" + }, + "kernelspec": { + "name": "python3", + "display_name": "Python 3 (ipykernel)", + "language": "python" + } + }, + "nbformat_minor": 5, + "nbformat": 4, + "cells": [ + { + "cell_type": "code", + "source": "#Conf_bug\n\nconf_herwijnen = ['radar_volume_full_herwijnen',\n 1.0,\n 'https://api.dataplatform.knmi.nl/open-data/v1/datasets/radar_volume_full_herwijnen/versions/1.0/files',\n 'NL/HRW'] \nconf_denhelder = ['radar_volume_full_denhelder',\n 2.0,\n 'https://api.dataplatform.knmi.nl/open-data/v1/datasets/radar_volume_denhelder/versions/2.0/files',\n 'NL/DHL']\nconf_radars = {'herwijnen' : conf_herwijnen,\n 'denhelder' : conf_denhelder}", + "metadata": { + "trusted": true + }, + "execution_count": null, + "outputs": [], + "id": "1781e699-c90a-43f2-8890-11ac185422bd" + }, + { + "cell_type": "code", + "source": "#Print_cong\n\nprint(conf_herwijnen)\nprint(conf_denhelder)\nprint(conf_radars)", + "metadata": {}, + "execution_count": null, + "outputs": [], + "id": "1b887d36-e009-46c0-9bc8-cbf633fb597d" + } + ] + } +} diff --git a/vreapis/tests/resources/notebooks/test_notebook.ipynb b/vreapis/tests/resources/notebooks/test_notebook.ipynb new file mode 100644 index 00000000..d632ab3e --- /dev/null +++ b/vreapis/tests/resources/notebooks/test_notebook.ipynb @@ -0,0 +1,228 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "e4f3e05b-90d9-4f19-85ef-413125f854ca", + "metadata": {}, + "outputs": [], + "source": [ + "# Configurations\n", + "\n", + "import fnmatch\n", + "import json\n", + "import getpass\n", + "import os\n", + "import pathlib\n", + "import datetime\n", + " \n", + "from dask.distributed import LocalCluster, SSHCluster \n", + "from laserfarm import Retiler, DataProcessing, GeotiffWriter, MacroPipeline\n", + "from laserfarm.remote_utils import get_wdclient, get_info_remote, list_remote\n", + "\n", + "conf_remote_path_root = pathlib.Path('/webdav')\n", + "conf_remote_path_ahn = pathlib.Path('/webdav/ahn')\n", + "conf_remote_path_retiled = pathlib.Path('/webdav/retiled/')\n", + "conf_remote_path_targets = pathlib.Path('/webdav/targets')\n", + "conf_local_tmp = pathlib.Path('/tmp')\n", + "\n", + "param_hostname = 'https://param_hostname'\n", + "param_login = 'param_login'\n", + "param_password = 'param_password'\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "aceacd5f-e791-444f-8526-62b355b5a967", + "metadata": {}, + "outputs": [], + "source": [ + "# Fetch 1\n", + "laz_files = [f for f in list_remote(get_wdclient(conf_wd_opts), conf_remote_path_ahn.as_posix())\n", + " if f.lower().endswith('.laz')]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d28f21bb-21c6-4838-9cb8-2c174d0b70f3", + "metadata": {}, + "outputs": [], + "source": [ + "# fetch\n", + "import time\n", + "import requests\n", + "from laserfarm.remote_utils import get_wdclient, get_info_remote, list_remote\n", + " \n", + "laz_files = [f for f in list_remote(get_wdclient(conf_wd_opts), conf_remote_path_ahn.as_posix())\n", + " if f.lower().endswith('.laz')]\n", + "\n", + "def send_annotation(start=None,end=None,message=None,tags=None):\n", + " if not tags:\n", + " tags = []\n", + " \n", + " tags.append(theNotebook)\n", + " \n", + " headers = {\n", + " 'Accept':'application/json',\n", + " 'Content-Type': 'application/json',\n", + " }\n", + " \n", + " data ={\n", + " \"dashboardId\":1,\n", + " # \"panelId\":8,\n", + " \"time\":start,\n", + " \"timeEnd\":end,\n", + " \"created\": end,\n", + " \"tags\":tags,\n", + " \"text\": message\n", + " }\n", + " resp = requests.post(grafana_base_url+'/api/annotations',verify=False,auth=('admin', grafana_pwd),headers=headers,json=data)\n", + " \n", + " data ={\n", + " \"dashboardId\":2,\n", + " # \"panelId\":8,\n", + " \"time\":start,\n", + " \"timeEnd\":end,\n", + " \"created\": end,\n", + " \"tags\":tags,\n", + " \"text\": message\n", + " }\n", + " resp = requests.post(grafana_base_url+'/api/annotations',verify=False,auth=('admin', grafana_pwd),headers=headers,json=data)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "75bcb98f-22d0-47d1-a63f-cef4be7a4733", + "metadata": {}, + "outputs": [], + "source": [ + "# Feature Extraction spi\n", + "from laserfarm import Retiler, DataProcessing, GeotiffWriter, MacroPipeline\n", + "import time\n", + "import requests \n", + " \n", + "def send_annotation(start=None,end=None,message=None,tags=None):\n", + " if not tags:\n", + " tags = []\n", + " \n", + " tags.append(theNotebook)\n", + " \n", + " headers = {\n", + " 'Accept':'application/json',\n", + " 'Content-Type': 'application/json',\n", + " }\n", + " \n", + " data ={\n", + " \"dashboardId\":1,\n", + " # \"panelId\":8,\n", + " \"time\":start,\n", + " \"timeEnd\":end,\n", + " \"created\": end,\n", + " \"tags\":tags,\n", + " \"text\": message\n", + " }\n", + " resp = requests.post(grafana_base_url+'/api/annotations',verify=False,auth=('admin', grafana_pwd),headers=headers,json=data)\n", + " \n", + " data ={\n", + " \"dashboardId\":2,\n", + " # \"panelId\":8,\n", + " \"time\":start,\n", + " \"timeEnd\":end,\n", + " \"created\": end,\n", + " \"tags\":tags,\n", + " \"text\": message\n", + " }\n", + " resp = requests.post(grafana_base_url+'/api/annotations',verify=False,auth=('admin', grafana_pwd),headers=headers,json=data)\n", + " \n", + "\n", + "\n", + "t = tiles[2]\n", + "\n", + "tile_mesh_size = 10.\n", + "features = ['perc_95_normalized_height']\n", + "\n", + "grid_feature = {\n", + " 'min_x': -113107.81,\n", + " 'max_x': 398892.19,\n", + " 'min_y': 214783.87,\n", + " 'max_y': 726783.87,\n", + " 'n_tiles_side': 512\n", + "}\n", + "\n", + "feature_extraction_input = {\n", + " 'setup_local_fs': {'tmp_folder': conf_local_tmp.as_posix()},\n", + " 'pullremote': conf_remote_path_retiled.as_posix(),\n", + " 'load': {'attributes': ['raw_classification']},\n", + " 'normalize': 1,\n", + " 'apply_filter': {\n", + " 'filter_type': 'select_equal', \n", + " 'attribute': 'raw_classification',\n", + " 'value': [1, 6]#ground surface (2), water (9), buildings (6), artificial objects (26), vegetation (?), and unclassified (1)\n", + " },\n", + " 'generate_targets': {\n", + " 'tile_mesh_size' : tile_mesh_size,\n", + " 'validate' : True,\n", + " **grid_feature\n", + " },\n", + " 'extract_features': {\n", + " 'feature_names': features,\n", + " 'volume_type': 'cell',\n", + " 'volume_size': tile_mesh_size\n", + " },\n", + " 'export_targets': {\n", + " 'attributes': features,\n", + " 'multi_band_files': False\n", + " },\n", + " 'pushremote': conf_remote_path_targets.as_posix(),\n", + "# 'cleanlocalfs': {}\n", + "}\n", + "idx = (t.split('_')[1:])\n", + "processing = DataProcessing(t, tile_index=idx).config(feature_extraction_input).setup_webdav_client(conf_wd_opts)\n", + "processing.run()\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "34b843b6-5949-41ba-a412-5af3804d2540", + "metadata": {}, + "outputs": [], + "source": [ + "# title_with_underscore\n", + "\n", + "a = 1\n", + "\n", + "b = 2\n", + "\n", + "c = a + b\n", + "\n", + "c\n" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.10" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/vreapis/tests/resources/notebooks/test_param_in_cell_notebook.json b/vreapis/tests/resources/notebooks/test_param_in_cell_notebook.json new file mode 100644 index 00000000..0ffa5e6b --- /dev/null +++ b/vreapis/tests/resources/notebooks/test_param_in_cell_notebook.json @@ -0,0 +1,159 @@ +{ + "save": false, + "kernel": "ipython", + "cell_index": 6, + "notebook": { + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.4" + } + }, + "nbformat_minor": 5, + "nbformat": 4, + "cells": [ + { + "cell_type": "code", + "source": "# a list\n\na_list = ['a','b','c','d']", + "metadata": { + "trusted": true + }, + "execution_count": 2, + "outputs": [], + "id": "5f68e3d0-8fa2-4237-8fe3-29d1ec2cfbc1" + }, + { + "cell_type": "code", + "source": "# process a list\n\nb_list = []\nfor elem in a_list:\n b_list.append(elem+'42')\n", + "metadata": { + "trusted": true + }, + "execution_count": 3, + "outputs": [], + "id": "e0621bdf-05ee-44b7-9519-c9b136b6ba62" + }, + { + "cell_type": "code", + "source": "# concat\n\nres = ''\nfor elem in b_list:\n res+=elem\n \nprint(res)\n ", + "metadata": { + "trusted": true + }, + "execution_count": 4, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": "a42b42c42d42\n" + } + ], + "id": "97969827-90fa-41fc-b32e-a3f8d583e08e" + }, + { + "cell_type": "code", + "source": "#verage\n\na = 2 \nnumbers = [a, 4, 6, 8, 10]\n", + "metadata": { + "trusted": true, + "tags": [] + }, + "execution_count": 1, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": "6\n" + } + ], + "id": "39790b85-3619-409d-af6c-a1096e36b751" + }, + { + "cell_type": "code", + "source": "# bb\nimport statistics\n\nave = statistics.mean(numbers)\n\n", + "metadata": { + "trusted": true + }, + "execution_count": null, + "outputs": [], + "id": "578545cc-8459-4092-a503-329928fc6ad1" + }, + { + "cell_type": "code", + "source": "# params\n\n", + "metadata": { + "trusted": true + }, + "execution_count": null, + "outputs": [], + "id": "de7cc7cf-56c9-4538-92be-1a258e5ee812" + }, + { + "cell_type": "code", + "source": "# Param check\n\n\nparam_density = 1\n\nif param_density == 1: \n param_CountingStrategy = 'density0'\n \n# check if mandatory fields are present\ndiameterofsedimentationchamber = 'diameterofsedimentationchamber'", + "metadata": { + "trusted": true, + "tags": [] + }, + "execution_count": 2, + "outputs": [], + "id": "9824a80c-b00b-441c-8152-7b2e671e081b" + }, + { + "cell_type": "code", + "source": "# print \n\nprint(param_biovolume)\nparam_totalbiovolume\nparam_density\nparam_surfacearea\nparam_surfacevolumeratio\nparam_cellcarboncontent\nparam_totalcarboncontent\nparam_CountingStrategy\n\nprint(diameterofsedimentationchamber)\ndiameterofsedimentationchamber\ndiameteroffieldofview\ntransectcounting\nnumberofcountedfields\nnumberoftransects\nsettlingvolume\ndilutionfactor", + "metadata": { + "trusted": true, + "tags": [] + }, + "execution_count": 3, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": "1\ndiameterofsedimentationchamber\n" + }, + { + "execution_count": 3, + "output_type": "execute_result", + "data": { + "text/plain": "'dilutionfactor'" + }, + "metadata": {} + } + ], + "id": "287f6cad-2029-4693-97da-5834db20c59e" + }, + { + "cell_type": "code", + "source": "", + "metadata": { + "trusted": true + }, + "execution_count": null, + "outputs": [], + "id": "c2b4ba69-640f-4708-8d4e-7f033721f20f" + }, + { + "cell_type": "code", + "source": "", + "metadata": { + "trusted": true + }, + "execution_count": null, + "outputs": [], + "id": "00dd23e8-531e-4dd8-90aa-43573c240a48" + } + ] + } +} diff --git a/vreapis/tests/resources/notebooks/test_param_outside_cell_notebook.json b/vreapis/tests/resources/notebooks/test_param_outside_cell_notebook.json new file mode 100644 index 00000000..afdd2046 --- /dev/null +++ b/vreapis/tests/resources/notebooks/test_param_outside_cell_notebook.json @@ -0,0 +1,159 @@ +{ + "save": false, + "kernel": "ipython", + "cell_index": 6, + "notebook": { + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.4" + } + }, + "nbformat_minor": 5, + "nbformat": 4, + "cells": [ + { + "cell_type": "code", + "source": "# a list\n\na_list = ['a','b','c','d']", + "metadata": { + "trusted": true + }, + "execution_count": 2, + "outputs": [], + "id": "5f68e3d0-8fa2-4237-8fe3-29d1ec2cfbc1" + }, + { + "cell_type": "code", + "source": "# process a list\n\nb_list = []\nfor elem in a_list:\n b_list.append(elem+'42')\n", + "metadata": { + "trusted": true + }, + "execution_count": 3, + "outputs": [], + "id": "e0621bdf-05ee-44b7-9519-c9b136b6ba62" + }, + { + "cell_type": "code", + "source": "# concat\n\nres = ''\nfor elem in b_list:\n res+=elem\n \nprint(res)\n ", + "metadata": { + "trusted": true + }, + "execution_count": 4, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": "a42b42c42d42\n" + } + ], + "id": "97969827-90fa-41fc-b32e-a3f8d583e08e" + }, + { + "cell_type": "code", + "source": "#verage\n\na = 2 \nnumbers = [a, 4, 6, 8, 10]\n", + "metadata": { + "trusted": true, + "tags": [] + }, + "execution_count": 1, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": "6\n" + } + ], + "id": "39790b85-3619-409d-af6c-a1096e36b751" + }, + { + "cell_type": "code", + "source": "# bb\nimport statistics\n\nave = statistics.mean(numbers)\n\n", + "metadata": { + "trusted": true + }, + "execution_count": null, + "outputs": [], + "id": "578545cc-8459-4092-a503-329928fc6ad1" + }, + { + "cell_type": "code", + "source": "# params\n\nparam_density = 1", + "metadata": { + "trusted": true + }, + "execution_count": null, + "outputs": [], + "id": "de7cc7cf-56c9-4538-92be-1a258e5ee812" + }, + { + "cell_type": "code", + "source": "# Param check\n\n\n\n\nif param_density == 1: \n param_CountingStrategy = 'density0'\n \n# check if mandatory fields are present\ndiameterofsedimentationchamber = 'diameterofsedimentationchamber'", + "metadata": { + "trusted": true, + "tags": [] + }, + "execution_count": 2, + "outputs": [], + "id": "9824a80c-b00b-441c-8152-7b2e671e081b" + }, + { + "cell_type": "code", + "source": "# print \n\nprint(param_biovolume)\nparam_totalbiovolume\nparam_density\nparam_surfacearea\nparam_surfacevolumeratio\nparam_cellcarboncontent\nparam_totalcarboncontent\nparam_CountingStrategy\n\nprint(diameterofsedimentationchamber)\ndiameterofsedimentationchamber\ndiameteroffieldofview\ntransectcounting\nnumberofcountedfields\nnumberoftransects\nsettlingvolume\ndilutionfactor", + "metadata": { + "trusted": true, + "tags": [] + }, + "execution_count": 3, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": "1\ndiameterofsedimentationchamber\n" + }, + { + "execution_count": 3, + "output_type": "execute_result", + "data": { + "text/plain": "'dilutionfactor'" + }, + "metadata": {} + } + ], + "id": "287f6cad-2029-4693-97da-5834db20c59e" + }, + { + "cell_type": "code", + "source": "", + "metadata": { + "trusted": true + }, + "execution_count": null, + "outputs": [], + "id": "c2b4ba69-640f-4708-8d4e-7f033721f20f" + }, + { + "cell_type": "code", + "source": "", + "metadata": { + "trusted": true + }, + "execution_count": null, + "outputs": [], + "id": "00dd23e8-531e-4dd8-90aa-43573c240a48" + } + ] + } +} diff --git a/vreapis/tests/resources/notebooks/test_param_values_Python.json b/vreapis/tests/resources/notebooks/test_param_values_Python.json new file mode 100644 index 00000000..044c1a84 --- /dev/null +++ b/vreapis/tests/resources/notebooks/test_param_values_Python.json @@ -0,0 +1,56 @@ +{ + "save": false, + "kernel": "ipython", + "cell_index": 1, + "notebook": { + "metadata": { + "kernelspec": { + "name": "conda-env-anaconda3-jupyterlab2-py", + "display_name": "Python [conda env:anaconda3-jupyterlab2]", + "language": "python" + }, + "language_info": { + "name": "python", + "version": "3.12.1", + "mimetype": "text/x-python", + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "pygments_lexer": "ipython3", + "nbconvert_exporter": "python", + "file_extension": ".py" + } + }, + "nbformat_minor": 5, + "nbformat": 4, + "cells": [ + { + "cell_type": "code", + "source": "# params\nparam_string = 'param_string value'\nparam_string_with_comment = 'param_string value' # comment\nparam_int = 1\nparam_float = 1.1\nparam_list = [1, 2, 3]", + "metadata": { + "trusted": true + }, + "execution_count": 1, + "outputs": [], + "id": "8519a0de-0023-40ab-8b53-b7342950fed3" + }, + { + "cell_type": "code", + "source": "# Param check\nprint(param_string)\nprint(param_string_with_comment)\nprint(param_int)\nprint(param_float)\nprint(param_list)", + "metadata": { + "trusted": true + }, + "execution_count": 2, + "outputs": [ + { + "name": "stdout", + "text": "param_string value\nparam_string value\n1\n1.1\n[1, 2, 3]\n", + "output_type": "stream" + } + ], + "id": "3c6969b8-ffd7-4f09-bb9a-01d1e01a834a" + } + ] + } +} \ No newline at end of file diff --git a/vreapis/tests/resources/notebooks/test_param_values_R.json b/vreapis/tests/resources/notebooks/test_param_values_R.json new file mode 100644 index 00000000..83bf34b1 --- /dev/null +++ b/vreapis/tests/resources/notebooks/test_param_values_R.json @@ -0,0 +1,59 @@ +{ + "save": false, + "kernel": "IRkernel", + "cell_index": 1, + "notebook": { + "metadata": { + "kernelspec": { + "display_name": "R", + "language": "R", + "name": "ir" + }, + "language_info": { + "codemirror_mode": "r", + "file_extension": ".r", + "mimetype": "text/x-r-source", + "name": "R", + "pygments_lexer": "r", + "version": "4.3.1" + } + }, + "nbformat_minor": 5, + "nbformat": 4, + "cells": [ + { + "cell_type": "code", + "source": [ + "# params\n", + "param_string = 'param_string value'\n", + "param_string_with_comment = 'param_string value' # comment\n", + "param_int = 1\n", + "param_float = 1.1" + ], + "metadata": { + "trusted": true + }, + "execution_count": null, + "outputs": [], + "id": "de7cc7cf-56c9-4538-92be-1a258e5ee812" + }, + { + "cell_type": "code", + "source": [ + "# Param check\n", + "print(param_string)\n", + "print(param_string_with_comment)\n", + "print(param_int)\n", + "print(param_float)" + ], + "metadata": { + "trusted": true, + "tags": [] + }, + "execution_count": null, + "outputs": [], + "id": "9824a80c-b00b-441c-8152-7b2e671e081b" + } + ] + } +} diff --git a/vreapis/tests/resources/search/keyword.json b/vreapis/tests/resources/search/keyword.json new file mode 100644 index 00000000..bd11dff1 --- /dev/null +++ b/vreapis/tests/resources/search/keyword.json @@ -0,0 +1,3 @@ +{ + "keyword": "data model" +} diff --git a/vreapis/tests/resources/workflows/NaaVRE/468f918f414f96df.json b/vreapis/tests/resources/workflows/NaaVRE/468f918f414f96df.json new file mode 100644 index 00000000..6306093e --- /dev/null +++ b/vreapis/tests/resources/workflows/NaaVRE/468f918f414f96df.json @@ -0,0 +1,160 @@ +{ + "chart": { + "offset": { + "x": 0, + "y": 0 + }, + "scale": 1, + "nodes": { + "442d10bc-d999-400b-8050-0289d8586031": { + "id": "442d10bc-d999-400b-8050-0289d8586031", + "position": { + "x": 414.66668701171875, + "y": 111 + }, + "orientation": 0, + "type": "input-output", + "ports": { + "rolling_mean_temp_str": { + "properties": { + "color": "#c5a987" + }, + "id": "rolling_mean_temp_str", + "type": "right", + "position": { + "x": 197.5, + "y": 60 + } + }, + "temperature_data_str": { + "properties": { + "color": "#aca953" + }, + "id": "temperature_data_str", + "type": "right", + "position": { + "x": 197.5, + "y": 88 + } + } + }, + "properties": { + "title": "Install and load the climwinb packages-dev-user-name-at-domain-com", + "vars": [ + { + "name": "rolling_mean_temp_str", + "direction": "output", + "type": "datatype", + "color": "#c5a987" + }, + { + "name": "temperature_data_str", + "direction": "output", + "type": "datatype", + "color": "#aca953" + } + ], + "params": [], + "inputs": [], + "outputs": [ + "rolling_mean_temp_str", + "temperature_data_str" + ], + "og_node_id": "7b4271d" + }, + "size": { + "width": 250, + "height": 150 + } + }, + "112085dc-408b-4b9d-b6e8-242257ffebea": { + "id": "112085dc-408b-4b9d-b6e8-242257ffebea", + "position": { + "x": 855.6666870117188, + "y": 76 + }, + "orientation": 0, + "type": "input-output", + "ports": { + "rolling_mean_temp_str": { + "properties": { + "color": "#c5a987" + }, + "id": "rolling_mean_temp_str", + "type": "left", + "position": { + "x": 50.5, + "y": 60 + } + }, + "temperature_data_str": { + "properties": { + "color": "#aca953" + }, + "id": "temperature_data_str", + "type": "left", + "position": { + "x": 50.5, + "y": 88 + } + } + }, + "properties": { + "title": "consume climwin-dev-user-name-at-domain-com", + "vars": [ + { + "name": "rolling_mean_temp_str", + "direction": "input", + "type": "datatype", + "color": "#c5a987" + }, + { + "name": "temperature_data_str", + "direction": "input", + "type": "datatype", + "color": "#aca953" + } + ], + "params": [], + "inputs": [ + "rolling_mean_temp_str", + "temperature_data_str" + ], + "outputs": [], + "og_node_id": "dd373f8" + }, + "size": { + "width": 250, + "height": 150 + } + } + }, + "links": { + "a88cacb6-cfe8-4c71-a116-c2f2ad919863": { + "id": "a88cacb6-cfe8-4c71-a116-c2f2ad919863", + "from": { + "nodeId": "442d10bc-d999-400b-8050-0289d8586031", + "portId": "rolling_mean_temp_str" + }, + "to": { + "nodeId": "112085dc-408b-4b9d-b6e8-242257ffebea", + "portId": "rolling_mean_temp_str" + } + }, + "4c2d5463-1da4-4a06-838f-c20a5cd433ca": { + "id": "4c2d5463-1da4-4a06-838f-c20a5cd433ca", + "from": { + "nodeId": "442d10bc-d999-400b-8050-0289d8586031", + "portId": "temperature_data_str" + }, + "to": { + "nodeId": "112085dc-408b-4b9d-b6e8-242257ffebea", + "portId": "temperature_data_str" + } + } + }, + "selected": {}, + "hovered": {} + }, + "params": {} +} diff --git a/vreapis/tests/resources/workflows/NaaVRE/test_R_workflow.json b/vreapis/tests/resources/workflows/NaaVRE/test_R_workflow.json new file mode 100644 index 00000000..ad6233d0 --- /dev/null +++ b/vreapis/tests/resources/workflows/NaaVRE/test_R_workflow.json @@ -0,0 +1,844 @@ +{ + "chart": { + "offset": { + "x": 0, + "y": 0 + }, + "scale": 1, + "nodes": { + "12883ad5-9f59-47c9-bd6d-8063db9c8ca8": { + "id": "12883ad5-9f59-47c9-bd6d-8063db9c8ca8", + "position": { + "x": 0, + "y": 0 + }, + "orientation": 0, + "type": "input-output", + "ports": { + "file_path": { + "properties": { + "color": "#663a78" + }, + "id": "file_path", + "type": "right", + "position": { + "x": 217.5, + "y": 74.5 + } + } + }, + "properties": { + "title": "Create file R-dev-user-name-at-domain-com", + "vars": [ + { + "name": "file_path", + "direction": "output", + "type": "datatype", + "color": "#663a78" + } + ], + "params": [], + "inputs": [], + "outputs": [ + "file_path" + ], + "og_node_id": "c45fb67" + }, + "size": { + "width": 250, + "height": 150 + } + }, + "f8326a6b-a14d-4762-a738-1e51e648665e": { + "id": "f8326a6b-a14d-4762-a738-1e51e648665e", + "position": { + "x": 485.1721079707927, + "y": 0 + }, + "orientation": 0, + "type": "input-output", + "ports": { + "file_path": { + "properties": { + "color": "#663a78" + }, + "id": "file_path", + "type": "left", + "position": { + "x": 30.5, + "y": 74.5 + } + }, + "lines": { + "properties": { + "color": "#ac535c" + }, + "id": "lines", + "type": "right", + "position": { + "x": 231, + "y": 74.5 + } + } + }, + "properties": { + "title": "read file lines R-dev-user-name-at-domain-com", + "vars": [ + { + "name": "file_path", + "direction": "input", + "type": "datatype", + "color": "#663a78" + }, + { + "name": "lines", + "direction": "output", + "type": "datatype", + "color": "#ac535c" + } + ], + "params": [], + "inputs": [ + "file_path" + ], + "outputs": [ + "lines" + ], + "og_node_id": "47b2d97" + }, + "size": { + "width": 250, + "height": 150 + } + }, + "a3363399-53a3-49f4-97a2-405db0ac69f7": { + "id": "a3363399-53a3-49f4-97a2-405db0ac69f7", + "position": { + "x": 494.3224347681782, + "y": 173.46405228758172 + }, + "orientation": 0, + "type": "input-output", + "ports": { + "file_path": { + "properties": { + "color": "#663a78" + }, + "id": "file_path", + "type": "left", + "position": { + "x": 30.5, + "y": 74.5 + } + }, + "lines": { + "properties": { + "color": "#ac535c" + }, + "id": "lines", + "type": "right", + "position": { + "x": 231, + "y": 74.5 + } + } + }, + "properties": { + "title": "read file lines R-dev-user-name-at-domain-com", + "vars": [ + { + "name": "file_path", + "direction": "input", + "type": "datatype", + "color": "#663a78" + }, + { + "name": "lines", + "direction": "output", + "type": "datatype", + "color": "#ac535c" + } + ], + "params": [], + "inputs": [ + "file_path" + ], + "outputs": [ + "lines" + ], + "og_node_id": "47b2d97" + }, + "size": { + "width": 250, + "height": 150 + } + }, + "9f57774e-1133-4b8e-ab8e-2882836d564c": { + "id": "9f57774e-1133-4b8e-ab8e-2882836d564c", + "position": { + "x": 873.4074020884393, + "y": 24.44444444444442 + }, + "orientation": 0, + "type": "splitter", + "ports": { + "splitter_source": { + "id": "splitter_source", + "type": "left", + "properties": { + "special_node": 1, + "color": "#000000" + }, + "position": { + "x": 0.5, + "y": 49.5 + } + }, + "splitter_target": { + "id": "splitter_target", + "type": "right", + "properties": { + "special_node": 1, + "color": "#000000" + }, + "position": { + "x": 197.5, + "y": 49.5 + } + } + }, + "properties": { + "title": "Splitter", + "scalingFactor": 1 + }, + "size": { + "width": 200, + "height": 100 + } + }, + "ecfb5854-8511-47db-8cc8-d26f0b63bc8f": { + "id": "ecfb5854-8511-47db-8cc8-d26f0b63bc8f", + "position": { + "x": 868.1786439185047, + "y": 186.53594771241845 + }, + "orientation": 0, + "type": "input-output", + "ports": { + "lines": { + "properties": { + "color": "#ac535c" + }, + "id": "lines", + "type": "left", + "position": { + "x": 17, + "y": 74.5 + } + }, + "count": { + "properties": { + "color": "#783d3a" + }, + "id": "count", + "type": "right", + "position": { + "x": 228, + "y": 74.5 + } + } + }, + "properties": { + "title": "loop file lines R-dev-user-name-at-domain-com", + "vars": [ + { + "name": "lines", + "direction": "input", + "type": "datatype", + "color": "#ac535c" + }, + { + "name": "count", + "direction": "output", + "type": "datatype", + "color": "#783d3a" + } + ], + "params": [], + "inputs": [ + "lines" + ], + "outputs": [ + "count" + ], + "og_node_id": "f893a78" + }, + "size": { + "width": 250, + "height": 150 + } + }, + "0d044bb6-3457-41df-8796-304f57a76a25": { + "id": "0d044bb6-3457-41df-8796-304f57a76a25", + "position": { + "x": 1223.7341994740607, + "y": 0 + }, + "orientation": 0, + "type": "input-output", + "ports": { + "lines": { + "properties": { + "color": "#ac535c" + }, + "id": "lines", + "type": "left", + "position": { + "x": 17, + "y": 74.5 + } + }, + "count": { + "properties": { + "color": "#783d3a" + }, + "id": "count", + "type": "right", + "position": { + "x": 228, + "y": 74.5 + } + } + }, + "properties": { + "title": "loop file lines R-dev-user-name-at-domain-com", + "vars": [ + { + "name": "lines", + "direction": "input", + "type": "datatype", + "color": "#ac535c" + }, + { + "name": "count", + "direction": "output", + "type": "datatype", + "color": "#783d3a" + } + ], + "params": [], + "inputs": [ + "lines" + ], + "outputs": [ + "count" + ], + "og_node_id": "f893a78" + }, + "size": { + "width": 250, + "height": 150 + } + }, + "6b3fdf4f-69f5-48b4-addc-1666407cdbe0": { + "id": "6b3fdf4f-69f5-48b4-addc-1666407cdbe0", + "position": { + "x": 1260.3355066636025, + "y": 187.84313725490202 + }, + "orientation": 0, + "type": "input-output", + "ports": { + "count": { + "properties": { + "color": "#783d3a" + }, + "id": "count", + "type": "left", + "position": { + "x": 20, + "y": 74.5 + } + } + }, + "properties": { + "title": "Add1 R-dev-user-name-at-domain-com", + "vars": [ + { + "name": "count", + "direction": "input", + "type": "datatype", + "color": "#783d3a" + } + ], + "params": [], + "inputs": [ + "count" + ], + "outputs": [], + "og_node_id": "d9acfa2" + }, + "size": { + "width": 250, + "height": 150 + } + }, + "0da9edb7-eb4b-4edb-a314-ebfd34ba56ea": { + "id": "0da9edb7-eb4b-4edb-a314-ebfd34ba56ea", + "position": { + "x": 832.8845262714459, + "y": 419.2156862745097 + }, + "orientation": 0, + "type": "input-output", + "ports": { + "list_of_ints": { + "properties": { + "color": "#5f1f93" + }, + "id": "list_of_ints", + "type": "right", + "position": { + "x": 204, + "y": 60.5 + } + }, + "list_of_paths": { + "properties": { + "color": "#79c5d2" + }, + "id": "list_of_paths", + "type": "right", + "position": { + "x": 204, + "y": 88.5 + } + } + }, + "properties": { + "title": "input lists R-dev-user-name-at-domain-com", + "vars": [ + { + "name": "list_of_ints", + "direction": "output", + "type": "datatype", + "color": "#5f1f93" + }, + { + "name": "list_of_paths", + "direction": "output", + "type": "datatype", + "color": "#79c5d2" + } + ], + "params": [], + "inputs": [], + "outputs": [ + "list_of_ints", + "list_of_paths" + ], + "og_node_id": "3cfdc6f" + }, + "size": { + "width": 250, + "height": 150 + } + }, + "7b288a3c-672e-4069-95f2-37a044600119": { + "id": "7b288a3c-672e-4069-95f2-37a044600119", + "position": { + "x": 1277.3289707158906, + "y": 476.73202614379073 + }, + "orientation": 0, + "type": "splitter", + "ports": { + "splitter_source": { + "id": "splitter_source", + "type": "left", + "properties": { + "special_node": 1, + "color": "#000000" + }, + "position": { + "x": 0.5, + "y": 49.5 + } + }, + "splitter_target": { + "id": "splitter_target", + "type": "right", + "properties": { + "special_node": 1, + "color": "#000000" + }, + "position": { + "x": 197.5, + "y": 49.5 + } + } + }, + "properties": { + "title": "Splitter", + "scalingFactor": 1 + }, + "size": { + "width": 200, + "height": 100 + } + }, + "d4c8fac5-606b-4d39-8fcd-d93e2603a6a7": { + "id": "d4c8fac5-606b-4d39-8fcd-d93e2603a6a7", + "position": { + "x": 1264.2570752910538, + "y": 353.85620915032683 + }, + "orientation": 0, + "type": "splitter", + "ports": { + "splitter_source": { + "id": "splitter_source", + "type": "left", + "properties": { + "special_node": 1, + "color": "#000000" + }, + "position": { + "x": 0.5, + "y": 49.5 + } + }, + "splitter_target": { + "id": "splitter_target", + "type": "right", + "properties": { + "special_node": 1, + "color": "#000000" + }, + "position": { + "x": 197.5, + "y": 49.5 + } + } + }, + "properties": { + "title": "Splitter", + "scalingFactor": 1 + }, + "size": { + "width": 200, + "height": 100 + } + }, + "27ff8437-75d0-4f60-b73a-e33e53d86dda": { + "id": "27ff8437-75d0-4f60-b73a-e33e53d86dda", + "position": { + "x": 1602.819166794322, + "y": 322.4836601307189 + }, + "orientation": 0, + "type": "input-output", + "ports": { + "list_of_ints": { + "properties": { + "color": "#5f1f93" + }, + "id": "list_of_ints", + "type": "left", + "position": { + "x": 37.5, + "y": 74.5 + } + } + }, + "properties": { + "title": "loop int list R-dev-user-name-at-domain-com", + "vars": [ + { + "name": "list_of_ints", + "direction": "input", + "type": "datatype", + "color": "#5f1f93" + } + ], + "params": [], + "inputs": [ + "list_of_ints" + ], + "outputs": [], + "og_node_id": "3285abd" + }, + "size": { + "width": 250, + "height": 150 + } + }, + "0a44c471-853b-4f86-b69b-17b41a079591": { + "id": "0a44c471-853b-4f86-b69b-17b41a079591", + "position": { + "x": 1649.8779903237335, + "y": 143.39869281045745 + }, + "orientation": 0, + "type": "input-output", + "ports": { + "list_of_ints": { + "properties": { + "color": "#5f1f93" + }, + "id": "list_of_ints", + "type": "left", + "position": { + "x": 37.5, + "y": 74.5 + } + } + }, + "properties": { + "title": "loop int list R-dev-user-name-at-domain-com", + "vars": [ + { + "name": "list_of_ints", + "direction": "input", + "type": "datatype", + "color": "#5f1f93" + } + ], + "params": [], + "inputs": [ + "list_of_ints" + ], + "outputs": [], + "og_node_id": "3285abd" + }, + "size": { + "width": 250, + "height": 150 + } + }, + "9be7f1a9-7fb8-4671-b9c5-0b9aa6263dc2": { + "id": "9be7f1a9-7fb8-4671-b9c5-0b9aa6263dc2", + "position": { + "x": 1609.3551145067397, + "y": 492.4183006535946 + }, + "orientation": 0, + "type": "input-output", + "ports": { + "list_of_paths": { + "properties": { + "color": "#79c5d2" + }, + "id": "list_of_paths", + "type": "left", + "position": { + "x": 44, + "y": 74.5 + } + } + }, + "properties": { + "title": "loop list R-dev-user-name-at-domain-com", + "vars": [ + { + "name": "list_of_paths", + "direction": "input", + "type": "datatype", + "color": "#79c5d2" + } + ], + "params": [], + "inputs": [ + "list_of_paths" + ], + "outputs": [], + "og_node_id": "943e143" + }, + "size": { + "width": 250, + "height": 150 + } + }, + "beb396d1-358b-4dcc-952d-e140acb5f13d": { + "id": "beb396d1-358b-4dcc-952d-e140acb5f13d", + "position": { + "x": 1524.3877942453023, + "y": 722.4836601307189 + }, + "orientation": 0, + "type": "input-output", + "ports": { + "list_of_paths": { + "properties": { + "color": "#79c5d2" + }, + "id": "list_of_paths", + "type": "left", + "position": { + "x": 44, + "y": 74.5 + } + } + }, + "properties": { + "title": "loop list R-dev-user-name-at-domain-com", + "vars": [ + { + "name": "list_of_paths", + "direction": "input", + "type": "datatype", + "color": "#79c5d2" + } + ], + "params": [], + "inputs": [ + "list_of_paths" + ], + "outputs": [], + "og_node_id": "943e143" + }, + "size": { + "width": 250, + "height": 150 + } + } + }, + "links": { + "f7cdf578-7c18-4486-9579-6398c4b171be": { + "id": "f7cdf578-7c18-4486-9579-6398c4b171be", + "from": { + "nodeId": "12883ad5-9f59-47c9-bd6d-8063db9c8ca8", + "portId": "file_path" + }, + "to": { + "nodeId": "f8326a6b-a14d-4762-a738-1e51e648665e", + "portId": "file_path" + } + }, + "2bd830fb-4305-4e56-9667-14ef6663dbc2": { + "id": "2bd830fb-4305-4e56-9667-14ef6663dbc2", + "from": { + "nodeId": "12883ad5-9f59-47c9-bd6d-8063db9c8ca8", + "portId": "file_path" + }, + "to": { + "nodeId": "a3363399-53a3-49f4-97a2-405db0ac69f7", + "portId": "file_path" + } + }, + "e4210144-6b90-427a-9a27-1408aa7f8062": { + "id": "e4210144-6b90-427a-9a27-1408aa7f8062", + "from": { + "nodeId": "f8326a6b-a14d-4762-a738-1e51e648665e", + "portId": "lines" + }, + "to": { + "nodeId": "9f57774e-1133-4b8e-ab8e-2882836d564c", + "portId": "splitter_source" + } + }, + "b25dc7dd-1015-4b35-bef9-54baadd95beb": { + "id": "b25dc7dd-1015-4b35-bef9-54baadd95beb", + "from": { + "nodeId": "a3363399-53a3-49f4-97a2-405db0ac69f7", + "portId": "lines" + }, + "to": { + "nodeId": "ecfb5854-8511-47db-8cc8-d26f0b63bc8f", + "portId": "lines" + } + }, + "020b28ad-59c0-46d9-b345-d27f729b29c1": { + "id": "020b28ad-59c0-46d9-b345-d27f729b29c1", + "from": { + "nodeId": "9f57774e-1133-4b8e-ab8e-2882836d564c", + "portId": "splitter_target" + }, + "to": { + "nodeId": "0d044bb6-3457-41df-8796-304f57a76a25", + "portId": "lines" + } + }, + "7916ab47-a8b1-4c38-9001-cf81298d4655": { + "id": "7916ab47-a8b1-4c38-9001-cf81298d4655", + "from": { + "nodeId": "ecfb5854-8511-47db-8cc8-d26f0b63bc8f", + "portId": "count" + }, + "to": { + "nodeId": "6b3fdf4f-69f5-48b4-addc-1666407cdbe0", + "portId": "count" + } + }, + "2d2b3c9c-02be-4a20-84c6-c8018f155c1f": { + "id": "2d2b3c9c-02be-4a20-84c6-c8018f155c1f", + "from": { + "nodeId": "0da9edb7-eb4b-4edb-a314-ebfd34ba56ea", + "portId": "list_of_ints" + }, + "to": { + "nodeId": "d4c8fac5-606b-4d39-8fcd-d93e2603a6a7", + "portId": "splitter_source" + } + }, + "c79aee7c-2516-40c4-a149-7a413df6db5e": { + "id": "c79aee7c-2516-40c4-a149-7a413df6db5e", + "from": { + "nodeId": "0da9edb7-eb4b-4edb-a314-ebfd34ba56ea", + "portId": "list_of_paths" + }, + "to": { + "nodeId": "7b288a3c-672e-4069-95f2-37a044600119", + "portId": "splitter_source" + } + }, + "e7e5aeaa-6222-406f-b8e3-ccc0b7a83bc7": { + "id": "e7e5aeaa-6222-406f-b8e3-ccc0b7a83bc7", + "from": { + "nodeId": "d4c8fac5-606b-4d39-8fcd-d93e2603a6a7", + "portId": "splitter_target" + }, + "to": { + "nodeId": "27ff8437-75d0-4f60-b73a-e33e53d86dda", + "portId": "list_of_ints" + } + }, + "fa489149-1708-4993-bf6f-099596614383": { + "id": "fa489149-1708-4993-bf6f-099596614383", + "from": { + "nodeId": "0da9edb7-eb4b-4edb-a314-ebfd34ba56ea", + "portId": "list_of_ints" + }, + "to": { + "nodeId": "0a44c471-853b-4f86-b69b-17b41a079591", + "portId": "list_of_ints" + } + }, + "172cf546-272f-4247-9488-5a7aa226fd0b": { + "id": "172cf546-272f-4247-9488-5a7aa226fd0b", + "from": { + "nodeId": "7b288a3c-672e-4069-95f2-37a044600119", + "portId": "splitter_target" + }, + "to": { + "nodeId": "9be7f1a9-7fb8-4671-b9c5-0b9aa6263dc2", + "portId": "list_of_paths" + } + }, + "8c1ed73b-b34b-48bc-bf2a-6af3aff5df27": { + "id": "8c1ed73b-b34b-48bc-bf2a-6af3aff5df27", + "from": { + "nodeId": "0da9edb7-eb4b-4edb-a314-ebfd34ba56ea", + "portId": "list_of_paths" + }, + "to": { + "nodeId": "beb396d1-358b-4dcc-952d-e140acb5f13d", + "portId": "list_of_paths" + } + } + }, + "selected": {}, + "hovered": {} + }, + "params": {} +} diff --git a/vreapis/tests/resources/workflows/NaaVRE/test_py_workflow.json b/vreapis/tests/resources/workflows/NaaVRE/test_py_workflow.json new file mode 100644 index 00000000..5a63be06 --- /dev/null +++ b/vreapis/tests/resources/workflows/NaaVRE/test_py_workflow.json @@ -0,0 +1,1378 @@ +{ + "chart": { + "offset": { + "x": 0, + "y": 0 + }, + "scale": 1, + "nodes": { + "e05a69bc-d0a4-4f79-93e2-7c79ba2b04e1": { + "id": "e05a69bc-d0a4-4f79-93e2-7c79ba2b04e1", + "position": { + "x": 0, + "y": 0 + }, + "orientation": 0, + "type": "input-output", + "ports": { + "file_path": { + "properties": { + "color": "#663a78" + }, + "id": "file_path", + "type": "right", + "position": { + "x": 217.5, + "y": 74.5 + } + } + }, + "properties": { + "title": "Create file-dev-user-name-at-domain-com", + "vars": [ + { + "name": "file_path", + "direction": "output", + "type": "datatype", + "color": "#663a78" + } + ], + "params": [], + "inputs": [], + "outputs": [ + "file_path" + ], + "og_node_id": "5a918c8" + }, + "size": { + "width": 250, + "height": 150 + } + }, + "7d1c8e18-880d-4bab-be5c-dd47d5eed5b4": { + "id": "7d1c8e18-880d-4bab-be5c-dd47d5eed5b4", + "position": { + "x": 365.51197725183823, + "y": 0 + }, + "orientation": 0, + "type": "input-output", + "ports": { + "file_path": { + "properties": { + "color": "#663a78" + }, + "id": "file_path", + "type": "left", + "position": { + "x": 30.5, + "y": 74.5 + } + }, + "lines": { + "properties": { + "color": "#ac535c" + }, + "id": "lines", + "type": "right", + "position": { + "x": 231, + "y": 74.5 + } + } + }, + "properties": { + "title": "read file lines-dev-user-name-at-domain-com", + "vars": [ + { + "name": "file_path", + "direction": "input", + "type": "datatype", + "color": "#663a78" + }, + { + "name": "lines", + "direction": "output", + "type": "datatype", + "color": "#ac535c" + } + ], + "params": [], + "inputs": [ + "file_path" + ], + "outputs": [ + "lines" + ], + "og_node_id": "4deac70" + }, + "size": { + "width": 250, + "height": 150 + } + }, + "f847ca0a-1a4d-4e2a-85bd-6ec7be87c3ed": { + "id": "f847ca0a-1a4d-4e2a-85bd-6ec7be87c3ed", + "position": { + "x": 364.20478770935455, + "y": 177.11111111111117 + }, + "orientation": 0, + "type": "input-output", + "ports": { + "file_path": { + "properties": { + "color": "#663a78" + }, + "id": "file_path", + "type": "left", + "position": { + "x": 30.5, + "y": 74.5 + } + }, + "lines": { + "properties": { + "color": "#ac535c" + }, + "id": "lines", + "type": "right", + "position": { + "x": 231, + "y": 74.5 + } + } + }, + "properties": { + "title": "read file lines-dev-user-name-at-domain-com", + "vars": [ + { + "name": "file_path", + "direction": "input", + "type": "datatype", + "color": "#663a78" + }, + { + "name": "lines", + "direction": "output", + "type": "datatype", + "color": "#ac535c" + } + ], + "params": [], + "inputs": [ + "file_path" + ], + "outputs": [ + "lines" + ], + "og_node_id": "4deac70" + }, + "size": { + "width": 250, + "height": 150 + } + }, + "8f44f21c-829c-4abc-881f-377159a2d2b1": { + "id": "8f44f21c-829c-4abc-881f-377159a2d2b1", + "position": { + "x": 697.5381210426875, + "y": 17.63398692810455 + }, + "orientation": 0, + "type": "splitter", + "ports": { + "splitter_source": { + "id": "splitter_source", + "type": "left", + "properties": { + "special_node": 1, + "color": "#000000" + }, + "position": { + "x": 0.5, + "y": 49.5 + } + }, + "splitter_target": { + "id": "splitter_target", + "type": "right", + "properties": { + "special_node": 1, + "color": "#000000" + }, + "position": { + "x": 197.5, + "y": 49.5 + } + } + }, + "properties": { + "title": "Splitter", + "scalingFactor": 1 + }, + "size": { + "width": 200, + "height": 100 + } + }, + "1d67688c-536d-48ef-a40f-4b0aaa3f3638": { + "id": "1d67688c-536d-48ef-a40f-4b0aaa3f3638", + "position": { + "x": 1000.8060948988971, + "y": 3.254901960784335 + }, + "orientation": 0, + "type": "input-output", + "ports": { + "lines": { + "properties": { + "color": "#ac535c" + }, + "id": "lines", + "type": "left", + "position": { + "x": 17, + "y": 74.5 + } + }, + "count": { + "properties": { + "color": "#783d3a" + }, + "id": "count", + "type": "right", + "position": { + "x": 228, + "y": 74.5 + } + } + }, + "properties": { + "title": "loop file lines-dev-user-name-at-domain-com", + "vars": [ + { + "name": "lines", + "direction": "input", + "type": "datatype", + "color": "#ac535c" + }, + { + "name": "count", + "direction": "output", + "type": "datatype", + "color": "#783d3a" + } + ], + "params": [], + "inputs": [ + "lines" + ], + "outputs": [ + "count" + ], + "og_node_id": "5d9de73" + }, + "size": { + "width": 250, + "height": 150 + } + }, + "473de030-05fc-4f09-9325-11b838bcc140": { + "id": "473de030-05fc-4f09-9325-11b838bcc140", + "position": { + "x": 982.5054413041258, + "y": 290.8366013071896 + }, + "orientation": 0, + "type": "input-output", + "ports": { + "lines": { + "properties": { + "color": "#ac535c" + }, + "id": "lines", + "type": "left", + "position": { + "x": 17, + "y": 74.5 + } + }, + "count": { + "properties": { + "color": "#783d3a" + }, + "id": "count", + "type": "right", + "position": { + "x": 228, + "y": 74.5 + } + } + }, + "properties": { + "title": "loop file lines-dev-user-name-at-domain-com", + "vars": [ + { + "name": "lines", + "direction": "input", + "type": "datatype", + "color": "#ac535c" + }, + { + "name": "count", + "direction": "output", + "type": "datatype", + "color": "#783d3a" + } + ], + "params": [], + "inputs": [ + "lines" + ], + "outputs": [ + "count" + ], + "og_node_id": "5d9de73" + }, + "size": { + "width": 250, + "height": 150 + } + }, + "62fe958c-92d6-4be1-bbcc-73bfe88b9e2a": { + "id": "62fe958c-92d6-4be1-bbcc-73bfe88b9e2a", + "position": { + "x": 1349.8257027420339, + "y": 177.77777777777786 + }, + "orientation": 0, + "type": "input-output", + "ports": { + "count": { + "properties": { + "color": "#783d3a" + }, + "id": "count", + "type": "left", + "position": { + "x": 20, + "y": 74.5 + } + }, + "a": { + "properties": { + "color": "#77862d" + }, + "id": "a", + "type": "right", + "position": { + "x": 243, + "y": 74.5 + } + } + }, + "properties": { + "title": "Add1-dev-user-name-at-domain-com", + "vars": [ + { + "name": "count", + "direction": "input", + "type": "datatype", + "color": "#783d3a" + }, + { + "name": "a", + "direction": "output", + "type": "datatype", + "color": "#77862d" + } + ], + "params": [], + "inputs": [ + "count" + ], + "outputs": [ + "a" + ], + "og_node_id": "49d5dda" + }, + "size": { + "width": 250, + "height": 150 + } + }, + "57c8b452-35f5-4b7b-b777-e2bee59ef99d": { + "id": "57c8b452-35f5-4b7b-b777-e2bee59ef99d", + "position": { + "x": 1694.9237419577207, + "y": 268.61437908496737 + }, + "orientation": 0, + "type": "input-output", + "ports": { + "a": { + "properties": { + "color": "#77862d" + }, + "id": "a", + "type": "left", + "position": { + "x": 20, + "y": 60.5 + } + }, + "count": { + "properties": { + "color": "#783d3a" + }, + "id": "count", + "type": "left", + "position": { + "x": 20, + "y": 88.5 + } + }, + "msg": { + "properties": { + "color": "#7bd279" + }, + "id": "msg", + "type": "right", + "position": { + "x": 232.5, + "y": 74.5 + } + } + }, + "properties": { + "title": "Anti-pattern-dev-user-name-at-domain-com", + "vars": [ + { + "name": "a", + "direction": "input", + "type": "datatype", + "color": "#77862d" + }, + { + "name": "count", + "direction": "input", + "type": "datatype", + "color": "#783d3a" + }, + { + "name": "msg", + "direction": "output", + "type": "datatype", + "color": "#7bd279" + } + ], + "params": [], + "inputs": [ + "a", + "count" + ], + "outputs": [ + "msg" + ], + "og_node_id": "44dec46" + }, + "size": { + "width": 250, + "height": 150 + } + }, + "12ae508b-b472-4924-af9a-a3051c7612fa": { + "id": "12ae508b-b472-4924-af9a-a3051c7612fa", + "position": { + "x": 2223.028317121121, + "y": 103.90849673202612 + }, + "orientation": 0, + "type": "input-output", + "ports": { + "msg": { + "properties": { + "color": "#7bd279" + }, + "id": "msg", + "type": "left", + "position": { + "x": 15.5, + "y": 74.5 + } + }, + "list_of_ints": { + "properties": { + "color": "#5f1f93" + }, + "id": "list_of_ints", + "type": "right", + "position": { + "x": 204, + "y": 60.5 + } + }, + "list_of_paths": { + "properties": { + "color": "#79c5d2" + }, + "id": "list_of_paths", + "type": "right", + "position": { + "x": 204, + "y": 88.5 + } + } + }, + "properties": { + "title": "input lists-dev-user-name-at-domain-com", + "vars": [ + { + "name": "msg", + "direction": "input", + "type": "datatype", + "color": "#7bd279" + }, + { + "name": "list_of_ints", + "direction": "output", + "type": "datatype", + "color": "#5f1f93" + }, + { + "name": "list_of_paths", + "direction": "output", + "type": "datatype", + "color": "#79c5d2" + } + ], + "params": [], + "inputs": [ + "msg" + ], + "outputs": [ + "list_of_ints", + "list_of_paths" + ], + "og_node_id": "3e4f548" + }, + "size": { + "width": 250, + "height": 150 + } + }, + "c8cf52ae-446f-402e-8ea5-d26dbf4ff903": { + "id": "c8cf52ae-446f-402e-8ea5-d26dbf4ff903", + "position": { + "x": 2203.420473983864, + "y": 302.6013071895425 + }, + "orientation": 0, + "type": "input-output", + "ports": { + "msg": { + "properties": { + "color": "#7bd279" + }, + "id": "msg", + "type": "left", + "position": { + "x": 15.5, + "y": 74.5 + } + }, + "list_of_ints": { + "properties": { + "color": "#5f1f93" + }, + "id": "list_of_ints", + "type": "right", + "position": { + "x": 204, + "y": 60.5 + } + }, + "list_of_paths": { + "properties": { + "color": "#79c5d2" + }, + "id": "list_of_paths", + "type": "right", + "position": { + "x": 204, + "y": 88.5 + } + } + }, + "properties": { + "title": "input lists-dev-user-name-at-domain-com", + "vars": [ + { + "name": "msg", + "direction": "input", + "type": "datatype", + "color": "#7bd279" + }, + { + "name": "list_of_ints", + "direction": "output", + "type": "datatype", + "color": "#5f1f93" + }, + { + "name": "list_of_paths", + "direction": "output", + "type": "datatype", + "color": "#79c5d2" + } + ], + "params": [], + "inputs": [ + "msg" + ], + "outputs": [ + "list_of_ints", + "list_of_paths" + ], + "og_node_id": "3e4f548" + }, + "size": { + "width": 250, + "height": 150 + } + }, + "c0f6bebc-1529-4901-85d7-42eb1342868e": { + "id": "c0f6bebc-1529-4901-85d7-42eb1342868e", + "position": { + "x": 2617.799558951182, + "y": 0 + }, + "orientation": 0, + "type": "splitter", + "ports": { + "splitter_source": { + "id": "splitter_source", + "type": "left", + "properties": { + "special_node": 1, + "color": "#000000" + }, + "position": { + "x": 0.5, + "y": 49.5 + } + }, + "splitter_target": { + "id": "splitter_target", + "type": "right", + "properties": { + "special_node": 1, + "color": "#000000" + }, + "position": { + "x": 197.5, + "y": 49.5 + } + } + }, + "properties": { + "title": "Splitter", + "scalingFactor": 1 + }, + "size": { + "width": 200, + "height": 100 + } + }, + "edd68621-780a-42df-958c-74f0a711b260": { + "id": "edd68621-780a-42df-958c-74f0a711b260", + "position": { + "x": 2634.793023003473, + "y": 145.73856209150335 + }, + "orientation": 0, + "type": "splitter", + "ports": { + "splitter_source": { + "id": "splitter_source", + "type": "left", + "properties": { + "special_node": 1, + "color": "#000000" + }, + "position": { + "x": 0.5, + "y": 49.5 + } + }, + "splitter_target": { + "id": "splitter_target", + "type": "right", + "properties": { + "special_node": 1, + "color": "#000000" + }, + "position": { + "x": 197.5, + "y": 49.5 + } + } + }, + "properties": { + "title": "Splitter", + "scalingFactor": 1 + }, + "size": { + "width": 200, + "height": 100 + } + }, + "1a85bb42-8eba-4b46-b691-ac0687985fab": { + "id": "1a85bb42-8eba-4b46-b691-ac0687985fab", + "position": { + "x": 3088.387794245301, + "y": 186.2614379084967 + }, + "orientation": 0, + "type": "input-output", + "ports": { + "list_of_paths": { + "properties": { + "color": "#79c5d2" + }, + "id": "list_of_paths", + "type": "left", + "position": { + "x": 44, + "y": 74.5 + } + } + }, + "properties": { + "title": "loop list-dev-user-name-at-domain-com", + "vars": [ + { + "name": "list_of_paths", + "direction": "input", + "type": "datatype", + "color": "#79c5d2" + } + ], + "params": [], + "inputs": [ + "list_of_paths" + ], + "outputs": [], + "og_node_id": "6ce40db" + }, + "size": { + "width": 250, + "height": 150 + } + }, + "8f94be49-2e5a-4b22-bef9-7a2d35b05d76": { + "id": "8f94be49-2e5a-4b22-bef9-7a2d35b05d76", + "position": { + "x": 2881.851846532883, + "y": 528.7450980392158 + }, + "orientation": 0, + "type": "input-output", + "ports": { + "list_of_paths": { + "properties": { + "color": "#79c5d2" + }, + "id": "list_of_paths", + "type": "left", + "position": { + "x": 44, + "y": 74.5 + } + } + }, + "properties": { + "title": "loop list-dev-user-name-at-domain-com", + "vars": [ + { + "name": "list_of_paths", + "direction": "input", + "type": "datatype", + "color": "#79c5d2" + } + ], + "params": [], + "inputs": [ + "list_of_paths" + ], + "outputs": [], + "og_node_id": "6ce40db" + }, + "size": { + "width": 250, + "height": 150 + } + }, + "01a6d66f-afb1-47c4-8f94-faace4874409": { + "id": "01a6d66f-afb1-47c4-8f94-faace4874409", + "position": { + "x": 3060.936813853145, + "y": 0 + }, + "orientation": 0, + "type": "input-output", + "ports": { + "list_of_ints": { + "properties": { + "color": "#5f1f93" + }, + "id": "list_of_ints", + "type": "left", + "position": { + "x": 37.5, + "y": 74.5 + } + }, + "a": { + "properties": { + "color": "#77862d" + }, + "id": "a", + "type": "right", + "position": { + "x": 243, + "y": 74.5 + } + } + }, + "properties": { + "title": "loop int list-dev-user-name-at-domain-com", + "vars": [ + { + "name": "list_of_ints", + "direction": "input", + "type": "datatype", + "color": "#5f1f93" + }, + { + "name": "a", + "direction": "output", + "type": "datatype", + "color": "#77862d" + } + ], + "params": [], + "inputs": [ + "list_of_ints" + ], + "outputs": [ + "a" + ], + "og_node_id": "e1578ef" + }, + "size": { + "width": 250, + "height": 150 + } + }, + "f5733922-b56b-497e-8066-c96cdb7b2a79": { + "id": "f5733922-b56b-497e-8066-c96cdb7b2a79", + "position": { + "x": 3104.074068755105, + "y": 347.0457516339869 + }, + "orientation": 0, + "type": "input-output", + "ports": { + "list_of_ints": { + "properties": { + "color": "#5f1f93" + }, + "id": "list_of_ints", + "type": "left", + "position": { + "x": 37.5, + "y": 74.5 + } + }, + "a": { + "properties": { + "color": "#77862d" + }, + "id": "a", + "type": "right", + "position": { + "x": 243, + "y": 74.5 + } + } + }, + "properties": { + "title": "loop int list-dev-user-name-at-domain-com", + "vars": [ + { + "name": "list_of_ints", + "direction": "input", + "type": "datatype", + "color": "#5f1f93" + }, + { + "name": "a", + "direction": "output", + "type": "datatype", + "color": "#77862d" + } + ], + "params": [], + "inputs": [ + "list_of_ints" + ], + "outputs": [ + "a" + ], + "og_node_id": "e1578ef" + }, + "size": { + "width": 250, + "height": 150 + } + }, + "c64cf08d-2caa-46bb-a199-c8fd25e9d590": { + "id": "c64cf08d-2caa-46bb-a199-c8fd25e9d590", + "position": { + "x": 3744.5969445720984, + "y": 345.73856209150324 + }, + "orientation": 0, + "type": "input-output", + "ports": { + "a": { + "properties": { + "color": "#77862d" + }, + "id": "a", + "type": "left", + "position": { + "x": 5, + "y": 74.5 + } + }, + "msg": { + "properties": { + "color": "#7bd279" + }, + "id": "msg", + "type": "right", + "position": { + "x": 232.5, + "y": 74.5 + } + } + }, + "properties": { + "title": "KNMI-vol-h5-to-ODIM-h5-dev-user-name-at-domain-com", + "vars": [ + { + "name": "a", + "direction": "input", + "type": "datatype", + "color": "#77862d" + }, + { + "name": "msg", + "direction": "output", + "type": "datatype", + "color": "#7bd279" + } + ], + "params": [], + "inputs": [ + "a" + ], + "outputs": [ + "msg" + ], + "og_node_id": "e25ec34" + }, + "size": { + "width": 250, + "height": 150 + } + }, + "223011e6-faff-479b-a859-a3f77db0fbb5": { + "id": "223011e6-faff-479b-a859-a3f77db0fbb5", + "position": { + "x": 3735.4466177747117, + "y": 515.6732026143787 + }, + "orientation": 0, + "type": "input-output", + "ports": { + "a": { + "properties": { + "color": "#77862d" + }, + "id": "a", + "type": "left", + "position": { + "x": 5, + "y": 74.5 + } + }, + "msg": { + "properties": { + "color": "#7bd279" + }, + "id": "msg", + "type": "right", + "position": { + "x": 232.5, + "y": 74.5 + } + } + }, + "properties": { + "title": "vol2bird-dev-user-name-at-domain-com", + "vars": [ + { + "name": "a", + "direction": "input", + "type": "datatype", + "color": "#77862d" + }, + { + "name": "msg", + "direction": "output", + "type": "datatype", + "color": "#7bd279" + } + ], + "params": [], + "inputs": [ + "a" + ], + "outputs": [ + "msg" + ], + "og_node_id": "ded63d7" + }, + "size": { + "width": 250, + "height": 150 + } + }, + "e9e3a59e-923f-4d05-b3b0-2ec79590acc9": { + "id": "e9e3a59e-923f-4d05-b3b0-2ec79590acc9", + "position": { + "x": 4251.78648705576, + "y": 340.5098039215685 + }, + "orientation": 0, + "type": "input-output", + "ports": { + "msg": { + "properties": { + "color": "#7bd279" + }, + "id": "msg", + "type": "left", + "position": { + "x": 15.5, + "y": 74.5 + } + } + }, + "properties": { + "title": "Print msg-dev-user-name-at-domain-com", + "vars": [ + { + "name": "msg", + "direction": "input", + "type": "datatype", + "color": "#7bd279" + } + ], + "params": [], + "inputs": [ + "msg" + ], + "outputs": [], + "og_node_id": "a74dc70" + }, + "size": { + "width": 250, + "height": 150 + } + }, + "cd47ca61-ba20-4826-b92f-7ee990822f3a": { + "id": "cd47ca61-ba20-4826-b92f-7ee990822f3a", + "position": { + "x": 4249.172107970791, + "y": 539.2026143790852 + }, + "orientation": 0, + "type": "input-output", + "ports": { + "msg": { + "properties": { + "color": "#7bd279" + }, + "id": "msg", + "type": "left", + "position": { + "x": 15.5, + "y": 74.5 + } + } + }, + "properties": { + "title": "Print msg-dev-user-name-at-domain-com", + "vars": [ + { + "name": "msg", + "direction": "input", + "type": "datatype", + "color": "#7bd279" + } + ], + "params": [], + "inputs": [ + "msg" + ], + "outputs": [], + "og_node_id": "a74dc70" + }, + "size": { + "width": 250, + "height": 150 + } + } + }, + "links": { + "a9475401-f066-46ec-893d-84f5ab3d1cf4": { + "id": "a9475401-f066-46ec-893d-84f5ab3d1cf4", + "from": { + "nodeId": "e05a69bc-d0a4-4f79-93e2-7c79ba2b04e1", + "portId": "file_path" + }, + "to": { + "nodeId": "7d1c8e18-880d-4bab-be5c-dd47d5eed5b4", + "portId": "file_path" + } + }, + "48a07267-919e-4852-8d08-c83a366b0c85": { + "id": "48a07267-919e-4852-8d08-c83a366b0c85", + "from": { + "nodeId": "e05a69bc-d0a4-4f79-93e2-7c79ba2b04e1", + "portId": "file_path" + }, + "to": { + "nodeId": "f847ca0a-1a4d-4e2a-85bd-6ec7be87c3ed", + "portId": "file_path" + } + }, + "43dcc09b-af0e-4738-bcba-faf9f6ae26dc": { + "id": "43dcc09b-af0e-4738-bcba-faf9f6ae26dc", + "from": { + "nodeId": "7d1c8e18-880d-4bab-be5c-dd47d5eed5b4", + "portId": "lines" + }, + "to": { + "nodeId": "8f44f21c-829c-4abc-881f-377159a2d2b1", + "portId": "splitter_source" + } + }, + "15096fc3-d464-4566-b9e7-1fd786d087a8": { + "id": "15096fc3-d464-4566-b9e7-1fd786d087a8", + "from": { + "nodeId": "8f44f21c-829c-4abc-881f-377159a2d2b1", + "portId": "splitter_target" + }, + "to": { + "nodeId": "1d67688c-536d-48ef-a40f-4b0aaa3f3638", + "portId": "lines" + } + }, + "01c23e09-97cb-4bb4-b965-c40ae9fa9b66": { + "id": "01c23e09-97cb-4bb4-b965-c40ae9fa9b66", + "from": { + "nodeId": "f847ca0a-1a4d-4e2a-85bd-6ec7be87c3ed", + "portId": "lines" + }, + "to": { + "nodeId": "473de030-05fc-4f09-9325-11b838bcc140", + "portId": "lines" + } + }, + "03a6f9db-f510-46e5-89d0-3235c2a51a6b": { + "id": "03a6f9db-f510-46e5-89d0-3235c2a51a6b", + "from": { + "nodeId": "473de030-05fc-4f09-9325-11b838bcc140", + "portId": "count" + }, + "to": { + "nodeId": "62fe958c-92d6-4be1-bbcc-73bfe88b9e2a", + "portId": "count" + } + }, + "4f74db47-a87d-4eb6-abf6-fd101ee4056c": { + "id": "4f74db47-a87d-4eb6-abf6-fd101ee4056c", + "from": { + "nodeId": "62fe958c-92d6-4be1-bbcc-73bfe88b9e2a", + "portId": "a" + }, + "to": { + "nodeId": "57c8b452-35f5-4b7b-b777-e2bee59ef99d", + "portId": "a" + } + }, + "cc20739a-d3a0-4286-a428-23ab1eaf883e": { + "id": "cc20739a-d3a0-4286-a428-23ab1eaf883e", + "from": { + "nodeId": "473de030-05fc-4f09-9325-11b838bcc140", + "portId": "count" + }, + "to": { + "nodeId": "57c8b452-35f5-4b7b-b777-e2bee59ef99d", + "portId": "count" + } + }, + "cb0a4d7c-c379-4194-9a8a-14d0adf2744c": { + "id": "cb0a4d7c-c379-4194-9a8a-14d0adf2744c", + "from": { + "nodeId": "57c8b452-35f5-4b7b-b777-e2bee59ef99d", + "portId": "msg" + }, + "to": { + "nodeId": "12ae508b-b472-4924-af9a-a3051c7612fa", + "portId": "msg" + } + }, + "1f8b4ec5-977c-4cde-9302-f43b134513e3": { + "id": "1f8b4ec5-977c-4cde-9302-f43b134513e3", + "from": { + "nodeId": "57c8b452-35f5-4b7b-b777-e2bee59ef99d", + "portId": "msg" + }, + "to": { + "nodeId": "c8cf52ae-446f-402e-8ea5-d26dbf4ff903", + "portId": "msg" + } + }, + "673af78d-9cc3-4348-83e3-cf9620d1a41d": { + "id": "673af78d-9cc3-4348-83e3-cf9620d1a41d", + "from": { + "nodeId": "12ae508b-b472-4924-af9a-a3051c7612fa", + "portId": "list_of_ints" + }, + "to": { + "nodeId": "c0f6bebc-1529-4901-85d7-42eb1342868e", + "portId": "splitter_source" + } + }, + "0508241e-6b83-46d2-be50-64b9e4b48e0a": { + "id": "0508241e-6b83-46d2-be50-64b9e4b48e0a", + "from": { + "nodeId": "12ae508b-b472-4924-af9a-a3051c7612fa", + "portId": "list_of_paths" + }, + "to": { + "nodeId": "edd68621-780a-42df-958c-74f0a711b260", + "portId": "splitter_source" + } + }, + "928092bb-0d60-4c46-9aef-0bd524cadc06": { + "id": "928092bb-0d60-4c46-9aef-0bd524cadc06", + "from": { + "nodeId": "c8cf52ae-446f-402e-8ea5-d26dbf4ff903", + "portId": "list_of_paths" + }, + "to": { + "nodeId": "8f94be49-2e5a-4b22-bef9-7a2d35b05d76", + "portId": "list_of_paths" + } + }, + "b1b4c25d-114c-4ade-bfaf-5055e8347806": { + "id": "b1b4c25d-114c-4ade-bfaf-5055e8347806", + "from": { + "nodeId": "edd68621-780a-42df-958c-74f0a711b260", + "portId": "splitter_target" + }, + "to": { + "nodeId": "1a85bb42-8eba-4b46-b691-ac0687985fab", + "portId": "list_of_paths" + } + }, + "822e43b9-bf44-4dd5-9132-2cd4301e2ce8": { + "id": "822e43b9-bf44-4dd5-9132-2cd4301e2ce8", + "from": { + "nodeId": "c0f6bebc-1529-4901-85d7-42eb1342868e", + "portId": "splitter_target" + }, + "to": { + "nodeId": "01a6d66f-afb1-47c4-8f94-faace4874409", + "portId": "list_of_ints" + } + }, + "4e0cbf61-f09e-4864-841f-c675d95daf46": { + "id": "4e0cbf61-f09e-4864-841f-c675d95daf46", + "from": { + "nodeId": "c8cf52ae-446f-402e-8ea5-d26dbf4ff903", + "portId": "list_of_ints" + }, + "to": { + "nodeId": "f5733922-b56b-497e-8066-c96cdb7b2a79", + "portId": "list_of_ints" + } + }, + "95a9c637-214c-44d4-8042-c20023e0220d": { + "id": "95a9c637-214c-44d4-8042-c20023e0220d", + "from": { + "nodeId": "f5733922-b56b-497e-8066-c96cdb7b2a79", + "portId": "a" + }, + "to": { + "nodeId": "c64cf08d-2caa-46bb-a199-c8fd25e9d590", + "portId": "a" + } + }, + "e72a8541-2a9c-4c88-afa7-0a159ea3fda4": { + "id": "e72a8541-2a9c-4c88-afa7-0a159ea3fda4", + "from": { + "nodeId": "f5733922-b56b-497e-8066-c96cdb7b2a79", + "portId": "a" + }, + "to": { + "nodeId": "223011e6-faff-479b-a859-a3f77db0fbb5", + "portId": "a" + } + }, + "94a01312-1a30-41f4-8959-e12a0c01269f": { + "id": "94a01312-1a30-41f4-8959-e12a0c01269f", + "from": { + "nodeId": "c64cf08d-2caa-46bb-a199-c8fd25e9d590", + "portId": "msg" + }, + "to": { + "nodeId": "e9e3a59e-923f-4d05-b3b0-2ec79590acc9", + "portId": "msg" + } + }, + "38d3d6f4-d54a-435d-b290-b489c8d2460b": { + "id": "38d3d6f4-d54a-435d-b290-b489c8d2460b", + "from": { + "nodeId": "223011e6-faff-479b-a859-a3f77db0fbb5", + "portId": "msg" + }, + "to": { + "nodeId": "cd47ca61-ba20-4826-b92f-7ee990822f3a", + "portId": "msg" + } + } + }, + "selected": {}, + "hovered": {} + }, + "params": {} +} diff --git a/vreapis/tests/resources/workflows/argo/argo_workflow.json b/vreapis/tests/resources/workflows/argo/argo_workflow.json new file mode 100644 index 00000000..1a430a74 --- /dev/null +++ b/vreapis/tests/resources/workflows/argo/argo_workflow.json @@ -0,0 +1,37 @@ +{ + "namespace": "argo", + "serverDryRun": false, + "workflow": { + "metadata": { + "generateName": "hello-world-", + "namespace": "argo", + "labels": { + "workflows.argoproj.io/completed": "false" + } + }, + "spec": { + "templates": [ + { + "name": "whalesay", + "arguments": {}, + "inputs": {}, + "outputs": {}, + "metadata": {}, + "container": { + "name": "", + "image": "docker/whalesay:latest", + "command": [ + "cowsay" + ], + "args": [ + "hello world" + ], + "resources": {} + } + } + ], + "entrypoint": "whalesay", + "arguments": {} + } + } +} \ No newline at end of file diff --git a/vreapis/utils/cors.py b/vreapis/utils/cors.py new file mode 100644 index 00000000..d926cb0f --- /dev/null +++ b/vreapis/utils/cors.py @@ -0,0 +1,7 @@ +import django.http +import rest_framework.request + + +def get_CORS_headers(request: django.http.HttpRequest | rest_framework.request.Request) -> dict[str, str]: + Origin: str = request.META.get('HTTP_ORIGIN') + return {'Access-Control-Allow-Origin': Origin} diff --git a/vreapis/vreapis/settings/base.py b/vreapis/vreapis/settings/base.py index d445320b..09e72d9c 100644 --- a/vreapis/vreapis/settings/base.py +++ b/vreapis/vreapis/settings/base.py @@ -18,11 +18,12 @@ SECRET_KEY = get_random_secret_key() -DEBUG = True +DEBUG = False ALLOWED_HOSTS = ['*'] -CSRF_TRUSTED_ORIGINS = [os.getenv('TRUSTED_ORIGINS')] +# CSRF_TRUSTED_ORIGINS = [os.getenv('TRUSTED_ORIGINS')] +# CSRF_TRUSTED_ORIGINS = os.getenv('TRUSTED_ORIGINS').split(sep=';') CORS_ALLOWED_ORIGIN_REGEXES = [ r'^http:\/\/localhost:\d+$', @@ -34,11 +35,12 @@ # Application definition INSTALLED_APPS = [ - 'cells', + 'catalog', 'workflows', 'virtual_labs', 'data_products', 'paas_configuration', + 'containerizer', 'rest_framework', 'rest_framework_gis', 'rest_framework.authtoken', @@ -177,6 +179,7 @@ }, } +APPEND_SLASH = False BASE_PATH = os.environ.get('BASE_PATH', '').strip('/') @@ -184,3 +187,17 @@ if BASE_PATH: STATIC_URL = f'{BASE_PATH}/static/' STATIC_ROOT = BASE_DIR / "staticfiles" + +# CUSTOM VARS # todo. delete default values + +# API_ENDPOINT: str = os.getenv('API_ENDPOINT', "https://naavre-dev.minikube.test/vre-api-test") +# +# KEYCLOAK_URL: str = os.getenv('KEYCLOAK_URL', 'https://naavre-dev.minikube.test/auth') +# KEYCLOAK_REALM: str = os.getenv('KEYCLOAK_REALM', 'vre') +# KEYCLOAK_LOGIN_URL: str = f'{KEYCLOAK_URL}/realms/{KEYCLOAK_REALM}/protocol/openid-connect/token' +# KEYCLOAK_VERIF_URL: str = f'{KEYCLOAK_URL}/realms/{KEYCLOAK_REALM}/protocol/openid-connect/certs' +# KEYCLOAK_CLIENT_ID: str = os.getenv('KEYCLOAK_CLIENT_ID', 'myclient') +# +# NAAVRE_API_TOKEN: str = os.getenv('NAAVRE_API_TOKEN') + +VENV_ACTIVATOR: str = os.getenv('VENV_ACTIVATOR', '/opt/venv/bin/activate') diff --git a/vreapis/vreapis/urls.py b/vreapis/vreapis/urls.py index 16312f26..2639f1df 100644 --- a/vreapis/vreapis/urls.py +++ b/vreapis/vreapis/urls.py @@ -16,8 +16,9 @@ from django.contrib import admin from django.urls import path, include from rest_framework import routers + +import containerizer.views from virtual_labs.views import VirtualLabViewSet, VirtualLabInstanceViewSet -from cells.views import CellsViewSet from workflows.views import WorkflowViewSet from data_products.views import DataProductsViewSet, GeoDataProductsViewSet from paas_configuration.views import PaasConfigurationViewSet @@ -31,14 +32,17 @@ router.register(r'vlabs', VirtualLabViewSet) router.register(r'vlab_instances', VirtualLabInstanceViewSet) router.register(r'workflows', WorkflowViewSet) -router.register(r'cells', CellsViewSet) router.register(r'dataprods', DataProductsViewSet) router.register(r'geodataprods', GeoDataProductsViewSet) router.register(r'paas_configuration', PaasConfigurationViewSet) urlpatterns = [ path('admin/', admin.site.urls), - path('api/', include(router.urls)) + path('api/', include(router.urls)), + path('api/containerizer/', include('containerizer.urls')), + path('api/catalog/cells/all', containerizer.views.CellsHandler.as_view({ + 'get': 'list', + })), ] if BASE_PATH: