Feature/local deployment

ansible/dispatcher-local.yml

@@ -0,0 +1,47 @@
+---
+- hosts: localhost
+  connection: local
+  become: true
+  become_user: root
+  tasks:
+    - name: Copy local version of repo
+      synchronize:
+        src: ../../
+        dest: "{{ dispatcher.dir }}"
+
+    - name: Install docker-compose-local.yml
+      ansible.builtin.template:
+        src: docker-compose-local.yml.j2
+        dest: "{{ dispatcher.dir }}/docker-compose.yml"
+
+    - name: Install ocm.json
+      ansible.builtin.template:
+        src: ocm.json.j2
+        dest: "{{ dispatcher.dir }}/nginx/ocm.json"
+
+    - name: Install nginx.conf
+      ansible.builtin.template:
+        src: nginx.conf.j2
+        dest: "{{ dispatcher.dir }}/nginx/nginx.conf"
+
+    - name: Install api-config file
+      ansible.builtin.template:
+        src: config.py.j2
+        dest: "{{ dispatcher.dir }}/app/config.py"
+
+    - name: Run generate certificates
+      community.docker.docker_compose_v2_run:
+        project_src: "{{ dispatcher.dir }}"
+        service: "certgen"
+
+    - name: Ensure docker-compose app is running
+      community.docker.docker_compose_v2:
+        project_src: "{{ dispatcher.dir }}"
+        state: present 
+        build: "always"
+
+    - name: Clean dangling images
+      community.docker.docker_prune:
+        images: true
+        images_filters:
+          dangling: true

ansible/group_vars/dispatchers/vars.yml → ansible/group_vars/all.yml

@@ -3,6 +3,7 @@ dispatcher:
   branch: master
   dir: /opt/edc/dispatcher
 
+
 git:
   # macos
   # backend: /Library/Developer/CommandLineTools/usr/libexec/git-core/git-http-backend
@@ -40,12 +41,4 @@ im:
   wait:
     max_time: 36000 # 10h
     sleep: 30
-    max_retries: 3
-    # # region: RegionOne
-
-    # In case of EGI Cloud Compute site
-    # In this case, the Dispatcher must use the EGI Check-in production instance to authenticate users
-    #type: demo
-    #site: "CESNET-MCCG2"
-    #VO: vo.eosc-data-commons.eu
-
+    max_retries: 3

ansible/host_vars/localhost/vars.yml

@@ -0,0 +1,3 @@
+oauth2:
+  client_id: "abc"
+  client_secret: "abcd"

ansible/hosts.yml

@@ -8,3 +8,6 @@ dispatchers:
       ansible_user: ubuntu
     dev4.player.eosc-data-commons.eu:
       ansible_user: ubuntu
+local:
+  hosts:
+    localhost

ansible/templates/docker-compose-local.yml.j2

@@ -0,0 +1,81 @@
+services:
+  redis:
+    image: redis:7
+
+  web:
+    build: .
+    command: uvicorn app.main:app --host 0.0.0.0 --ssl-keyfile {{nginx.cert}}/privkey.pem --ssl-certfile {{nginx.cert}}/fullchain.pem --log-level debug --reload 
+    ports:
+      - 8004:8000
+    volumes:
+      - ./app:/usr/src/app
+      - ./nginx/certbot/conf:{{nginx.cert}}
+    environment:
+      - CELERY_BROKER_URL=redis://redis:{{ redis.port }}/0
+      - CELERY_RESULT_BACKEND=redis://redis:{{ redis.port }}/0
+      - ENV=development
+#    env_file:
+#      - .env
+    depends_on:
+      - redis
+      - certgen
+
+  worker:
+    build: .
+    command: celery -A app.celery.worker.celery worker --loglevel=info 
+    volumes:
+      - git-repos-volume:/{{git.repos}}
+      - .:/usr/src
+    user: "celery" 
+    environment:
+      - CELERY_BROKER_URL=redis://redis:{{ redis.port }}/0
+      - CELERY_RESULT_BACKEND=redis://redis:{{ redis.port }}/0
+      - ENV=development
+    depends_on:
+      - web
+      - redis
+
+  nginx:
+    image: nginxinc/nginx-unprivileged:latest
+    ports:
+      - "443:443"
+    volumes:
+      - ./:/code
+      - ./nginx/certbot/conf:{{nginx.cert}}
+      - ./nginx/certbot/www:/var/www/certbot
+      - ./nginx/nginx.conf:/etc/nginx/nginx.conf
+      - ./nginx/ocm.json:/etc/nginx/.well-known/ocm.json
+      - fcgi-socket:/nginx/temp
+    depends_on:
+      - web
+      - git-fcgi
+      - certgen
+
+  git-fcgi:
+    image: alpine:latest
+    volumes:
+      - git-repos-volume:{{git.repos}}
+      - fcgi-socket:/var/run/fcgiwrap
+    command: >
+      sh -c "
+        apk add --no-cache fcgiwrap spawn-fcgi git git-daemon;
+        adduser -D -H -u 1000 -G www-data www-data 2>/dev/null || true;
+        git config --global --add safe.directory '*';
+        chmod 777 {{git.repos}};
+        mkdir -p /var/run/fcgiwrap;
+        chmod 755 /var/run/fcgiwrap;
+        spawn-fcgi -s /var/run/fcgiwrap/fcgi.sock -M 666 -- /usr/bin/fcgiwrap;
+        tail -f /dev/null
+      "
+
+  certgen:
+    image: alpine:latest
+    volumes:
+        - ./nginx/certbot/conf:/etc/ssl/nginx
+    command: >
+        sh -c "apk add --no-cache openssl && echo 'Generating self-signed certs...' && openssl req -x509 -nodes -days 365 -newkey rsa:2048 -keyout /etc/ssl/nginx/privkey.pem -out /etc/ssl/nginx/fullchain.pem -subj '/CN=localhost' -addext 'subjectAltName=DNS:localhost,DNS:*.localhost,DNS:host.docker.internal,IP:127.0.0.1,IP:::1' && chmod 644 /etc/ssl/nginx/privkey.pem && chmod 644 /etc/ssl/nginx/fullchain.pem && echo 'Done'"
+networks:
+  default:
+volumes:
+  git-repos-volume:
+  fcgi-socket:

app/celery/tasks.py

@@ -17,7 +17,7 @@ def vre_from_zipfile(self, parsed_zipfile: tuple[Dict, bytes], token):
     vre_handler = vre_factory(
         crate=crate, body=zip_file, token=token, update_state=self.update_state
     )
-    return {"url": vre_handler.post(self.request.id)}
+    return {"url": vre_handler.post()}
 
 
 @celery.task(
@@ -30,4 +30,4 @@ def vre_from_zipfile(self, parsed_zipfile: tuple[Dict, bytes], token):
 def vre_from_rocrate(self, data: Dict, token):
     crate = ROCrate(source=copy.deepcopy(data))
     vre_handler = vre_factory(crate=crate, token=token, update_state=self.update_state)
-    return {"url": vre_handler.post(self.request.id)}
+    return {"url": vre_handler.post()}

app/main.py

@@ -1,3 +1,5 @@
+import os
+from typing import Optional
 from app.routers import requests, auth, anonymous_requests
 from fastapi import FastAPI
 from fastapi_oauth2.middleware import OAuth2Middleware
@@ -23,19 +25,36 @@ class TestEGICheckinOpenIdConnect(EGICheckinOpenIdConnect):
     CHECKIN_ENV = settings.egi_checkin_env
 
 
-client = OAuth2Client(
-    backend=TestEGICheckinOpenIdConnect,
-    scope=[
-        "openid email profile entitlements voperson_id voperson_external_affiliation eduperson_entitlement"
-    ],
-    client_id=settings.client_id,
-    client_secret=settings.client_secret,
-    redirect_uri=settings.redirect_uri,
-    claims=Claims(
-        identity=lambda user: f"{user.provider}:{user.id}",
-    ),
-)
+class DummyOAuth2Client(OAuth2Client):
+    async def authenticate(self, code: Optional[str] = None, **kwargs):
+        return {
+            "id": "dummy-user",
+            "email": "dev@example.com",
+            "name": "Dev User",
+            "picture": None,
+        }
+
+
+def get_oauth2_client() -> OAuth2Client:
+    if os.getenv("ENV") == "development":
+        return DummyOAuth2Client(
+            backend=TestEGICheckinOpenIdConnect, client_id="", client_secret=""
+        )
+    return OAuth2Client(
+        backend=TestEGICheckinOpenIdConnect,
+        scope=[
+            "openid email profile entitlements voperson_id voperson_external_affiliation eduperson_entitlement"
+        ],
+        client_id=settings.client_id,
+        client_secret=settings.client_secret,
+        redirect_uri=settings.redirect_uri,
+        claims=Claims(
+            identity=lambda user: f"{user.provider}:{user.id}",
+        ),
+    )
+
 
 app.add_middleware(
-    OAuth2Middleware, config=OAuth2Config(clients=[client], same_site="none")
+    OAuth2Middleware,
+    config=OAuth2Config(clients=[get_oauth2_client()], same_site="none"),
 )

app/vres/base_vre.py

@@ -25,6 +25,7 @@ def __init__(
         self,
         crate: Any | None = None,
         update_state: Optional[Callable] = None,
+        request_id: Optional[int] = None,
         body: Any | None = None,
         token: str | None = None,
         im_factory: Callable[[str | None], IMClientProtocol] | None = None,
@@ -34,6 +35,7 @@ def __init__(
         self.body = body
         self.token = token
         self._update_state = update_state
+        self._request_id = request_id
         self._im_factory = im_factory or self._default_im_factory
         self.svc_url = self.setup_service().rstrip("/")
         # Store any additional kwargs for subclasses
@@ -110,6 +112,7 @@ def __call__(
         crate,
         body=None,
         update_state: Optional[Callable] = None,
+        request_id: Optional[int] = None,
         **kwargs,
     ):
         elang = crate.mainEntity.get("programmingLanguage").get("identifier")
@@ -119,7 +122,11 @@ def __call__(
         logger.debug(f"elang {elang}")
         logger.debug(self.table[elang])
         return self.table[elang](
-            crate=crate, body=body, update_state=update_state, **kwargs
+            crate=crate,
+            body=body,
+            update_state=update_state,
+            request_id=request_id,
+            **kwargs,
         )
 
 

app/vres/binder.py

@@ -18,13 +18,13 @@ class VREBinder(VRE):
     def get_default_service(self):
         return BINDER_DEFAULT_SERVICE
 
-    def post(self, request_id):
-        repo = self._generate_repository_name(request_id)
+    def post(self):
+        repo = self._generate_repository_name(self._request_id)
         self._create(repo)
         self._write_source_files(repo)
         self._initialize_temporary_git_repo(repo)
         self._write_example_file(repo)
-        return self._get_binder_url(request_id)
+        return self._get_binder_url(self._request_id)
 
     def _create(self, repo):
         os.mkdir(repo)

test/conftest.py

@@ -149,7 +149,7 @@ def binder_vre(dummy_binder_crate):
     vre.crate = dummy_binder_crate
     vre.svc_url = "https://mybinder.org"
     vre.body = create_test_zip_body()
-
+    vre._request_id = 0
     return vre
 
 

test/unit/test_binder.py

@@ -7,10 +7,9 @@
 
 
 def test_post_happy_path(binder_vre):
-    request_id = "abcd1234"
-    local_git_url = f"https://{settings.host}/git/{request_id}"
+    local_git_url = f"https://{settings.host}/git/{binder_vre._request_id}"
 
-    final_url = binder_vre.post(request_id)
+    final_url = binder_vre.post()
 
     assert (
         final_url
@@ -19,49 +18,41 @@ def test_post_happy_path(binder_vre):
 
 
 def test_post_dir_created(binder_vre, tmpdir):
-    request_id = "abcd1234"
+    binder_vre.post()
 
-    binder_vre.post(request_id)
-
-    assert os.path.isdir(f"{tmpdir}/{request_id}")
+    assert os.path.isdir(f"{tmpdir}/{binder_vre._request_id}")
 
 
 def test_post_git_repo_initialized(binder_vre, tmpdir):
-    request_id = "abcd1234"
-
-    binder_vre.post(request_id)
+    binder_vre.post()
 
-    assert os.path.isdir(f"{tmpdir}/{request_id}/.git")
+    assert os.path.isdir(f"{tmpdir}/{binder_vre._request_id}/.git")
 
 
 def test_post_git_deamon_export_created(binder_vre, tmpdir):
-    request_id = "abcd1234"
-
-    binder_vre.post(request_id)
+    binder_vre.post()
 
-    assert os.path.isfile(f"{tmpdir}/{request_id}/.git/git-daemon-export-ok")
+    assert os.path.isfile(
+        f"{tmpdir}/{binder_vre._request_id}/.git/git-daemon-export-ok"
+    )
 
 
 def test_post_git_commit(binder_vre, tmpdir):
-    request_id = "abcd1234"
-
-    binder_vre.post(request_id)
-    repo = Repo(f"{tmpdir}/{request_id}")
+    binder_vre.post()
+    repo = Repo(f"{tmpdir}/{binder_vre._request_id}")
 
     assert repo.head.commit.message == "on the fly"
 
 
 def test_post_permission_denied(binder_vre):
-    request_id = "abcd1234"
     settings.git_repos = "/"
 
     with pytest.raises(PermissionError):
-        binder_vre.post(request_id)
+        binder_vre.post()
 
 
 def test_post_not_found(binder_vre, tmpdir):
-    request_id = "abcd1234"
     settings.git_repos = f"{tmpdir}/../abc"
 
     with pytest.raises(FileNotFoundError):
-        binder_vre.post(request_id)
+        binder_vre.post()