Accumulated changes to the integration branch

.env.dist

@@ -0,0 +1,5 @@
+# Use the Official DBpedia Spotlight API ...
+DBPEDIA_SPOTLIGHT_URL="https://api.dbpedia-spotlight.org/en/annotate"
+
+# ... or a custom  DBpedia Spotlight API endpoint
+#DBPEDIA_SPOTLIGHT_URL="https://example.org/rest/annotate"

.github/workflows/build.yml

@@ -35,7 +35,7 @@ jobs:
 
     steps:
     - name: Checkout repository
-      uses: actions/checkout@v4
+      uses: actions/checkout@v6
 
     - name: Set up QEMU
       uses: docker/setup-qemu-action@v3

.github/workflows/release.yml

@@ -11,7 +11,7 @@ jobs:
 
     steps:
     - name: Checkout repository
-      uses: actions/checkout@v4
+      uses: actions/checkout@v6
 
     - name: Check for stable semver
       id:   semver

.gitignore

@@ -14,7 +14,7 @@ pids
 *.pid
 *.seed
 *.pid.lock
-public/uploads
+/webserver/public/uploads
 
 # Directory for instrumented libs generated by jscoverage/JSCover
 lib-cov
@@ -266,3 +266,9 @@ ser_data.txt
 coursemapper-kg/app/algorithms/elmo_2x4096_512_2048cnn_2xhighway_options.json
 coursemapper-kg/app/algorithms/elmo_2x4096_512_2048cnn_2xhighway_weights.hdf5
 
+coursemapper-kg/=2.2.2
+venv*
+.DS_Store
+
+coursemapper-kg/app
+webserver/public/uploads/pdfs

.k8s/base/coursemapper-kg/concept-map/deployment.yaml

@@ -32,6 +32,9 @@ spec:
           value: postgres://postgres:password@$(WP_PG_SERVICE):5432
         - name: WIKIPEDIA_FALLBACK
           value: "false"
+        envFrom:
+        - configMapRef:
+            name: dbpedia-spotlight
         resources:
           requests:
             cpu: 100m

.k8s/base/coursemapper-kg/concept-map/kustomization.yaml

@@ -1,8 +1,10 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 
-commonLabels:
-  app.kubernetes.io/component: coursemapper-kg-concept-map
+labels:
+- includeSelectors: true
+  pairs:
+    app.kubernetes.io/component: coursemapper-kg-concept-map
 
 resources:
 - deployment.yaml

.k8s/base/coursemapper-kg/pgadmin/ingress.yaml

@@ -5,7 +5,6 @@ metadata:
   annotations:
     kubernetes.io/tls-acme: "true"
     cert-manager.io/cluster-issuer: letsencrypt-prod
-    traefik.ingress.kubernetes.io/router.middlewares: default-headers@kubernetescrd
 spec:
   rules:
   - host: pgadmin.$(FQDN)

.k8s/base/coursemapper-kg/pgadmin/kustomization.yaml

@@ -1,8 +1,10 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 
-commonLabels:
-  app.kubernetes.io/component: coursemapper-kg-pgadmin
+labels:
+- includeSelectors: true
+  pairs:
+    app.kubernetes.io/component: coursemapper-kg-pgadmin
 
 resources:
 - deployment.yaml

.k8s/base/coursemapper-kg/recommendation/kustomization.yaml

@@ -1,8 +1,10 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 
-commonLabels:
-  app.kubernetes.io/component: coursemapper-kg-recommendation
+labels:
+- includeSelectors: true
+  pairs:
+    app.kubernetes.io/component: coursemapper-kg-recommendation
 
 resources:
 - deployment.yaml

.k8s/base/coursemapper-kg/wp-pg/kustomization.yaml

@@ -1,8 +1,10 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 
-commonLabels:
-  app.kubernetes.io/component: coursemapper-kg-wp-pg
+labels:
+- includeSelectors: true
+  pairs:
+    app.kubernetes.io/component: coursemapper-kg-wp-pg
 
 resources:
 - deployment.yaml

.k8s/base/ingress.yaml

@@ -6,7 +6,6 @@ metadata:
     description: CourseMapper-webserver
     kubernetes.io/tls-acme: "true"
     cert-manager.io/cluster-issuer: letsencrypt-prod
-    traefik.ingress.kubernetes.io/router.middlewares: default-headers@kubernetescrd
 spec:
   rules:
   - host: $(FQDN)

.k8s/base/kustomization.yaml

@@ -19,6 +19,9 @@ configMapGenerator:
 - name: ingress
   literals:
   - host=example.org
+- name: dbpedia-spotlight
+  literals: []
+  # - DBPEDIA_SPOTLIGHT_URL=https://example.org/rest/annotate
 
 images:
 - name: coursemapper-kg-concept-map

.k8s/base/mongodb/deployment.yaml

@@ -26,10 +26,10 @@ spec:
         resources:
           requests:
             memory: 128Mi
-            cpu: 10m
+            cpu: 500m
           limits:
             memory: 512Mi
-            cpu: 2
+            cpu: 1150m
             ephemeral-storage: 1Gi
       containers:
       - name: mongodb

.k8s/base/webserver/neo4j/deployment.yaml

@@ -20,10 +20,10 @@ spec:
         resources:
           requests:
             cpu: 12m
-            memory: 800Mi
+            memory: 1100Mi
           limits:
             cpu: 4
-            memory: 1.2Gi
+            memory: 1.5Gi
         volumeMounts:
         - name: data
           mountPath: "/data"

.k8s/base/webserver/neo4j/kustomization.yaml

@@ -1,8 +1,10 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 
-commonLabels:
-  app.kubernetes.io/component: webserver-neo4j
+labels:
+- includeSelectors: true
+  pairs:
+    app.kubernetes.io/component: webserver-neo4j
 
 resources:
 - deployment.yaml

.k8s/base/webserver/redis/kustomization.yaml

@@ -1,8 +1,10 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 
-commonLabels:
-  app.kubernetes.io/component: webserver-redis
+labels:
+- includeSelectors: true
+  pairs:
+    app.kubernetes.io/component: webserver-redis
 
 resources:
 - deployment.yaml

.k8s/base/webserver/web/deployment.yaml

@@ -28,6 +28,9 @@ spec:
           httpGet:
             path: /api/healthz
             port: 8080
+          timeoutSeconds: 30
+          failureThreshold: 5
+          periodSeconds: 60
         env:
         - name: PORT
           value: "8080"
@@ -39,6 +42,13 @@ spec:
           value: $(REDIS_SERVICE)
         - name: REDIS_PORT
           value: "6379"
+        - name: MEMORY_REQUEST
+          valueFrom:
+            resourceFieldRef:
+              divisor: "1Mi"
+              resource: requests.memory
+        - name: NODE_OPTIONS
+          value: "--max-old-space-size=$(MEMORY_REQUEST)"
         envFrom:
         - secretRef:
             name: webserver
@@ -54,11 +64,11 @@ spec:
           subPath: videos
         resources:
           requests:
-            memory: 360Mi
+            memory: 2Gi
             cpu: 100m
           limits:
-            memory: 1Gi
-            cpu: "1"
+            memory: 2.5Gi
+            cpu: "1.15"
             ephemeral-storage: 1Gi
       volumes:
       - name: webserver-uploads

.k8s/base/webserver/web/kustomization.yaml

@@ -1,8 +1,10 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 
-commonLabels:
-  app.kubernetes.io/component: webserver-web
+labels:
+- includeSelectors: true
+  pairs:
+    app.kubernetes.io/component: webserver-web
 
 resources:
 - deployment.yaml

Makefile

@@ -1,6 +1,7 @@
 help:
 	@cat $(MAKEFILE_LIST) | docker run --rm -i --platform linux/amd64 xanders/make-help
 
+export COMPOSE_BAKE=1
 compose = docker compose -f compose.yaml
 
 all: clean build run
@@ -32,7 +33,7 @@ build:
 
 # Push container images to remote registry
 push:
-	@docker compose push
+	@$(compose) push
 
 # Start all services for development using Tilt for live container updates
 tilt:

README.md

@@ -6,27 +6,27 @@ CourseMapper is a collaborative course annotation and analytics platform that fo
 
 ## 🚀 Get Started
 
-#### Live instances
+#### 🌎 Live instances
 
 - Production:
   [coursemapper.de](https://coursemapper.de/)
   ([branch `main`](https://github.com/ude-soco/CourseMapper-webserver/tree/main))
-  ![status](https://argocd.cluster.soco.inko.cloud/api/badge?name=cmw-prod)
+  ![production status](https://argocd.soco.ude.systems/api/badge?name=cmw-prod)
 - Preview:
   [br-dev.coursemapper.de](https://br-dev.coursemapper.de/)
   ([branch `dev`](https://github.com/ude-soco/CourseMapper-webserver/tree/dev))
-  ![status](https://argocd.cluster.soco.inko.cloud/api/badge?name=cmw-br-dev)
+  ![branch preview status](https://argocd.soco.ude.systems/api/badge?name=cmw-br-dev)
 
 *Note:* Stable [releases](https://github.com/ude-soco/CourseMapper-webserver/releases) are currently not running in production.
 
-#### Build and run
+#### 🐳 Compose
 
-- `make up` to run the application using _Docker Compose_
-- `make tilt` to automatically rebuild during development using _Tilt_
-- `make mounted` to run processes using _Docker Compose_, but mount source code from host machine
-- see the manual below to install dependencies and run processes _locally, without containers_
-
-Visit the [proxy service on port 8000](http://localhost:8000/) to use the application.
+1. Set up `.env` file (see `.env.dist`)
+2. Build and run
+   - `make up` to run the application using _Docker Compose_
+   - `make tilt` to automatically rebuild during development using _Tilt_
+   - `make mounted` to run processes using _Docker Compose_, but mount source code from host machine
+3. Visit the [proxy service on port 8000](http://localhost:8000/) to use the application.
 
 ## 🖥️ Application stack
 

coursemapper-kg/concept-map/Dockerfile

@@ -1,4 +1,4 @@
-# syntax=docker/dockerfile:1.6
+# syntax=docker/dockerfile:1.15
 FROM alpine as MODELS
 WORKDIR /download
 
@@ -7,7 +7,7 @@ ADD https://uni-duisburg-essen.sciebo.de/s/nO06q2wY0t5h8SO/download stanford-cor
 RUN unzip stanford-corenlp-full-2018-02-27.zip -d stanford-corenlp-full-2018-02-27 && rm stanford-corenlp-full-2018-02-27.zip
 
 
-from python:3.11.5
+FROM python:3.11.5
 
 # APT requires archive URLs
 COPY debian/sources.list /etc/apt/sources.list
@@ -30,10 +30,10 @@ USER 1000
 WORKDIR /home/app
 
 # Configure Python environment
-ENV PYTHONDONTWRITEBYTECODE 1
-ENV PYTHONUNBUFFERED 1
-ENV PIPENV_VENV_IN_PROJECT 1
-ENV PIPENV_VERBOSITY -1
+ENV PYTHONDONTWRITEBYTECODE=1
+ENV PYTHONUNBUFFERED=1
+ENV PIPENV_VENV_IN_PROJECT=1
+ENV PIPENV_VERBOSITY=-1
 ENV VIRTUAL_ENV=/home/app/.venv
 RUN python -m venv $VIRTUAL_ENV
 ENV PATH="$VIRTUAL_ENV/bin:$PATH"
@@ -42,7 +42,7 @@ ENV PATH="$VIRTUAL_ENV/bin:$PATH"
 COPY --chown=app:app Pipfile* ./
 RUN  --mount=type=cache,sharing=private,target=/home/app/.cache/pip,uid=1000 \
      --mount=type=cache,sharing=private,target=/home/app/.cache/pypoetry,uid=1000 <<EOF
-  pip install --no-color pipenv
+  pip install --no-color --disable-pip-version-check pipenv
   pipenv install --deploy
 EOF
 
@@ -63,6 +63,6 @@ COPY --from=MODELS --chown=app /download/* .
 # Add source code
 COPY . .
 
-ENV OMP_NUM_THREADS 1
+ENV OMP_NUM_THREADS=1
 EXPOSE $PORT
 CMD [ "bin/worker" ]

coursemapper-kg/concept-map/compose.yaml

@@ -14,6 +14,7 @@ services:
       PIPELINES: concept-map
       WIKIPEDIA_DATABASE_CONNECTION_STRING: "postgres://postgres:password@wp-pg:5432/"
       WIKIPEDIA_FALLBACK: false
+      DBPEDIA_SPOTLIGHT_URL: ${DBPEDIA_SPOTLIGHT_URL}
     depends_on:
       neo4j:
         condition: service_healthy

coursemapper-kg/concept-map/example.env

@@ -1,9 +1,15 @@
-WIKIPEDIA_DATABASE_CONNECTION_STRING=
+PIPELINES=concept-map,modify-graph,expand-material
+
 NEO4J_URI=bolt://localhost:7687
 NEO4J_USER=neo4j
 NEO4J_PASSWORD=1234qwer!
+
 REDIS_HOST=localhost
 REDIS_PORT=6379
 REDIS_DATABASE=0
 REDIS_PASSWORD=
-PIPELINES=concept-map,modify-graph,expand-material
+
+WIKIPEDIA_DATABASE_CONNECTION_STRING=
+
+DBPEDIA_SPOTLIGHT_URL="https://api.dbpedia-spotlight.org/en/annotate" # official DBpedia Spotlight API ...
+#DBPEDIA_SPOTLIGHT_URL="https://example.org/rest/annotate"             # ... or custom instance

coursemapper-kg/concept-map/src/config.py

@@ -19,9 +19,7 @@ class Config:
     NEO4J_PASSWORD = os.getenv('NEO4J_PASSWORD', 'password')
     NEO4J_SAVE_TO_DB = os.getenv('NEO4J_SAVE_TO_DB', 'true').lower() == 'true'
 
-    DBPEDIA_SPOTLIGHT_URL = os.getenv('DBPEDIA_SPOTLIGHT_URL', 'https://dbpedia-spotlight-en.soco.inko.cloud/rest/annotate')
-    # Uncomment the line below to use the official DBpedia Spotlight API
-    # DBPEDIA_SPOTLIGHT_URL = os.getenv('DBPEDIA_SPOTLIGHT_URL', 'https://api.dbpedia-spotlight.org/en/annotate')
+    DBPEDIA_SPOTLIGHT_URL = os.getenv('DBPEDIA_SPOTLIGHT_URL')
     DBPEDIA_SPOTLIGHT_CONFIDENCE = float(os.getenv('DBPEDIA_SPOTLIGHT_CONFIDENCE', '0.35'))
     DBPEDIA_SPOTLIGHT_SUPPORT = int(os.getenv('DBPEDIA_SPOTLIGHT_SUPPORT', '5'))
 

coursemapper-kg/concept-map/src/graph_db.py

@@ -27,6 +27,7 @@ def load_graph(self, material_id: str):
 
     def save_graph(self, graph: Graph):
         with self.driver.session() as session:
+            # session.execute_write(_save_graph, graph),
             session.write_transaction(_save_graph, graph)
 
     def get_node(self, material_id: str, node_id: str):