Dev backend libs upgrade

.github/workflows/auto_tests.yml

@@ -4,7 +4,7 @@ on: push
 
 jobs:
   server_tests:
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
 
     services:
       postgres:
@@ -30,8 +30,8 @@ jobs:
         run: |
           cd server
           sudo apt-get -y install libsqlite3-mod-spatialite
-          pip3 install pipenv==2024.0.1
-          pipenv install --dev --verbose
+          pip install pipenv==2026.0.3
+          pipenv install --dev --verbose --python 3.12
 
       - name: Run tests
         run: |

server/Dockerfile

@@ -1,4 +1,4 @@
-FROM ubuntu:jammy-20240627.1
+FROM ubuntu:noble-20251013
 MAINTAINER Martin Varga "martin.varga@lutraconsulting.co.uk"
 
 # this is to do choice of timezone upfront, because when "tzdata" package gets installed,
@@ -19,10 +19,6 @@ RUN apt-get update -y && \
     gcc build-essential binutils cmake extra-cmake-modules libsqlite3-mod-spatialite libmagic1 && \
     rm -rf /var/lib/apt/lists/*
 
-
-# needed for geodiff
-RUN pip3 install --upgrade pip==24.0
-
 # create mergin user to run container with
 RUN groupadd -r mergin -g 901
 RUN groupadd -r mergin-family -g 999
@@ -32,12 +28,16 @@ RUN useradd -u 901 -r --home-dir /app --create-home -g mergin -G mergin-family -
 COPY . /app
 WORKDIR /app
 
-RUN pip3 install pipenv==2024.0.1
+# keep installing to system packages
+ENV PIP_BREAK_SYSTEM_PACKAGES=1
+ENV PIP_IGNORE_INSTALLED=1
+
+RUN pip install pipenv==2026.0.3
 # for locale check this http://click.pocoo.org/5/python3/
 ENV LC_ALL=C.UTF-8
 ENV LANG=C.UTF-8
 
-RUN pipenv install --system --deploy --verbose
+RUN pipenv install --system --deploy --verbose --python 3.12
 
 USER mergin
 

server/Pipfile

@@ -4,25 +4,25 @@ verify_ssl = true
 name = "pypi"
 
 [packages]
-connexion = {extras = ["swagger-ui"],version = "==2.14.1"}
-flask = "==2.2.5"
+connexion = {extras = ["swagger-ui"],version = "==2.15.1"}
+flask = "==3.1.2"
 python-dateutil = "==2.8.2"
-marshmallow = "==3.20.1"
-flask-marshmallow = "==0.14.0"
-marshmallow-sqlalchemy = "==1.1.0"
+marshmallow = "==3.26.1"
+flask-marshmallow = "==0.15.0"
+marshmallow-sqlalchemy = "==1.4.1"
 psycopg2-binary = "==2.9.9"
 itsdangerous = "==2.2.0"
-Flask-SQLAlchemy = "==2.5.1"
-sqlalchemy = "==1.4.53"
-gunicorn = {extras = ["gevent"],version = "==19.9"}
+Flask-SQLAlchemy = "==3.1.1"
+sqlalchemy = "==2.0.44"
+gunicorn = {extras = ["gevent"],version = "==23.0"}
 python-dotenv = "==0.20.0"
-flask-login = "==0.6.2"
+flask-login = "==0.6.3"
 bcrypt = "==4.2.0"
-wtforms = {extras = ["email"],version = "==3.1.2"}
-flask-wtf = "==1.0.1"
+wtforms = {extras = ["email"],version = "==3.2.1"}
+flask-wtf = "==1.2.2"
 flask-mail = "==0.10.0"
 safe = "==0.4"
-flask-migrate = "==2.6.0" # 3.1.0
+flask-migrate = "==3.1.0"
 wtforms-json = "==0.3.5"
 pytz = "==2022.2.1"
 scikit-build = "==0.18.1"
@@ -57,4 +57,4 @@ pre-commit = "==4.1.0"
 atomicwrites = "==1.4.0"
 
 [requires]
-python_version = "3.10"
+python_version = "3.12"

server/mergin/app.py

@@ -10,6 +10,7 @@
 import gevent
 from marshmallow import fields
 from sqlalchemy.schema import MetaData
+from sqlalchemy import text
 from flask_sqlalchemy import SQLAlchemy
 from flask_marshmallow import Marshmallow
 from flask import (
@@ -27,7 +28,6 @@
 from flask_wtf.csrf import generate_csrf, CSRFProtect
 from flask_migrate import Migrate
 from flask_mail import Mail
-from connexion.apps.flask_app import FlaskJSONEncoder
 from flask_wtf import FlaskForm
 from wtforms import StringField
 from pathlib import Path
@@ -37,7 +37,6 @@
 from werkzeug.exceptions import HTTPException
 from typing import List, Dict, Optional, Tuple
 
-from .sync.utils import get_blacklisted_dirs, get_blacklisted_files
 from .config import Configuration
 from .commands import add_commands as server_commands
 
@@ -139,7 +138,6 @@ def create_simple_app() -> Flask:
     app = connexion.FlaskApp(__name__, specification_dir=os.path.join(this_dir))
     flask_app = app.app
 
-    flask_app.json_encoder = FlaskJSONEncoder
     flask_app.config.from_object(Configuration)
     db.init_app(flask_app)
     ma.init_app(flask_app)
@@ -155,54 +153,36 @@ def create_simple_app() -> Flask:
 def create_app(public_keys: List[str] = None) -> Flask:
     """Factory function to create Flask app instance"""
     from itsdangerous import BadTimeSignature, BadSignature
-    from .auth import auth_required, decode_token
-    from .auth.models import User
 
-    # from .celery import celery
-    from .sync.db_events import register_events
-    from .sync.workspace import GlobalWorkspaceHandler
-    from .sync.config import Configuration as SyncConfig
-    from .sync.commands import add_commands
-    from .auth import register as register_auth
+    from .auth import auth_required, decode_token, register as register_auth
+    from .auth.models import User
+    from .sync.app import register as register_sync
     from .sync.project_handler import ProjectHandler
+    from .sync.utils import get_blacklisted_dirs, get_blacklisted_files
+    from .sync.workspace import GlobalWorkspaceHandler
 
     app = create_simple_app().connexion_app
 
-    app.add_api(
-        "sync/public_api.yaml",
-        arguments={"title": "Mergin"},
-        options={"swagger_ui": Configuration.SWAGGER_UI},
-        validate_responses=True,
-    )
-    app.add_api(
-        "sync/public_api_v2.yaml",
-        arguments={"title": "Mergin"},
-        options={"swagger_ui": Configuration.SWAGGER_UI},
-        validate_responses=True,
-    )
-    app.add_api(
-        "sync/private_api.yaml",
-        base_path="/app",
-        arguments={"title": "Mergin"},
-        options={"swagger_ui": False, "serve_spec": False},
-        validate_responses=True,
-    )
     app.add_api(
         "api.yaml",
         arguments={"title": "Mergin"},
         options={"swagger_ui": False, "serve_spec": False},
         validate_responses=True,
     )
+    app.app.blueprints["/"].name = "main"
+    app.app.blueprints["main"] = app.app.blueprints.pop("/")
 
-    app.app.config.from_object(SyncConfig)
-    app.app.connexion_app = app
+    # register sync module
+    register_sync(app.app)
 
+    # initialize extensions
     mail.init_app(app.app)
-    app.mail = mail
     csrf.init_app(app.app)
     login_manager.init_app(app.app)
+
     # register auth blueprint
     register_auth(app.app)
+
     server_commands(app.app)
 
     # adjust login manager
@@ -228,8 +208,6 @@ def load_user_from_header(header_val):  # pylint: disable=W0613,W0612
             except (BadSignature, BadTimeSignature, KeyError):
                 pass
 
-    # csrf = app.app.extensions['csrf']
-
     @app.app.before_request
     def check_maintenance():
         allowed_endpoints = ["/project/by_names", "/auth/login", "/alive"]
@@ -275,9 +253,6 @@ def get_startup_data():
         }
         return data
 
-    # update celery config with flask app config
-    # celery.conf.update(app.app.config)
-
     @app.route("/alive", methods=["POST"])
     @csrf.exempt
     def alive():  # pylint: disable=E0722
@@ -287,7 +262,7 @@ def alive():  # pylint: disable=E0722
         start_time = time.time()
         try:
             with db.engine.connect() as con:
-                rs = con.execute("SELECT 2 * 2")
+                rs = con.execute(text("SELECT 2 * 2"))
                 assert rs.fetchone()[0] == 4
         except:
             """Although bad form, we have deliberate left this except broad. When we have an uncaught exception in
@@ -388,7 +363,6 @@ def init():  # pylint: disable=W0612
         response.headers.set("X-CSRF-Token", generate_csrf())
         return response
 
-    register_events()
     application = app.app
 
     @application.errorhandler(Exception)
@@ -467,8 +441,6 @@ def config():
             cfg["build_hash"] = application.config["BUILD_HASH"]
             return jsonify(cfg), 200
 
-    # append project commands (from default sync module)
-    add_commands(application)
     return application
 
 

server/mergin/auth/controller.py

@@ -396,7 +396,7 @@ def register_user():  # pylint: disable=W0613,W0612
 
     form = UserRegistrationForm()
     form.username.data = User.generate_username(form.email.data)
-    if form.validate_on_submit():
+    if form.is_submitted() and form.validate():
         user = User.create(form.username.data, form.email.data, form.password.data)
         user_created.send(user, source="admin")
         token = generate_confirmation_token(
@@ -500,8 +500,9 @@ def get_paginated_users(
     elif not descending and order_by:
         users = users.order_by(asc(User.__table__.c[order_by]))
 
-    result = users.paginate(page, per_page).items
-    total = users.paginate(page, per_page).total
+    paginate = users.paginate(page=page, per_page=per_page)
+    result = paginate.items
+    total = paginate.total
 
     result_users = UserSchema(many=True).dump(result)
 

server/mergin/auth/tasks.py

@@ -14,7 +14,7 @@
 @celery.task
 def anonymize_removed_users():
     """Permanently 'delete' users marked for removal by removing personal information"""
-    db.session.info = {"msg": "anonymize_removed_users"}
+    db.session.info["msg"] = "anonymize_removed_users"
     before_expiration = datetime.today() - timedelta(Configuration.ACCOUNT_EXPIRATION)
     users = User.query.filter(
         isnot(User.active, True),

server/mergin/commands.py

@@ -7,6 +7,7 @@
 import random
 import string
 import os
+from sqlalchemy import inspect
 
 
 def _echo_title(title):
@@ -136,7 +137,8 @@ def _check_server(app: Flask):  # pylint: disable=W0612
     else:
         _echo_error("No service ID set.")
 
-    tables = db.engine.table_names()
+    inspect_engine = inspect(db.engine)
+    tables = inspect_engine.get_table_names()
     if not tables:
         _echo_error("Database not initialized. Run flask init-db command")
     else:
@@ -157,9 +159,9 @@ def _init_db(app: Flask):
         label="Creating database", length=4, show_eta=False
     ) as progress_bar:
         progress_bar.update(0)
-        db.drop_all(bind=None)
+        db.drop_all(bind_key=None)
         progress_bar.update(1)
-        db.create_all(bind=None)
+        db.create_all(bind_key=None)
         progress_bar.update(2)
         db.session.commit()
         progress_bar.update(3)
@@ -202,7 +204,8 @@ def init(email: str, recreate: bool):
         """Initialize database if does not exist or -r is provided. Perform check of server configuration. Send statistics, respecting your setup."""
         from .auth.models import User, UserProfile
 
-        tables = db.engine.table_names()
+        inspect_engine = inspect(db.engine)
+        tables = inspect_engine.get_table_names()
         if recreate and tables:
             click.confirm(
                 "Are you sure you want to recreate database and admin user? This will remove all data!",

server/mergin/controller.py

@@ -1,15 +1,14 @@
-import json
-import logging
-import os
+# Copyright (C) Lutra Consulting Limited
+#
+# SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-MerginMaps-Commercial
+
 from flask import abort, current_app, request
 from flask_login import current_user
 from magic import from_buffer
-import time
-
 import requests
 
 from .utils import save_diagnostic_log_file
-from .app import parse_version_string, db
+from .app import parse_version_string
 
 
 def get_latest_version():

server/mergin/stats/models.py

@@ -2,11 +2,12 @@
 #
 # SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-MerginMaps-Commercial
 
+import uuid
 from dataclasses import dataclass
 from typing import Optional
-import uuid
-from sqlalchemy.dialects.postgresql import UUID, JSONB
-from datetime import datetime, timezone
+from datetime import datetime
+from sqlalchemy.dialects.postgresql import JSONB
+from sqlalchemy.orm import Mapped, mapped_column
 
 from ..app import db
 
@@ -30,8 +31,8 @@ class ServerCallhomeData:
 class MerginInfo(db.Model):
     """Information about deployment"""
 
-    service_id = db.Column(UUID(as_uuid=True), primary_key=True)
-    last_reported = db.Column(db.DateTime)
+    service_id: Mapped[uuid.UUID] = mapped_column(primary_key=True)
+    last_reported: Mapped[Optional[datetime]]
 
     def __init__(self, service_id: str = None):
         if service_id:
@@ -43,9 +44,7 @@ def __init__(self, service_id: str = None):
 class MerginStatistics(db.Model):
     """Information about deployment"""
 
-    id = db.Column(db.Integer, primary_key=True, autoincrement=True)
-    created_at = db.Column(
-        db.DateTime, index=True, nullable=False, default=datetime.utcnow
-    )
+    id: Mapped[int] = mapped_column(primary_key=True)
+    created_at: Mapped[datetime] = mapped_column(index=True, default=datetime.utcnow)
     # data with statistics
-    data = db.Column(JSONB, nullable=False)
+    data: Mapped[dict] = mapped_column(JSONB)