Dev r85 v2 pull

deployment/community/docker-compose.yml

@@ -23,7 +23,7 @@ services:
     networks:
       - merginmaps
   server:
-    image: lutraconsulting/merginmaps-backend:2025.2.2
+    image: lutraconsulting/merginmaps-backend:2025.7.3
     container_name: merginmaps-server
     restart: always
     user: 901:999
@@ -39,7 +39,7 @@ services:
     networks:
       - merginmaps
   celery-beat:
-    image: lutraconsulting/merginmaps-backend:2025.2.2
+    image: lutraconsulting/merginmaps-backend:2025.7.3
     container_name: celery-beat
     restart: always
     env_file:
@@ -54,7 +54,7 @@ services:
     networks:
       - merginmaps
   celery-worker:
-    image: lutraconsulting/merginmaps-backend:2025.2.2
+    image: lutraconsulting/merginmaps-backend:2025.7.3
     container_name: celery-worker
     restart: always
     user: 901:999
@@ -73,7 +73,7 @@ services:
     networks:
       - merginmaps
   web:
-    image: lutraconsulting/merginmaps-frontend:2025.2.2
+    image: lutraconsulting/merginmaps-frontend:2025.7.3
     container_name: merginmaps-web
     restart: always
     depends_on:

deployment/enterprise/.env.template

@@ -117,7 +117,7 @@ LOCAL_PROJECTS=/data
 
 # data download
 
-#MAX_DOWNLOAD_ARCHIVE_SIZE=1024 * 1024 * 1024  # max total files size in bytes for archive download
+#MAX_DOWNLOAD_ARCHIVE_SIZE=1024 * 1024 * 1024 * 10  # max total files size in bytes for archive download
 
 #USE_X_ACCEL=False  # use nginx (in front of gunicorn) to serve files (https://www.nginx.com/resources/wiki/start/topics/examples/x-accel/)
 USE_X_ACCEL=1
@@ -214,12 +214,8 @@ VECTOR_TILES_URL=https://tiles-ee.merginmaps.com/data/default/{z}/{x}/{y}.pbf
 
 VECTOR_TILES_STYLE_URL=https://tiles-ee.merginmaps.com//styles/default.json
 
-#QGIS_EXTRACTOR_API_URL=http://mergin-qgis-extractor:8000
-
 #WMTS_SERVER_URL=http://mergin-qgis-nginx:80
 
-#QGIS_EXTRACTOR_TIMEOUT=60
-
 #OVERVIEW_MAX_FILE_SIZE=1048576 # 1MB
 
 ### Diagnostic logs from Mobile and QGIS Plugin

deployment/enterprise/docker-compose.maps.yml

@@ -30,13 +30,16 @@ services:
       - ./qgis_nginx.conf:/etc/nginx/conf.d/default.conf
   qgis_extractor:
     container_name: mergin-qgis-extractor
-    image: 433835555346.dkr.ecr.eu-west-1.amazonaws.com/mergin/qgis-extractor-ee:2025.1.0
+    image: 433835555346.dkr.ecr.eu-west-1.amazonaws.com/mergin/qgis-extractor-ee:2025.3.0
     user: 901:999
     networks:
       - mergin-net
     environment:
       - OVERVIEWS_DATA_DIR=/data
       - MM_WMS_TILE_BUFFER=100
       - MM_WMS_AVOID_ARTIFACTS=1
+      - BROKER_URL=redis://merginmaps-redis:6379/0
+      - CELERY_RESULT_BACKEND=redis://merginmaps-redis:6379/0
+      - 'CELERY_TASK_ROUTES={"src.maps.tasks.finish_overview": {"queue": "celery"}}'
     volumes:
       - ./map_data:/data

deployment/enterprise/docker-compose.yml

@@ -5,7 +5,7 @@ networks:
 
 services:
   server:
-    image: 433835555346.dkr.ecr.eu-west-1.amazonaws.com/mergin/mergin-ee-back:2025.5.1
+    image: 433835555346.dkr.ecr.eu-west-1.amazonaws.com/mergin/mergin-ee-back:2025.7.3
     container_name: merginmaps-server
     restart: always
     user: 901:999
@@ -22,7 +22,7 @@ services:
     networks:
       - mergin
   web:
-    image: 433835555346.dkr.ecr.eu-west-1.amazonaws.com/mergin/mergin-ee-front:2025.5.1
+    image: 433835555346.dkr.ecr.eu-west-1.amazonaws.com/mergin/mergin-ee-front:2025.7.3
     container_name: merginmaps-web
     restart: always
     depends_on:
@@ -52,7 +52,7 @@ services:
       - server
 
   celery-beat:
-    image: 433835555346.dkr.ecr.eu-west-1.amazonaws.com/mergin/mergin-ee-back:2025.5.1
+    image: 433835555346.dkr.ecr.eu-west-1.amazonaws.com/mergin/mergin-ee-back:2025.7.3
     container_name: merginmaps-celery-beat
     restart: always
     user: 901:999
@@ -66,7 +66,7 @@ services:
       - mergin
 
   celery-worker:
-    image: 433835555346.dkr.ecr.eu-west-1.amazonaws.com/mergin/mergin-ee-back:2025.5.1
+    image: 433835555346.dkr.ecr.eu-west-1.amazonaws.com/mergin/mergin-ee-back:2025.7.3
     container_name: merginmaps-celery-worker
     restart: always
     user: 901:999
@@ -76,6 +76,8 @@ services:
       - ./map_data:/overviews
     env_file:
       - .prod.env
+    environment:
+      - CELERY_ROUTES={ 'qgis-extractor.*':{'queue':'extractor'} }
     depends_on:
       - db
       - redis

server/mergin/sync/commands.py

@@ -127,3 +127,50 @@ def remove(project_name):
         project.removed_by = None
         db.session.commit()
         click.secho("Project removed", fg="green")
+
+    @project.command()
+    @click.argument("project-name", callback=normalize_input(lowercase=False))
+    @click.option("--since", type=int, required=False)
+    @click.option("--to", type=int, required=False)
+    def create_checkpoint(project_name, since=None, to=None):
+        """Create project delta checkpoint, corresponding lower checkpoints and merged diffs for project"""
+        ws, name = split_project_path(project_name)
+        workspace = current_app.ws_handler.get_by_name(ws)
+        if not workspace:
+            click.secho("ERROR: Workspace does not exist", fg="red", err=True)
+            sys.exit(1)
+        project = (
+            Project.query.filter_by(workspace_id=workspace.id, name=name)
+            .filter(Project.storage_params.isnot(None))
+            .first()
+        )
+        if not project:
+            click.secho("ERROR: Project does not exist", fg="red", err=True)
+            sys.exit(1)
+
+        since = since if since is not None else 0
+        to = to if to is not None else project.latest_version
+        if since < 0 or to < 1:
+            click.secho(
+                "ERROR: Invalid version number, minimum version for 'since' is 0 and minimum version for 'to' is 1",
+                fg="red",
+                err=True,
+            )
+            sys.exit(1)
+
+        if to > project.latest_version:
+            click.secho(
+                "ERROR: 'to' version exceeds latest project version", fg="red", err=True
+            )
+            sys.exit(1)
+
+        if since >= to:
+            click.secho(
+                "ERROR: 'since' version must be less than 'to' version",
+                fg="red",
+                err=True,
+            )
+            sys.exit(1)
+
+        project.get_delta_changes(since, to)
+        click.secho("Project checkpoint(s) created", fg="green")

server/mergin/sync/config.py

@@ -75,6 +75,8 @@ class Configuration(object):
     UPLOAD_CHUNKS_EXPIRATION = config(
         "UPLOAD_CHUNKS_EXPIRATION", default=86400, cast=int
     )
+    # whether client can pull using v2 apis
+    V2_PULL_ENABLED = config("V2_PULL_ENABLED", default=True, cast=bool)
     EXCLUDED_CLONE_FILENAMES = config(
         "EXCLUDED_CLONE_FILENAMES", default="qgis_cfg.xml", cast=Csv()
     )

server/mergin/sync/errors.py

@@ -95,3 +95,10 @@ def to_dict(self) -> Dict:
 class BigChunkError(ResponseError):
     code = "BigChunkError"
     detail = f"Chunk size exceeds maximum allowed size {MAX_CHUNK_SIZE} MB"
+
+
+class DiffDownloadError(ResponseError):
+    code = "DiffDownloadError"
+    detail = (
+        "Required diff file could not be downloaded as it could not be reconstructed"
+    )

server/mergin/sync/files.py

@@ -4,11 +4,18 @@
 import datetime
 from enum import Enum
 import os
-from dataclasses import dataclass
+from dataclasses import dataclass, field
 from typing import Optional, List
 import uuid
 from flask import current_app
-from marshmallow import ValidationError, fields, EXCLUDE, post_dump, validates_schema
+from marshmallow import (
+    ValidationError,
+    fields,
+    EXCLUDE,
+    post_dump,
+    validates_schema,
+    post_load,
+)
 from pathvalidate import sanitize_filename
 
 from .utils import (
@@ -231,11 +238,125 @@ def validate(self, data, **kwargs):
 
 class ProjectFileSchema(FileSchema):
     mtime = DateTimeWithZ()
-    diff = fields.Nested(FileSchema())
+    diff = fields.Nested(
+        FileSchema(),
+    )
 
     @post_dump
     def patch_field(self, data, **kwargs):
         # drop 'diff' key entirely if empty or None as clients would expect
         if not data.get("diff"):
             data.pop("diff", None)
         return data
+
+
+@dataclass
+class DeltaDiffFile:
+    """Diff file path in diffs list"""
+
+    id: str
+
+
+class DeltaChangeDiffFileSchema(ma.Schema):
+    """Schema for diff file path in diffs list"""
+
+    id = fields.String(required=True)
+
+
+@dataclass
+class DeltaChangeBase(File):
+    """Base class for changes stored in json list or returned from delta endpoint"""
+
+    change: PushChangeType
+    version: int
+
+
+@dataclass
+class DeltaChangeMerged(DeltaChangeBase):
+    """Delta item with merged diffs to list of multiple diff files"""
+
+    diffs: List[DeltaDiffFile] = field(default_factory=list)
+
+    def to_data_delta(self):
+        """Convert DeltaMerged to DeltaData with single diff"""
+        result = DeltaChange(
+            path=self.path,
+            size=self.size,
+            checksum=self.checksum,
+            change=self.change,
+            version=self.version,
+        )
+        if self.diffs:
+            result.diff = self.diffs[0].id
+        return result
+
+
+@dataclass
+class DeltaChange(DeltaChangeBase):
+    """Change items stored in database as list of this item with single diff file"""
+
+    diff: Optional[str] = None
+
+    def to_merged(self) -> DeltaChangeMerged:
+        """Convert to DeltaMerged with multiple diffs"""
+        result = DeltaChangeMerged(
+            path=self.path,
+            size=self.size,
+            checksum=self.checksum,
+            change=self.change,
+            version=self.version,
+        )
+        if self.diff:
+            result.diffs = [DeltaDiffFile(id=self.diff)]
+        return result
+
+
+class DeltaChangeBaseSchema(ma.Schema):
+    """Base schema for delta json and response from delta endpoint"""
+
+    path = fields.String(required=True)
+    size = fields.Integer(required=True)
+    checksum = fields.String(required=True)
+    version = fields.Integer(required=True)
+    change = fields.Enum(PushChangeType, by_value=True, required=True)
+
+
+class DeltaChangeSchema(DeltaChangeBaseSchema):
+    """Schema for change data in changes column"""
+
+    diff = fields.String(required=False)
+
+    @post_load
+    def make_object(self, data, **kwargs):
+        return DeltaChange(**data)
+
+    @post_dump
+    def patch_field(self, data, **kwargs):
+        # drop 'diff' key entirely if empty or None as database would expect
+        if not data.get("diff"):
+            data.pop("diff", None)
+        return data
+
+
+class DeltaChangeItemSchema(DeltaChangeBaseSchema):
+    """Schema for delta changes response"""
+
+    version = fields.Function(lambda obj: f"v{obj.version}")
+    diffs = fields.List(fields.Nested(DeltaChangeDiffFileSchema()))
+
+    @post_dump
+    def patch_field(self, data, **kwargs):
+        # drop 'diffs' key entirely if empty or None as clients would expect
+        if not data.get("diffs"):
+            data.pop("diffs", None)
+        return data
+
+
+class DeltaChangeRespSchema(ma.Schema):
+    """Schema for list of delta changes wrapped in items field"""
+
+    items = fields.List(fields.Nested(DeltaChangeItemSchema()))
+    to_version = fields.String(required=True)
+
+    class Meta:
+        unknown = EXCLUDE