From 60882efabf64d1352b2005d4f78e2306d9a7ad8f Mon Sep 17 00:00:00 2001 From: Mirko Dietrich Date: Thu, 6 Nov 2025 15:21:00 +0100 Subject: [PATCH 1/4] feat(backend): add `add_options_file` and `get_options_file` to `FilesystemStateManager` Ref: #214 --- questionpy_sdk/webserver/errors.py | 4 + questionpy_sdk/webserver/server.py | 2 +- questionpy_sdk/webserver/state.py | 103 +++++- tests/questionpy_sdk/webserver/test_state.py | 322 +++++++++++++++++- .../webserver/test_webserver.py | 13 +- 5 files changed, 411 insertions(+), 33 deletions(-) diff --git a/questionpy_sdk/webserver/errors.py b/questionpy_sdk/webserver/errors.py index 161fd5f6..628a447c 100644 --- a/questionpy_sdk/webserver/errors.py +++ b/questionpy_sdk/webserver/errors.py @@ -28,6 +28,10 @@ class DuplicateQuestionError(StateError): message = "The question already exists." +class MissingOptionsFileError(MissingStateError): + message = "An options file is missing." + + class MissingAttemptStateError(MissingStateError): message = "The attempt state is missing." diff --git a/questionpy_sdk/webserver/server.py b/questionpy_sdk/webserver/server.py index 4380ccf7..5168d017 100644 --- a/questionpy_sdk/webserver/server.py +++ b/questionpy_sdk/webserver/server.py @@ -95,7 +95,7 @@ async def __aenter__(self) -> Self: # Initialize state manager pkg_dirname = f"{self._manifest.namespace}-{self._manifest.short_name}-{self._manifest.version}" - self._state_manager = FilesystemStateManager(self._state_storage_root / pkg_dirname) + self._state_manager = FilesystemStateManager(self._state_storage_root, pkg_dirname) # Create web app self._app = self._create_webapp() diff --git a/questionpy_sdk/webserver/state.py b/questionpy_sdk/webserver/state.py index 735ac470..9a88b421 100644 --- a/questionpy_sdk/webserver/state.py +++ b/questionpy_sdk/webserver/state.py @@ -3,19 +3,25 @@ # (c) Technische Universität Berlin, innoCampus import asyncio -import contextlib import json import re +import shutil +from collections.abc import Iterator +from contextlib import AbstractContextManager, contextmanager, suppress from enum import StrEnum from pathlib import Path -from typing import NamedTuple, Protocol +from typing import Any, BinaryIO, NamedTuple, Protocol -from pydantic import JsonValue, TypeAdapter +from pydantic import JsonValue, RootModel, TypeAdapter, ValidationError import questionpy_sdk.webserver.errors as webserver_errors from questionpy import ScoreModel +from questionpy.form import OptionsFile from questionpy_sdk.webserver.constants import ID_RE +OptionsFiles = RootModel[dict[str, OptionsFile]] +"""A mapping of file_refs to OptionsFiles.""" + class Attempt(NamedTuple): state: str @@ -37,6 +43,11 @@ async def write_question_state(self, question_id: str, question_state: str) -> N async def delete_question(self, question_id: str) -> None: ... async def delete_all_questions(self) -> None: ... + async def add_options_file(self, file_ref: str, filepath: Path) -> None: ... + async def get_options_file( + self, question_id: str, name: str, file_ref: str + ) -> tuple[OptionsFile, AbstractContextManager[BinaryIO]]: ... + async def read_attempts(self, question_id: str) -> dict[str, Attempt]: ... async def read_attempt_state(self, question_id: str, attempt_id: str) -> str: ... async def write_attempt_state(self, question_id: str, attempt_id: str, attempt_state: str) -> None: ... @@ -70,8 +81,13 @@ class StateFilename(StrEnum): ATTEMPT_SCORE = "score.json" ATTEMPT_DATA = "attempt_data.json" - def __init__(self, package_state_dir: Path) -> None: - self._package_state_dir = package_state_dir + OPTIONS_FILES_DIR = "options_files" + + def __init__(self, storage_root: Path, package_state_dir: str) -> None: + self._storage_root = storage_root + self._package_state_path = storage_root / package_state_dir + + # ------ Questions async def read_question_states(self) -> dict[str, str]: def _read_question_states() -> dict[str, str]: @@ -113,7 +129,7 @@ def _delete_question_sync(self, question_id: str) -> None: if self._is_id_dir(path): self._delete_attempt_data_sync(question_id, path.name) - with contextlib.suppress(OSError): + with suppress(OSError): # Ignore in case of non-empty directory question_path.rmdir() @@ -124,6 +140,73 @@ def _delete_all_questions() -> None: await asyncio.to_thread(_delete_all_questions) + # ------ Options files + + async def add_options_file(self, file_ref: str, filepath: Path) -> None: + await asyncio.to_thread(self._add_options_file_sync, file_ref, filepath) + + def _add_options_file_sync(self, file_ref: str, filepath: Path) -> None: + target_path = self._get_sharded_path(file_ref) + # If the same content exists already, no need to overwrite + if not target_path.exists(): + target_path.parent.mkdir(parents=True, exist_ok=True) + shutil.move(filepath, target_path) + + async def get_options_file( + self, question_id: str, name: str, file_ref: str + ) -> tuple[OptionsFile, AbstractContextManager[BinaryIO]]: + state = await self.read_question_state(question_id) + options_file = self._find_file(json.loads(state)["options"], name, file_ref) + content_path = self._get_sharded_path(file_ref) + + @contextmanager + def read_bytes() -> Iterator[BinaryIO]: + try: + with content_path.open("rb") as f: + yield f + except FileNotFoundError as err: + raise webserver_errors.MissingOptionsFileError from err + + return options_file, read_bytes() + + @staticmethod + def _find_file(opts: dict[str, Any], name: str, file_ref: str) -> OptionsFile: + """Find an `OptionsFile` inside the form data.""" + + def get_item(cur: dict[str, Any] | list[Any], p: str | int) -> Any: + # Makes mypy happy about cur/p types + if isinstance(p, int): + if not isinstance(cur, list): + msg = f"Expected list for index {p}, got {type(cur).__name__}" + raise TypeError(msg) + return cur[p] + if not isinstance(cur, dict): + msg = f"Expected dict for key {p}, got {type(cur).__name__}" + raise TypeError(msg) + return cur[p] + + # Parse path + parts = [int(p) if p.isdigit() else p for p in name.split(".")] + if parts and parts[0] == "general": + parts = parts[1:] # treat `general` as root-level + + cur: dict[str, Any] | list[Any] = opts + with suppress(KeyError, IndexError, TypeError, StopIteration, ValidationError): + # Descend into options tree + for p in parts: + cur = get_item(cur, p) + if isinstance(cur, dict) and cur.get("file_ref") == file_ref: + # File found + return OptionsFile.model_validate(cur) + + raise webserver_errors.MissingOptionsFileError + + def _get_sharded_path(self, file_ref: str) -> Path: + """Get sharded file path, e.g. `options_files/ab/abcdef...`.""" + return self._storage_root / self.OPTIONS_FILES_DIR / file_ref[:2] / file_ref + + # ------ Attempts + async def read_attempts(self, question_id: str) -> dict[str, Attempt]: return await asyncio.to_thread(self._read_attempts_sync, question_id) @@ -220,10 +303,12 @@ def _delete_attempt_data_sync(self, question_id: str, attempt_id: str) -> None: self.StateFilename.ATTEMPT_SEED, ): (attempt_path / fname).unlink(missing_ok=True) - with contextlib.suppress(OSError): + with suppress(OSError): # Ignore in case of non-empty directory attempt_path.rmdir() + # ------ Helpers + async def _read_state_file(self, path: Path, filename: "FilesystemStateManager.StateFilename") -> str: return await asyncio.to_thread((path / filename).read_text) @@ -235,14 +320,14 @@ def _write_state_file() -> None: await asyncio.to_thread(_write_state_file) def _get_question_path(self, question_id: str) -> Path: - return self._package_state_dir / question_id + return self._package_state_path / question_id def _get_attempt_path(self, question_id: str, attempt_id: str) -> Path: return self._get_question_path(question_id) / attempt_id def _get_question_state_paths(self) -> list[Path]: try: - return [p for p in self._package_state_dir.iterdir() if self._is_id_dir(p)] + return [p for p in self._package_state_path.iterdir() if self._is_id_dir(p)] except FileNotFoundError: # Package dir might not have been created yet return [] diff --git a/tests/questionpy_sdk/webserver/test_state.py b/tests/questionpy_sdk/webserver/test_state.py index e93616bd..e877d162 100644 --- a/tests/questionpy_sdk/webserver/test_state.py +++ b/tests/questionpy_sdk/webserver/test_state.py @@ -2,8 +2,9 @@ # The QuestionPy SDK is free software released under terms of the MIT license. See LICENSE.md. # (c) Technische Universität Berlin, innoCampus +import json from pathlib import Path -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any import pytest @@ -17,7 +18,7 @@ @pytest.fixture def state_manager(tmp_path: Path) -> StateManager: - return FilesystemStateManager(tmp_path) + return FilesystemStateManager(tmp_path, "test-test-1.00.0") async def test_read_question_states(state_manager: StateManager) -> None: @@ -50,7 +51,7 @@ async def test_delete_question(state_manager: StateManager, tmp_path: Path) -> N await state_manager.write_attempt_state("myuQ2JWl", "UY9ryXzq", "attempt data") await state_manager.delete_question("myuQ2JWl") - assert not (tmp_path / "myuQ2JWl").exists() + assert not (tmp_path / "test-test-1.00.0" / "myuQ2JWl").exists() async def test_delete_all_questions(state_manager: StateManager, tmp_path: Path) -> None: @@ -61,21 +62,316 @@ async def test_delete_all_questions(state_manager: StateManager, tmp_path: Path) await state_manager.delete_all_questions() - assert not (tmp_path / "myuQ2JWl").exists() - assert not (tmp_path / "nYKEjBaA").exists() - assert not (tmp_path / "5YfGRyRs").exists() + assert not (tmp_path / "test-test-1.00.0" / "myuQ2JWl").exists() + assert not (tmp_path / "test-test-1.00.0" / "nYKEjBaA").exists() + assert not (tmp_path / "test-test-1.00.0" / "5YfGRyRs").exists() async def test_delete_question_leaves_other_files(state_manager: StateManager, tmp_path: Path) -> None: await state_manager.write_question_state("myuQ2JWl", "data") - (tmp_path / "myuQ2JWl" / "some_file").touch() + (tmp_path / "test-test-1.00.0" / "myuQ2JWl" / "some_file").touch() await state_manager.delete_question("myuQ2JWl") - question_path = tmp_path / "myuQ2JWl" + question_path = tmp_path / "test-test-1.00.0" / "myuQ2JWl" assert not (question_path / FilesystemStateManager.StateFilename.QUESTION_STATE).exists() assert question_path.exists() +async def test_add_options_file(state_manager: FilesystemStateManager, tmp_path: Path) -> None: + file_ref = "abcdef1234567890abcdef1234567890abcdef12" + source_file = tmp_path / "source_file.txt" + source_file.write_text("test content") + + await state_manager.add_options_file(file_ref, source_file) + + expected_path = tmp_path / "options_files" / "ab" / file_ref + assert expected_path.exists() + assert expected_path.read_text() == "test content" + assert not source_file.exists() + + +async def test_add_options_file_existing_target(state_manager: FilesystemStateManager, tmp_path: Path) -> None: + file_ref = "abcdef1234567890abcdef1234567890abcdef12" + source_file = tmp_path / "source_file.txt" + source_file.write_text("test content") + + # Create the target file with the same content (same hash) + target_path = tmp_path / "options_files" / "ab" / file_ref + target_path.parent.mkdir(parents=True, exist_ok=True) + target_path.write_text("test content") + original_mtime = target_path.stat().st_mtime + + await state_manager.add_options_file(file_ref, source_file) + + assert target_path.exists() + assert target_path.read_text() == "test content" + + # Check that the source file still exists (was not moved); target was not overwritten + assert source_file.exists() + assert target_path.stat().st_mtime == original_mtime + + +async def test_get_options_file(state_manager: FilesystemStateManager, tmp_path: Path) -> None: + file_ref = "abcdef1234567890abcdef1234567890abcdef12" + source_file = tmp_path / "source_file.txt" + source_file.write_text("test content") + + await state_manager.add_options_file(file_ref, source_file) + + question_state = { + "options": { + "file_upload": [ + { + "path": "/", + "filename": "test_file.txt", + "file_ref": file_ref, + "uploaded_at": "2025-11-06T15:13:15.631214Z", + "mime_type": "text/plain", + "size": 12, + } + ] + } + } + + await state_manager.write_question_state("myuQ2JWl", json.dumps(question_state)) + options_file, file_context = await state_manager.get_options_file("myuQ2JWl", "file_upload.0", file_ref) + + assert options_file.filename == "test_file.txt" + assert options_file.file_ref == file_ref + assert options_file.mime_type == "text/plain" + + with file_context as f: + content = f.read() + assert content == b"test content" + + +async def test_get_options_file_inside_section(state_manager: FilesystemStateManager, tmp_path: Path) -> None: + file_ref = "1234567890abcdef1234567890abcdef12345678" + source_file = tmp_path / "source_file.txt" + source_file.write_text("inside section") + + await state_manager.add_options_file(file_ref, source_file) + + question_state = { + "options": { + "foo_section": { + "foo_file_upload": [ + { + "path": "/", + "filename": "section_file.txt", + "file_ref": file_ref, + "uploaded_at": "2025-11-06T15:13:28.997740Z", + "mime_type": "text/plain", + "size": 14, + } + ] + } + } + } + + await state_manager.write_question_state("myuQ2JWl", json.dumps(question_state)) + options_file, file_context = await state_manager.get_options_file( + "myuQ2JWl", "foo_section.foo_file_upload.0", file_ref + ) + + assert options_file.filename == "section_file.txt" + assert options_file.file_ref == file_ref + assert options_file.mime_type == "text/plain" + + with file_context as f: + content = f.read() + assert content == b"inside section" + + +async def test_get_options_file_inside_group(state_manager: FilesystemStateManager, tmp_path: Path) -> None: + file_ref = "fedcba0987654321fedcba0987654321fedcba09" + source_file = tmp_path / "source_file.txt" + source_file.write_text("inside group") + + await state_manager.add_options_file(file_ref, source_file) + + question_state = { + "options": { + "bar_group": { + "some_input": "", + "bar_file_upload": [ + { + "path": "/", + "filename": "group_file.txt", + "file_ref": file_ref, + "uploaded_at": "2025-11-06T15:13:25.082918Z", + "mime_type": "text/plain", + "size": 12, + } + ], + } + } + } + + await state_manager.write_question_state("myuQ2JWl", json.dumps(question_state)) + options_file, file_context = await state_manager.get_options_file( + "myuQ2JWl", "bar_group.bar_file_upload.0", file_ref + ) + + assert options_file.filename == "group_file.txt" + assert options_file.file_ref == file_ref + assert options_file.mime_type == "text/plain" + + with file_context as f: + content = f.read() + assert content == b"inside group" + + +async def test_get_options_file_inside_repetition(state_manager: FilesystemStateManager, tmp_path: Path) -> None: + file_ref = "0123456789abcdef0123456789abcdef01234567" + source_file = tmp_path / "source_file.txt" + source_file.write_text("inside repetition") + + await state_manager.add_options_file(file_ref, source_file) + + question_state = { + "options": { + "my_repetition": [ + {"some_input": "", "bar_file_upload": []}, + { + "some_input": "", + "bar_file_upload": [ + { + "path": "/", + "filename": "repetition_file.txt", + "file_ref": file_ref, + "uploaded_at": "2025-11-06T15:13:22.512159Z", + "mime_type": "text/plain", + "size": 17, + } + ], + }, + ] + } + } + + await state_manager.write_question_state("myuQ2JWl", json.dumps(question_state)) + options_file, file_context = await state_manager.get_options_file( + "myuQ2JWl", "my_repetition.1.bar_file_upload.0", file_ref + ) + + assert options_file.filename == "repetition_file.txt" + assert options_file.file_ref == file_ref + assert options_file.mime_type == "text/plain" + + with file_context as f: + content = f.read() + assert content == b"inside repetition" + + +@pytest.mark.parametrize( + "path", + [ + "file_upload.0", + "foo_section.foo_file_upload.0", + "bar_group.bar_file_upload.0", + "my_repetition.1.bar_file_upload.0", + ], +) +async def test_get_options_file_missing_question(state_manager: FilesystemStateManager, path: str) -> None: + with pytest.raises(webserver_errors.MissingQuestionStateError): + await state_manager.get_options_file("nonexistent_question", path, "abcdef1234567890abcdef1234567890abcdef12") + + +def make_question_state(file_ref: str) -> dict[str, Any]: + return { + "options": { + "file_upload": [ + { + "path": "/", + "filename": "file.txt", + "file_ref": file_ref, + "uploaded_at": "2025-11-06T15:13:15.631214Z", + "mime_type": "text/plain", + "size": 4, + } + ], + "foo_section": { + "foo_file_upload": [ + { + "path": "/", + "filename": "file.txt", + "file_ref": file_ref, + "uploaded_at": "2025-11-06T15:13:28.997740Z", + "mime_type": "text/plain", + "size": 4, + } + ] + }, + "bar_group": { + "some_input": "", + "bar_file_upload": [ + { + "path": "/", + "filename": "file.txt", + "file_ref": file_ref, + "uploaded_at": "2025-11-06T15:13:25.082918Z", + "mime_type": "text/plain", + "size": 4, + } + ], + }, + "my_repetition": [ + {"some_input": ""}, + { + "some_input": "", + "bar_file_upload": [ + { + "path": "/", + "filename": "file.txt", + "file_ref": file_ref, + "uploaded_at": "2025-11-06T15:13:22.512159Z", + "mime_type": "text/plain", + "size": 4, + } + ], + }, + ], + } + } + + +@pytest.mark.parametrize( + "path", + [ + "file_upload.0", + "foo_section.foo_file_upload.0", + "bar_group.bar_file_upload.0", + "my_repetition.1.bar_file_upload.0", + ], +) +async def test_get_options_file_missing_file(state_manager: FilesystemStateManager, path: str) -> None: + question_state = make_question_state("different_file_ref") + await state_manager.write_question_state("myuQ2JWl", json.dumps(question_state)) + + with pytest.raises(webserver_errors.MissingOptionsFileError): + await state_manager.get_options_file("myuQ2JWl", path, "abcdef1234567890abcdef1234567890abcdef12") + + +@pytest.mark.parametrize( + "path", + [ + "file_upload.0", + "foo_section.foo_file_upload.0", + "bar_group.bar_file_upload.0", + "my_repetition.1.bar_file_upload.0", + ], +) +async def test_get_options_file_missing_content(state_manager: FilesystemStateManager, path: str) -> None: + file_ref = "abcdef1234567890abcdef1234567890abcdef12" + question_state = make_question_state(file_ref) + await state_manager.write_question_state("myuQ2JWl", json.dumps(question_state)) + + _, file_context = await state_manager.get_options_file("myuQ2JWl", path, file_ref) + with pytest.raises(webserver_errors.MissingOptionsFileError), file_context: + pass + + async def test_read_attempts(state_manager: StateManager) -> None: await state_manager.write_question_state("myuQ2JWl", "data1") await state_manager.write_question_state("nYKEjBaA", "data2") @@ -151,15 +447,17 @@ async def test_delete_attempt(state_manager: StateManager, tmp_path: Path) -> No ) await state_manager.delete_attempt("myuQ2JWl", "UY9ryXzq") - assert not (tmp_path / "myuQ2JWl" / "UY9ryXzq").exists() - assert (tmp_path / "myuQ2JWl" / FilesystemStateManager.StateFilename.QUESTION_STATE).exists() + assert not (tmp_path / "test-test-1.00.0" / "myuQ2JWl" / "UY9ryXzq").exists() + assert (tmp_path / "test-test-1.00.0" / "myuQ2JWl" / FilesystemStateManager.StateFilename.QUESTION_STATE).exists() async def test_delete_attempt_leaves_other_files(state_manager: StateManager, tmp_path: Path) -> None: await state_manager.write_attempt_seed("myuQ2JWl", "UY9ryXzq", 123) - some_file_path = tmp_path / "myuQ2JWl" / "UY9ryXzq" / "some_file" + some_file_path = tmp_path / "test-test-1.00.0" / "myuQ2JWl" / "UY9ryXzq" / "some_file" some_file_path.touch() await state_manager.delete_attempt("myuQ2JWl", "UY9ryXzq") - assert not (tmp_path / "myuQ2JWl" / "UY9ryXzq" / FilesystemStateManager.StateFilename.ATTEMPT_SEED).exists() + assert not ( + tmp_path / "test-test-1.00.0" / "myuQ2JWl" / "UY9ryXzq" / FilesystemStateManager.StateFilename.ATTEMPT_SEED + ).exists() assert some_file_path.exists() diff --git a/tests/questionpy_sdk/webserver/test_webserver.py b/tests/questionpy_sdk/webserver/test_webserver.py index e219750e..d2e1b4d8 100644 --- a/tests/questionpy_sdk/webserver/test_webserver.py +++ b/tests/questionpy_sdk/webserver/test_webserver.py @@ -16,7 +16,6 @@ from questionpy_common.api.qtype import QuestionTypeInterface from questionpy_common.constants import DIST_DIR from questionpy_sdk._package import build_qpy_package -from questionpy_sdk._package._helper import create_normalized_filename from questionpy_sdk._package.source import PackageSource from questionpy_sdk.webserver.server import WebServer from questionpy_server.hash import calculate_hash @@ -37,10 +36,7 @@ def mock_state_manager(monkeypatch: pytest.MonkeyPatch) -> Iterator[Mock]: async def test_webserver_startup( - mock_worker_pool: tuple[Mock, MagicMock], - mock_worker: AsyncMock, - mock_web_components: tuple[Mock, AsyncMock], - mock_state_manager: Mock, + mock_worker_pool: tuple[Mock, MagicMock], mock_web_components: tuple[Mock, AsyncMock], mock_state_manager: Mock ) -> None: mock_worker_pool_cls, mock_worker_pool_instance = mock_worker_pool mock_app_runner, mock_tcp_site = mock_web_components @@ -50,13 +46,8 @@ async def test_webserver_startup( async with WebServer(package_location=package_location, state_storage_path=state_storage_path): mock_worker_pool_cls.assert_called_once() - mock_worker_pool_instance.get_worker.assert_not_called() - - file_name = create_normalized_filename(package_location.manifest) - expected_path = state_storage_path / file_name - mock_state_manager.assert_called_once_with(expected_path.with_suffix("")) - + mock_state_manager.assert_called_once_with(state_storage_path, "test-init-0.1.0-debug") mock_app_runner.setup.assert_awaited_once() mock_tcp_site.return_value.start.assert_awaited_once() From b7a33b8c569016e7c2d36d1a1177868a2f1cc7a0 Mon Sep 17 00:00:00 2001 From: Mirko Dietrich Date: Thu, 6 Nov 2025 15:22:43 +0100 Subject: [PATCH 2/4] feat(backend): add file upload and retrieval functionality for question options Ref: #214 --- questionpy_sdk/webserver/constants.py | 1 + questionpy_sdk/webserver/controllers/file.py | 1 + .../webserver/controllers/question.py | 40 ++++- questionpy_sdk/webserver/routes/question.py | 70 ++++++++- .../webserver/controllers/test_question.py | 29 ++++ .../webserver/routes/test_question.py | 144 +++++++++++++++++- 6 files changed, 280 insertions(+), 5 deletions(-) diff --git a/questionpy_sdk/webserver/constants.py b/questionpy_sdk/webserver/constants.py index 11008b87..21036560 100644 --- a/questionpy_sdk/webserver/constants.py +++ b/questionpy_sdk/webserver/constants.py @@ -16,6 +16,7 @@ API_PATH_PREFIX = "/api" ID_RE = r"[A-Za-z0-9_-]{8}" +FILE_REF_RE = r"[a-zA-Z0-9\-_=]{1,64}" WEBSERVER_KEY: web.AppKey["WebServer"] = web.AppKey("qpy_webserver") REQUEST_CONTROLLER_KEY = "qpy_controller" diff --git a/questionpy_sdk/webserver/controllers/file.py b/questionpy_sdk/webserver/controllers/file.py index 732815ed..ea64bff1 100644 --- a/questionpy_sdk/webserver/controllers/file.py +++ b/questionpy_sdk/webserver/controllers/file.py @@ -1,6 +1,7 @@ # This file is part of the QuestionPy SDK. (https://questionpy.org) # The QuestionPy SDK is free software released under terms of the MIT license. See LICENSE.md. # (c) Technische Universität Berlin, innoCampus + from questionpy_sdk.webserver.controllers.base import BaseController from questionpy_server.worker import PackageFileData diff --git a/questionpy_sdk/webserver/controllers/question.py b/questionpy_sdk/webserver/controllers/question.py index 2c9b44bb..a0d71951 100644 --- a/questionpy_sdk/webserver/controllers/question.py +++ b/questionpy_sdk/webserver/controllers/question.py @@ -2,8 +2,13 @@ # The QuestionPy SDK is free software released under terms of the MIT license. See LICENSE.md. # (c) Technische Universität Berlin, innoCampus -from collections.abc import Mapping -from typing import cast +import hashlib +import tempfile +from collections.abc import AsyncIterable, Mapping +from contextlib import AbstractContextManager +from datetime import UTC, datetime +from pathlib import Path +from typing import IO, BinaryIO, cast from pydantic import ConfigDict from pydantic.dataclasses import dataclass @@ -110,3 +115,34 @@ async def clone_question(self, question_id: str, new_question_id: str) -> None: state = await self._state_manager.read_question_state(question_id) await self._state_manager.write_question_state(new_question_id, state) + + async def add_file(self, filename: str, mime_type: str, reader: AsyncIterable[bytes]) -> OptionsFile: + # Temp file needed to hash content before saving; reader is single-use. + # Likely fails on Windows (moving open files) + with tempfile.NamedTemporaryFile() as tmp_file: + file_ref, size = await self._process_file(reader, tmp_file) + await self._state_manager.add_options_file(file_ref, Path(tmp_file.name)) + + return OptionsFile( + path="/", + filename=filename, + file_ref=file_ref, + uploaded_at=datetime.now(UTC), + mime_type=mime_type, + size=size, + ) + + async def get_file( + self, question_id: str, name: str, file_ref: str + ) -> tuple[OptionsFile, AbstractContextManager[BinaryIO]]: + return await self._state_manager.get_options_file(question_id, name, file_ref) + + async def _process_file(self, reader: AsyncIterable[bytes], file: IO[bytes]) -> tuple[str, int]: + sha1_hash = hashlib.sha1() # noqa: S324 + total_size = 0 + async for chunk in reader: + sha1_hash.update(chunk) + total_size += len(chunk) + file.write(chunk) + file.flush() + return sha1_hash.hexdigest(), total_size diff --git a/questionpy_sdk/webserver/routes/question.py b/questionpy_sdk/webserver/routes/question.py index d1a76e42..f0f37586 100644 --- a/questionpy_sdk/webserver/routes/question.py +++ b/questionpy_sdk/webserver/routes/question.py @@ -2,16 +2,21 @@ # The QuestionPy SDK is free software released under terms of the MIT license. See LICENSE.md. # (c) Technische Universität Berlin, innoCampus -from aiohttp import web +from typing import TYPE_CHECKING + +from aiohttp import BodyPartReader, web from aiohttp.web_exceptions import HTTPNotFound, HTTPUnprocessableEntity from pydantic import RootModel from questionpy import OptionsFormValidationError -from questionpy_sdk.webserver.constants import ID_RE +from questionpy_sdk.webserver.constants import FILE_REF_RE, ID_RE from questionpy_sdk.webserver.controllers.question import QuestionController from questionpy_sdk.webserver.errors import DuplicateQuestionError, MissingQuestionStateError from questionpy_sdk.webserver.routes.base import BaseView +if TYPE_CHECKING: + from questionpy.form import OptionsFile + routes = web.RouteTableDef() @@ -87,3 +92,64 @@ async def post(self) -> web.Response: raise web.HTTPConflict(text=str(err)) from err return self.json_success_response() + + +@routes.view(f"/question/{{question_id:{ID_RE}}}/file/{{name}}/{{file_ref:{FILE_REF_RE}}}", name="question.file") +class QuestionFileView(QuestionBaseView): + async def get(self) -> web.StreamResponse: + """Retrieves an option file.""" + question_id = self.request.match_info["question_id"] + name = self.request.match_info["name"] + file_ref = self.request.match_info["file_ref"] + + options_file, file_reader = await self.controller.get_file(question_id, name, file_ref) + + headers = { + # Content-addressable -> never changes + "Cache-Control": "public, max-age=31536000, immutable", + "Content-Disposition": f'inline; filename="{options_file.filename}"', + "Content-Length": str(options_file.size), + "Content-Type": options_file.mime_type, + "Last-Modified": options_file.uploaded_at.strftime("%a, %d %b %Y %H:%M:%S GMT"), + } + + resp = web.StreamResponse(headers=headers) + await resp.prepare(self.request) + + # stream in chunks + chunk_size = 64 * 1024 # 64 KiB + with file_reader as f: + while chunk := f.read(chunk_size): + await resp.write(chunk) + await resp.write_eof() + + return resp + + +@routes.view("/question/file-upload", name="question.file-upload") +class QuestionFileUploadView(QuestionBaseView): + async def post(self) -> web.Response: + """Processes and stores option files.""" + files: list[OptionsFile] = [] + + async for part in await self.request.multipart(): + if isinstance(part, BodyPartReader): + if part.name != "file": + return web.json_response( + {"error": "Expected part name to be 'file'"}, status=HTTPUnprocessableEntity.status_code + ) + + if not part.filename: + return web.json_response({"error": "Missing filename"}, status=HTTPUnprocessableEntity.status_code) + + file = await self.controller.add_file( + part.filename, + part.headers.get("Content-Type", "application/octet-stream"), + part, + ) + files.append(file) + + if len(files) == 0: + return web.json_response({"error": "No files in form data"}, status=HTTPUnprocessableEntity.status_code) + + return self.json_model_response(RootModel(files)) diff --git a/tests/questionpy_sdk/webserver/controllers/test_question.py b/tests/questionpy_sdk/webserver/controllers/test_question.py index 7bad0831..9b8ab48b 100644 --- a/tests/questionpy_sdk/webserver/controllers/test_question.py +++ b/tests/questionpy_sdk/webserver/controllers/test_question.py @@ -2,6 +2,8 @@ # The QuestionPy SDK is free software released under terms of the MIT license. See LICENSE.md. # (c) Technische Universität Berlin, innoCampus +from collections.abc import AsyncIterable +from datetime import datetime from unittest.mock import AsyncMock, Mock import pytest @@ -101,3 +103,30 @@ async def test_clone_question( async def test_clone_question_duplicate(controller: QuestionController, mock_state_manager: AsyncMock) -> None: with pytest.raises(DuplicateQuestionError): await controller.clone_question("QaKxpanc", "tKVJTdsv") + + +async def test_add_file(controller: QuestionController, mock_state_manager: AsyncMock) -> None: + async def mock_reader() -> AsyncIterable[bytes]: # noqa: RUF029 + yield b"some test content" + + options_file = await controller.add_file("test_file.txt", "text/plain", mock_reader()) + + mock_state_manager.add_options_file.assert_called_once() + assert options_file.filename == "test_file.txt" + assert options_file.mime_type == "text/plain" + assert options_file.file_ref == "dbabd43828eccd27e3a109b58454e4ff43c8673e" + assert options_file.size == 17 + assert options_file.path == "/" + assert isinstance(options_file.uploaded_at, datetime) + + +async def test_get_file(controller: QuestionController, mock_state_manager: AsyncMock) -> None: + expected_options_file = Mock() + expected_file_manager = Mock() + mock_state_manager.get_options_file.return_value = (expected_options_file, expected_file_manager) + + options_file, file_manager = await controller.get_file("QaKxpanc", "my_file_upload", "abcdef0123456") + + mock_state_manager.get_options_file.assert_called_once_with("QaKxpanc", "my_file_upload", "abcdef0123456") + assert options_file == expected_options_file + assert file_manager == expected_file_manager diff --git a/tests/questionpy_sdk/webserver/routes/test_question.py b/tests/questionpy_sdk/webserver/routes/test_question.py index 884cb8fc..c33abbf1 100644 --- a/tests/questionpy_sdk/webserver/routes/test_question.py +++ b/tests/questionpy_sdk/webserver/routes/test_question.py @@ -2,18 +2,34 @@ # The QuestionPy SDK is free software released under terms of the MIT license. See LICENSE.md. # (c) Technische Universität Berlin, innoCampus -from unittest.mock import AsyncMock +from datetime import UTC, datetime +from io import BytesIO +from unittest.mock import ANY, AsyncMock, Mock import pytest +from aiohttp import FormData from aiohttp.test_utils import TestClient from aiohttp.web_exceptions import HTTPConflict, HTTPNotFound, HTTPOk, HTTPUnprocessableEntity from questionpy import OptionsFormValidationError +from questionpy.form import OptionsFile from questionpy_common.elements import OptionsFormDefinition from questionpy_sdk.webserver.errors import DuplicateQuestionError, MissingQuestionStateError from questionpy_sdk.webserver.routes.question import routes +@pytest.fixture +def options_file() -> OptionsFile: + return OptionsFile( + path="/", + filename="test_file.txt", + file_ref="dbabd43828eccd27e3a109b58454e4ff43c8673e", + size=17, + uploaded_at=datetime(2025, 1, 1, 12, 0, 0, tzinfo=UTC), + mime_type="text/plain", + ) + + @pytest.mark.app_routes(routes) async def test_get_question(client: TestClient, mock_controller: AsyncMock) -> None: mock_controller.get_form_definition.return_value = OptionsFormDefinition() @@ -96,3 +112,129 @@ async def test_post_question_clone_duplicate(client: TestClient, mock_controller async with client.post("/question/myuQ2JWl/clone/Bu2boh5u") as resp: assert resp.status == HTTPConflict.status_code + + +@pytest.mark.app_routes(routes) +async def test_get_question_file(client: TestClient, mock_controller: AsyncMock, options_file: OptionsFile) -> None: + mock_file_reader = Mock() + mock_file_reader.__enter__ = Mock(return_value=mock_file_reader) + mock_file_reader.__exit__ = Mock(return_value=None) + mock_file_reader.read = Mock(side_effect=[b"some test content"]) + mock_controller.get_file.return_value = (options_file, mock_file_reader) + + async with client.get("/question/myuQ2JWl/file/my_file_upload/dbabd43828eccd27e3a109b58454e4ff43c8673e") as resp: + mock_controller.get_file.assert_awaited_once_with( + "myuQ2JWl", "my_file_upload", "dbabd43828eccd27e3a109b58454e4ff43c8673e" + ) + + assert resp.status == HTTPOk.status_code + assert resp.headers["Content-Type"] == "text/plain" + assert resp.headers["Last-Modified"] == "Wed, 01 Jan 2025 12:00:00 GMT" + assert resp.headers["Content-Length"] == "17" + assert "immutable" in resp.headers["Cache-Control"] + assert "test_file.txt" in resp.headers["Content-Disposition"] + assert await resp.text() == "some test content" + + +@pytest.mark.app_routes(routes) +async def test_post_question_file_upload_single_file( + client: TestClient, mock_controller: AsyncMock, options_file: OptionsFile +) -> None: + mock_controller.add_file.return_value = options_file + + data = FormData() + data.add_field("file", b"some test content", filename="test.txt", content_type="text/plain") + + async with client.post("/question/file-upload", data=data) as resp: + assert resp.status == HTTPOk.status_code + response_data = await resp.json() + assert len(response_data) == 1 + of = response_data[0] + assert of["filename"] == "test_file.txt" + assert of["file_ref"] == "dbabd43828eccd27e3a109b58454e4ff43c8673e" + assert of["size"] == 17 + assert of["mime_type"] == "text/plain" + assert of["uploaded_at"] == "2025-01-01T12:00:00Z" + assert of["path"] == "/" + mock_controller.add_file.assert_awaited_once_with("test.txt", "text/plain", ANY) + + +@pytest.mark.app_routes(routes) +async def test_post_question_file_upload_multiple_files(client: TestClient, mock_controller: AsyncMock) -> None: + mock_controller.add_file.side_effect = [ + OptionsFile( + path="/", + filename="test1.jpg", + file_ref="041bb7ae61ffe07323f642b8c72ba96c72c5d2b0", + size=11, + uploaded_at=datetime(2025, 1, 1, 12, 0, 0, tzinfo=UTC), + mime_type="image/jpeg", + ), + OptionsFile( + path="/", + filename="test2.jpg", + file_ref="c84ce717f09efd2583ca97d3feb69a333e342b5a", + size=11, + uploaded_at=datetime(2025, 1, 1, 12, 0, 1, tzinfo=UTC), + mime_type="image/jpeg", + ), + ] + + data = FormData() + data.add_field("file", b"jpgcontent1", filename="test1.jpg", content_type="image/jpeg") + data.add_field("file", b"jpgcontent2", filename="test2.jpg", content_type="image/jpeg") + + async with client.post("/question/file-upload", data=data) as resp: + assert resp.status == HTTPOk.status_code + response_data = await resp.json() + assert len(response_data) == 2 + of1, of2 = response_data + + assert of1["filename"] == "test1.jpg" + assert of1["file_ref"] == "041bb7ae61ffe07323f642b8c72ba96c72c5d2b0" + assert of1["size"] == 11 + assert of1["mime_type"] == "image/jpeg" + assert of1["uploaded_at"] == "2025-01-01T12:00:00Z" + assert of1["path"] == "/" + + assert of2["filename"] == "test2.jpg" + assert of2["file_ref"] == "c84ce717f09efd2583ca97d3feb69a333e342b5a" + assert of2["size"] == 11 + assert of2["mime_type"] == "image/jpeg" + assert of2["uploaded_at"] == "2025-01-01T12:00:01Z" + assert of2["path"] == "/" + + assert mock_controller.add_file.call_count == 2 + + +@pytest.mark.app_routes(routes) +async def test_post_question_file_upload_no_file(client: TestClient, mock_controller: AsyncMock) -> None: + async with client.post("/question/file-upload", data=FormData(default_to_multipart=True)) as resp: + assert resp.status == HTTPUnprocessableEntity.status_code + data = await resp.json() + assert data["error"] == "No files in form data" + mock_controller.add_file.assert_not_called() + + +@pytest.mark.app_routes(routes) +async def test_post_question_file_upload_wrong_part_name(client: TestClient, mock_controller: AsyncMock) -> None: + data = FormData() + data.add_field("wrongname", b"content", filename="test.txt", content_type="text/plain") + + async with client.post("/question/file-upload", data=data) as resp: + assert resp.status == HTTPUnprocessableEntity.status_code + response_data = await resp.json() + assert response_data["error"] == "Expected part name to be 'file'" + mock_controller.add_file.assert_not_called() + + +@pytest.mark.app_routes(routes) +async def test_post_question_file_upload_missing_filename(client: TestClient, mock_controller: AsyncMock) -> None: + data = FormData() + data.add_field("file", BytesIO(b"content"), filename="", content_type="text/plain") + + async with client.post("/question/file-upload", data=data) as resp: + assert resp.status == HTTPUnprocessableEntity.status_code + response_data = await resp.json() + assert response_data["error"] == "Missing filename" + mock_controller.add_file.assert_not_called() From 8546839d355f5c4cf17581357f24a9d9c17c1b5c Mon Sep 17 00:00:00 2001 From: Mirko Dietrich Date: Thu, 6 Nov 2025 15:23:08 +0100 Subject: [PATCH 3/4] feat(frontend): add `` Ref: #214 --- frontend/src/components/ErrorModal.vue | 9 +- .../src/components/common/ActionButton.vue | 37 ++++++ .../src/components/common/UploadThumbnail.vue | 109 ++++++++++++++++++ .../components/question/QuestionOptions.vue | 4 +- .../question/elements/FileUploadElement.vue | 78 +++++++++++++ .../src/components/question/elements/index.ts | 2 + .../question/elements/useIsDisabled.ts | 1 + .../src/composables/question/formDataUtils.ts | 3 + frontend/src/composables/question/index.ts | 1 + .../src/composables/question/useFileUpload.ts | 89 ++++++++++++++ .../composables/question/useFormDataState.ts | 30 ++++- .../composables/question/useRepetitions.ts | 3 +- frontend/src/queries/fetch.ts | 83 ++++++++++--- frontend/src/queries/index.ts | 3 +- frontend/src/queries/question.ts | 20 ++++ .../src/stores/usePendingOperationsStore.ts | 13 ++- frontend/src/types/index.ts | 4 + 17 files changed, 457 insertions(+), 32 deletions(-) create mode 100644 frontend/src/components/common/ActionButton.vue create mode 100644 frontend/src/components/common/UploadThumbnail.vue create mode 100644 frontend/src/components/question/elements/FileUploadElement.vue create mode 100644 frontend/src/composables/question/useFileUpload.ts diff --git a/frontend/src/components/ErrorModal.vue b/frontend/src/components/ErrorModal.vue index c0bb4276..4ed93fbe 100644 --- a/frontend/src/components/ErrorModal.vue +++ b/frontend/src/components/ErrorModal.vue @@ -5,14 +5,7 @@ -->