diff --git a/docs/middleware.md b/docs/middleware.md new file mode 100644 index 0000000..491c691 --- /dev/null +++ b/docs/middleware.md @@ -0,0 +1,34 @@ +# Middleware Management API + +## Overview + +The Middleware Management API provides REST endpoints for dynamically managing middleware components in proTES at runtime. This API allows administrators and developers to add, configure, update, and remove middleware without service restarts. + +## Background + +proTES uses a middleware architecture to process task execution requests. Previously, middleware configuration was static and required service restarts for any changes. The Middleware Management API enables dynamic runtime configuration, making it easier to adapt the service to changing requirements and deploy new middleware components. + +## API Specification + +The Middleware Management API is defined using OpenAPI 3.0 specification. For comprehensive, interactive documentation with the ability to explore endpoints, request/response schemas, and examples, please visit: + +**[Swagger Editor - Middleware Management API](https://editor.swagger.io/?url=https://raw.githubusercontent.com/elixir-cloud-aai/proTES/refs/heads/dev/pro_tes/api/middleware_management.yaml)** + +The interactive documentation provides: +- Complete endpoint definitions with request/response examples +- Detailed schema specifications for all data models +- Parameter descriptions and validation rules +- Error response definitions +- The ability to test API calls directly + +## File Structure + +``` +pro_tes/ +├── api/ +│ └── middleware_management.yaml (OpenAPI specification) +└── config.yaml (References the specification) + +docs/ +└── middleware.md (This documentation) +``` diff --git a/pro_tes/api/middleware_management.yaml b/pro_tes/api/middleware_management.yaml new file mode 100644 index 0000000..9b73e40 --- /dev/null +++ b/pro_tes/api/middleware_management.yaml @@ -0,0 +1,691 @@ +openapi: 3.0.3 +info: + title: proTES Middleware Management API + description: | + API for dynamically managing middleware in proTES (GA4GH Task Execution Service Proxy). + This API allows runtime configuration of middleware components that process task execution requests. + version: 1.0.0 + contact: + name: ELIXIR Cloud & AAI + url: https://github.com/elixir-cloud-aai/proTES + license: + name: Apache 2.0 + url: https://www.apache.org/licenses/LICENSE-2.0.html + +servers: + - url: /protes/v1 + description: proTES Middleware Management API base path + +tags: + - name: Middleware Management + description: Operations for managing middleware stack + +paths: + /middlewares: + get: + summary: List all middlewares + description: | + Retrieve all configured middlewares with their order, metadata, and status. + Results are sorted by execution order (ascending) by default. + operationId: ListMiddlewares + tags: + - Middleware Management + parameters: + - name: page_size + in: query + description: Maximum number of results to return per page + required: false + schema: + type: integer + minimum: 1 + maximum: 100 + default: 50 + - name: page + in: query + description: Page number to retrieve (0-indexed) + required: false + schema: + type: integer + minimum: 0 + default: 0 + - name: sort_by + in: query + description: Field to sort by + required: false + schema: + type: string + enum: [order, name, created_at, updated_at] + default: order + - name: source + in: query + description: Filter by middleware source type + required: false + schema: + type: string + enum: [local, github, pypi] + responses: + '200': + description: Successful response with list of middlewares + content: + application/json: + schema: + $ref: '#/components/schemas/MiddlewareList' + '400': + description: Bad request (invalid parameters) + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + '401': + description: Unauthorized (when authentication is configured) + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + '403': + description: Forbidden (when authentication is configured and permissions are insufficient) + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + '500': + description: Internal server error + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + + post: + summary: Add a new middleware + description: | + Add a new middleware to the execution stack. Middleware can be loaded from: + - GitHub repositories: Git repositories containing setup.py or pyproject.toml (recommended) + - PyPI packages: Packages from PyPI or other package registries (recommended) + - Local packages: Installed Python packages (**deprecated** - for development only) + + If order is not specified, defaults to 0. If a middleware already exists at that + position, existing middlewares at that position or higher are shifted up by one. + + If name is not provided, it will be derived from the package or repository name. + + Fallback groups can be created by providing an array of source configurations. + If the first middleware in the group fails, the system tries the next one in the + fallback group, allowing mixed sources (GitHub, PyPI, local) in a single entry. + operationId: AddMiddleware + tags: + - Middleware Management + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/MiddlewareCreate' + examples: + github_middleware: + summary: Add middleware from GitHub + value: + name: "Custom Load Balancer" + source: + type: "github" + repository: "https://github.com/user/repo.git" + entry_point: "custom_middleware.LoadBalancer" + order: 0 + enabled: true + pypi_middleware: + summary: Add middleware from PyPI + value: + name: "Third-party Middleware" + source: + type: "pypi" + package: "protes-middleware-custom" + entry_point: "custom.Middleware" + version: "1.0.0" + enabled: true + local_middleware: + summary: Add local middleware (deprecated - development only) + value: + name: "Distance-based Router" + source: + type: "local" + entry_point: "pro_tes.plugins.middlewares.task_distribution.distance.TaskDistributionDistance" + order: 0 + enabled: true + fallback_group: + summary: Add fallback group with mixed sources + value: + name: "Load Balancing Group" + source: + - type: "github" + repository: "https://github.com/org/primary.git" + entry_point: "primary.DistanceRouter" + - type: "github" + repository: "https://github.com/org/fallback.git" + entry_point: "fallback.RandomRouter" + - type: "pypi" + package: "protes-fallback" + entry_point: "fallback.LastResort" + order: 0 + enabled: true + responses: + '201': + description: Middleware created successfully + content: + application/json: + schema: + $ref: '#/components/schemas/MiddlewareCreateResponse' + '400': + description: Invalid request (duplicate name/entry_point, invalid source, validation failed) + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + '401': + description: Unauthorized (when authentication is configured) + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + '403': + description: Forbidden (when authentication is configured and permissions are insufficient) + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + '500': + description: Internal server error + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + + /middlewares/{middleware_id}: + get: + summary: Get middleware details + description: Retrieve detailed information about a specific middleware by ID + operationId: GetMiddleware + tags: + - Middleware Management + parameters: + - name: middleware_id + in: path + description: Unique identifier of the middleware + required: true + schema: + type: string + pattern: '^[a-f0-9]{24}$' + responses: + '200': + description: Successful response with middleware details + content: + application/json: + schema: + $ref: '#/components/schemas/MiddlewareConfig' + '400': + description: Bad request (invalid middleware ID format) + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + '401': + description: Unauthorized (when authentication is configured) + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + '403': + description: Forbidden (when authentication is configured and permissions are insufficient) + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + '404': + description: Middleware not found + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + '500': + description: Internal server error + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + + put: + summary: Update middleware configuration + description: | + Update middleware configuration. Only name, order, config, and enabled fields + can be updated. Source configuration (package type, repository, entry point) + cannot be modified for security reasons. + operationId: UpdateMiddleware + tags: + - Middleware Management + parameters: + - name: middleware_id + in: path + description: Unique identifier of the middleware + required: true + schema: + type: string + pattern: '^[a-f0-9]{24}$' + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/MiddlewareUpdate' + responses: + '200': + description: Middleware updated successfully + content: + application/json: + schema: + $ref: '#/components/schemas/MiddlewareConfig' + '400': + description: Invalid request (invalid middleware ID format, invalid parameters) + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + '401': + description: Unauthorized (when authentication is configured) + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + '403': + description: Forbidden (when authentication is configured and permissions are insufficient) + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + '404': + description: Middleware not found + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + '409': + description: Conflict (e.g., duplicate name) + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + '500': + description: Internal server error + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + + delete: + summary: Remove a middleware + description: | + Permanently remove a middleware from the execution stack and database. + To temporarily disable a middleware, use the UPDATE endpoint to set enabled=false. + operationId: DeleteMiddleware + tags: + - Middleware Management + parameters: + - name: middleware_id + in: path + description: Unique identifier of the middleware + required: true + schema: + type: string + pattern: '^[a-f0-9]{24}$' + responses: + '204': + description: Middleware deleted successfully + '400': + description: Bad request (invalid middleware ID format) + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + '401': + description: Unauthorized (when authentication is configured) + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + '403': + description: Forbidden (when authentication is configured and permissions are insufficient) + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + '404': + description: Middleware not found + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + '500': + description: Internal server error + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + + /middlewares/reorder: + put: + summary: Reorder middleware stack + description: | + Reorder the entire middleware execution stack by providing an ordered array + of middleware IDs. All middleware IDs must be provided in the desired execution order. + operationId: ReorderMiddlewares + tags: + - Middleware Management + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/MiddlewareOrder' + responses: + '200': + description: Middlewares reordered successfully + content: + application/json: + schema: + type: object + properties: + message: + type: string + example: "Middleware stack reordered successfully" + middlewares: + type: array + items: + $ref: '#/components/schemas/MiddlewareConfig' + '400': + description: Invalid request (missing IDs, invalid IDs, duplicate IDs) + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + '401': + description: Unauthorized (when authentication is configured) + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + '403': + description: Forbidden (when authentication is configured and permissions are insufficient) + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + '500': + description: Internal server error + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + +components: + schemas: + MiddlewareConfig: + type: object + description: Complete middleware configuration object + required: + - _id + - source + - order + - enabled + - created_at + - updated_at + properties: + _id: + type: string + description: Unique identifier (MongoDB ObjectId) + readOnly: true + example: "507f1f77bcf86cd799439011" + name: + type: string + description: Human-readable name for the middleware. If not provided, derived from package or repository name. + nullable: true + example: "Distance-based Router" + source: + oneOf: + - $ref: '#/components/schemas/MiddlewareSource' + - type: array + description: Fallback group (array of middleware sources) + items: + $ref: '#/components/schemas/MiddlewareSource' + minItems: 2 + example: + - type: "github" + repository: "https://github.com/org/primary.git" + entry_point: "primary.DistanceRouter" + - type: "github" + repository: "https://github.com/org/fallback.git" + entry_point: "fallback.RandomRouter" + order: + type: integer + description: Execution order (0 = first) + minimum: 0 + default: 0 + example: 0 + config: + type: object + description: Middleware-specific configuration + nullable: true + additionalProperties: true + example: + timeout: 30 + retries: 3 + enabled: + type: boolean + description: Whether the middleware is active + example: true + created_at: + type: string + format: date-time + description: Creation timestamp (set by system) + readOnly: true + example: "2026-01-24T10:30:00Z" + updated_at: + type: string + format: date-time + description: Last update timestamp (set by system) + readOnly: true + example: "2026-01-24T10:30:00Z" + + MiddlewareSource: + type: object + description: Middleware package source configuration + required: + - type + - entry_point + properties: + type: + type: string + description: Source type for the middleware package + enum: [local, github, pypi] + entry_point: + type: string + description: Class path entry point (e.g., 'package.module.ClassName') + example: "pro_tes.plugins.middlewares.task_distribution.distance.TaskDistributionDistance" + package: + type: string + description: Package name (required for pypi type) + example: "protes-middleware-custom" + repository: + type: string + description: Git repository URL (required for github type) + pattern: '^https://github\.com/.+\.git$' + example: "https://github.com/user/repo.git" + version: + type: string + description: Package version (optional, for pypi or github tag/branch) + example: "1.0.0" + + MiddlewareCreate: + type: object + description: Request body for creating a middleware + required: + - source + properties: + name: + type: string + description: Human-readable name for the middleware. If not provided, will be derived from package or repository name. + minLength: 1 + maxLength: 255 + nullable: true + example: "Distance-based Router" + source: + oneOf: + - $ref: '#/components/schemas/MiddlewareSource' + - type: array + description: Fallback group (array of middleware sources) + items: + $ref: '#/components/schemas/MiddlewareSource' + minItems: 2 + example: + - type: "github" + repository: "https://github.com/org/primary.git" + entry_point: "primary.DistanceRouter" + - type: "github" + repository: "https://github.com/org/fallback.git" + entry_point: "fallback.RandomRouter" + order: + type: integer + description: Execution order (omit to append to end) + minimum: 0 + default: 0 + nullable: true + example: 0 + config: + type: object + description: Middleware-specific configuration + nullable: true + additionalProperties: true + example: + timeout: 30 + retries: 3 + enabled: + type: boolean + description: Whether the middleware should be active + default: true + example: true + + MiddlewareUpdate: + type: object + description: Request body for updating a middleware + properties: + name: + type: string + description: Human-readable name for the middleware + minLength: 1 + maxLength: 255 + example: "Distance-based Router v2" + order: + type: integer + description: Execution order + minimum: 0 + example: 1 + config: + type: object + description: Middleware-specific configuration + nullable: true + additionalProperties: true + example: + timeout: 60 + retries: 5 + enabled: + type: boolean + description: Whether the middleware is active + example: false + + MiddlewareList: + type: object + description: Paginated list of middlewares + required: + - middlewares + - pagination + properties: + middlewares: + type: array + description: Array of middleware configurations + items: + $ref: '#/components/schemas/MiddlewareConfig' + pagination: + type: object + description: Pagination information following GA4GH guidelines + required: + - page + - page_size + - total + properties: + page: + type: integer + description: Current page number (0-indexed) + example: 0 + page_size: + type: integer + description: Number of results per page + example: 50 + total: + type: integer + description: Total number of middlewares available + example: 5 + total_pages: + type: integer + description: Total number of pages available + example: 1 + + MiddlewareCreateResponse: + type: object + description: Response after creating a middleware + required: + - _id + - order + - message + properties: + _id: + type: string + description: Unique identifier of created middleware + example: "507f1f77bcf86cd799439011" + order: + type: integer + description: Assigned execution order + example: 0 + message: + type: string + description: Success message + example: "Middleware added successfully" + + MiddlewareOrder: + type: object + description: Request body for reordering middlewares + required: + - ordered_ids + properties: + ordered_ids: + type: array + description: Array of middleware IDs in desired execution order + items: + type: string + pattern: '^[a-f0-9]{24}$' + minItems: 1 + example: + - "507f1f77bcf86cd799439011" + - "507f1f77bcf86cd799439012" + - "507f1f77bcf86cd799439013" + + ErrorResponse: + type: object + description: Standard error response + required: + - code + - message + properties: + code: + type: integer + description: HTTP status code + example: 404 + message: + type: string + description: Human-readable error message + example: "Middleware with ID '507f1f77bcf86cd799439011' not found" diff --git a/pro_tes/api/middlewares/__init__.py b/pro_tes/api/middlewares/__init__.py new file mode 100644 index 0000000..92cf2d9 --- /dev/null +++ b/pro_tes/api/middlewares/__init__.py @@ -0,0 +1 @@ +"""Middleware Management API controllers.""" diff --git a/pro_tes/api/middlewares/controllers.py b/pro_tes/api/middlewares/controllers.py new file mode 100644 index 0000000..c74eef0 --- /dev/null +++ b/pro_tes/api/middlewares/controllers.py @@ -0,0 +1,476 @@ +"""Controllers for middleware management API. + +This module implements all REST API endpoints for managing middlewares +dynamically at runtime. All implementations match the finalized OpenAPI +specification from PR #1 (middleware-api-spec branch). +""" + +import logging +import math +from datetime import datetime +from typing import Optional, Any + +from bson import ObjectId +from flask import current_app, request +from pymongo.errors import PyMongoError +from werkzeug.exceptions import InternalServerError + +from pro_tes.exceptions import ( + BadRequest, + MiddlewareNotFound, + MiddlewareDuplicateName, +) +from pro_tes.api.middlewares.models import ( + MiddlewareCreate, + MiddlewareUpdate, +) + +logger = logging.getLogger(__name__) + + +def get_middleware_collection(): + """Get middleware collection from database.""" + return current_app.config.foca.db.dbs["taskStore"].collections[ + "middlewares" + ].client + + +def _extract_entry_points(source): + """Extract all entry points from a source (single or fallback group). + + Args: + source: Single MiddlewareSource dict/object or list of + MiddlewareSource dicts/objects + + Returns: + List of entry point strings + """ + if isinstance(source, list): + return [ + (s.entry_point if hasattr(s, 'entry_point') + else s.get("entry_point")) + for s in source + if ((hasattr(s, 'entry_point') and s.entry_point) or + (hasattr(s, 'get') and s.get("entry_point"))) + ] + # Handle both Pydantic objects and dicts + if hasattr(source, 'entry_point'): + return [source.entry_point] if source.entry_point else [] + return [source.get("entry_point")] if source.get("entry_point") else [] + + +def _derive_name_from_source(source): + """Derive middleware name from source configuration. + + Args: + source: Single MiddlewareSource dict/object or list of + MiddlewareSource dicts/objects + + Returns: + Derived name string + """ + if isinstance(source, list): + # For fallback groups, use first source + source = source[0] + + # Handle both Pydantic objects and dicts + # Try to get package name + package = (getattr(source, 'package', None) + if hasattr(source, 'package') + else (source.get('package') + if hasattr(source, 'get') else None)) + if package: + return package + + # Try to get repository name + repository = (getattr(source, 'repository', None) + if hasattr(source, 'repository') + else (source.get('repository') + if hasattr(source, 'get') else None)) + if repository: + # Extract repo name from URL + repo_name = repository.rstrip("/").rstrip(".git").split("/")[-1] + return repo_name + + # Try to get entry_point + entry_point = (getattr(source, 'entry_point', None) + if hasattr(source, 'entry_point') + else (source.get('entry_point') + if hasattr(source, 'get') else None)) + if entry_point: + # Use last part of entry_point + return entry_point.split(".")[-1] + + return "Unnamed Middleware" + + +def ListMiddlewares( + page_size: int = 50, + page: int = 0, + sort_by: str = "order", + enabled: Optional[bool] = None, + source: Optional[str] = None, +) -> dict: + """List all middlewares with pagination and filtering. + + Args: + page_size: Maximum number of results to return per page. + page: Page number to retrieve (0-indexed). + sort_by: Field to sort by. + enabled: Filter by enabled status. + source: Filter by source type. + + Returns: + Dictionary with middlewares list and pagination info. + """ + try: + collection = get_middleware_collection() + + filter_dict: dict = {} + if enabled is not None: + filter_dict["enabled"] = enabled + if source is not None: + filter_dict["source.type"] = source + + # Calculate pagination + skip = page * page_size + + cursor = collection.find( + filter_dict + ).sort(sort_by, 1).skip(skip).limit(page_size) + + middlewares = [] + for doc in cursor: + doc["_id"] = str(doc["_id"]) # Convert ObjectId to string + middlewares.append(doc) + + total = collection.count_documents(filter_dict) + total_pages = math.ceil(total / page_size) if total > 0 else 0 + + return { + "middlewares": middlewares, + "pagination": { + "page": page, + "page_size": page_size, + "total": total, + "total_pages": total_pages + } + } + except PyMongoError as e: + logger.error(f"Database error: {e}") + raise InternalServerError("Database operation failed") + + +def AddMiddleware() -> tuple: + """Add a new middleware to the execution stack. + + Returns: + Tuple of response dict and HTTP status code. + """ + try: + collection = get_middleware_collection() + data = request.json + + middleware = MiddlewareCreate(**data) # type: ignore[arg-type] + + # Derive name if not provided + name = middleware.name + if not name: + name = _derive_name_from_source(middleware.source) + + # Check for duplicate name + if name: + existing = collection.find_one({"name": name}) + if existing: + raise MiddlewareDuplicateName( + f"Middleware with name '{name}' already exists" + ) + + # Check for duplicate entry_point + entry_points = _extract_entry_points(middleware.source) + for entry_point in entry_points: + existing_ep = collection.find_one( + {"source.entry_point": entry_point} + ) + if existing_ep: + raise BadRequest( + f"Middleware with entry_point " + f"'{entry_point}' already exists" + ) + + # Handle order assignment + order = middleware.order if middleware.order is not None else 0 + + if order is not None: + # Shift existing middlewares at this position or higher + collection.update_many( + {"order": {"$gte": order}}, + {"$inc": {"order": 1}} + ) + + now = datetime.utcnow().isoformat() + "Z" + + # Convert Pydantic model source to dict for MongoDB storage + source_data: Any = middleware.source + if isinstance(source_data, list): + source_data = [ + s.model_dump() if hasattr(s, 'model_dump') else s + for s in source_data + ] # type: ignore[misc] + elif hasattr(source_data, 'model_dump'): + source_data = source_data.model_dump() + + doc = { + "name": name, + "source": source_data, + "order": order, + "enabled": middleware.enabled, + "config": middleware.config, + "created_at": now, + "updated_at": now + } + + result = collection.insert_one(doc) + middleware_id = str(result.inserted_id) + + logger.info(f"Created middleware: {name} (ID: {middleware_id})") + + return { + "_id": middleware_id, + "order": order, + "message": "Middleware created successfully" + }, 201 + + except (BadRequest, MiddlewareDuplicateName): + raise + except Exception as e: + logger.error(f"Error creating middleware: {e}") + raise InternalServerError("Failed to create middleware") + + +def GetMiddleware(middleware_id: str) -> dict: + """Get middleware details by ID. + + Args: + middleware_id: Middleware identifier. + + Returns: + Middleware configuration dict. + """ + try: + collection = get_middleware_collection() + + if not ObjectId.is_valid(middleware_id): + raise BadRequest("Invalid middleware ID format") + + document = collection.find_one({"_id": ObjectId(middleware_id)}) + + if document is None: + raise MiddlewareNotFound( + f"Middleware with ID '{middleware_id}' not found" + ) + + # Convert ObjectId to string for JSON serialization + document["_id"] = str(document["_id"]) + + return document + + except (BadRequest, MiddlewareNotFound): + raise + except Exception as e: + logger.error(f"Error retrieving middleware: {e}") + raise InternalServerError("Failed to retrieve middleware") + + +def UpdateMiddleware(middleware_id: str) -> dict: + """Update middleware configuration. + + Args: + middleware_id: Middleware identifier. + + Returns: + Updated middleware configuration. + """ + try: + collection = get_middleware_collection() + + if not ObjectId.is_valid(middleware_id): + raise BadRequest("Invalid middleware ID format") + + existing = collection.find_one({"_id": ObjectId(middleware_id)}) + if not existing: + raise MiddlewareNotFound( + f"Middleware with ID '{middleware_id}' not found" + ) + + data = request.json + update_data = MiddlewareUpdate(**data) # type: ignore[arg-type] + + update_dict: dict = {} + + if update_data.name is not None: + if update_data.name != existing.get("name"): + name_exists = collection.find_one({"name": update_data.name}) + if name_exists: + raise MiddlewareDuplicateName( + f"Middleware with name " + f"'{update_data.name}' already exists" + ) + update_dict["name"] = update_data.name # type: ignore[assignment] + + if (update_data.order is not None and + update_data.order != existing["order"]): + old_order = existing["order"] + new_order = update_data.order + + if new_order > old_order: + collection.update_many( + {"order": {"$gt": old_order, "$lte": new_order}}, + {"$inc": {"order": -1}} + ) + else: + collection.update_many( + {"order": {"$gte": new_order, "$lt": old_order}}, + {"$inc": {"order": 1}} + ) + + update_dict["order"] = new_order # type: ignore[assignment] + + if update_data.config is not None: + # type: ignore[assignment] + update_dict["config"] = update_data.config + + if update_data.enabled is not None: + # type: ignore[assignment] + update_dict["enabled"] = update_data.enabled + + update_dict["updated_at"] = datetime.utcnow().isoformat() + "Z" + + collection.update_one( + {"_id": ObjectId(middleware_id)}, + {"$set": update_dict} + ) + + updated_doc = collection.find_one({"_id": ObjectId(middleware_id)}) + if updated_doc: + updated_doc["_id"] = str(updated_doc["_id"]) + + logger.info(f"Updated middleware: {middleware_id}") + + return updated_doc + + except (BadRequest, MiddlewareNotFound): + raise + except Exception as e: + logger.error(f"Error updating middleware: {e}") + raise InternalServerError("Failed to update middleware") + + +def DeleteMiddleware(middleware_id: str) -> tuple: + """Delete middleware (hard delete only - soft delete removed). + + Args: + middleware_id: Middleware identifier. + + Returns: + Empty tuple with status code 204. + """ + try: + collection = get_middleware_collection() + + if not ObjectId.is_valid(middleware_id): + raise BadRequest("Invalid middleware ID format") + + middleware = collection.find_one({"_id": ObjectId(middleware_id)}) + if not middleware: + raise MiddlewareNotFound( + f"Middleware with ID '{middleware_id}' not found" + ) + + deleted_order = middleware["order"] + + # Hard delete (soft delete feature removed per PR #1 review) + collection.delete_one({"_id": ObjectId(middleware_id)}) + + # Shift down middlewares with higher order + collection.update_many( + {"order": {"$gt": deleted_order}}, + {"$inc": {"order": -1}} + ) + + logger.info(f"Deleted middleware: {middleware_id}") + + return "", 204 + + except (BadRequest, MiddlewareNotFound): + raise + except Exception as e: + logger.error(f"Error deleting middleware: {e}") + raise InternalServerError("Failed to delete middleware") + + +def ReorderMiddlewares() -> dict: + """Reorder the entire middleware stack. + + Returns: + Success message with updated middleware list. + """ + try: + collection = get_middleware_collection() + data = request.json + + # Use correct field name from OpenAPI spec + middleware_ids = data.get("ordered_ids", []) if data else [] + + if not middleware_ids: + raise BadRequest("ordered_ids array is required") + + if len(middleware_ids) != len(set(middleware_ids)): + raise BadRequest("Duplicate middleware IDs in array") + + # Count total middlewares (no soft delete filter needed) + total_count = collection.count_documents({}) + if len(middleware_ids) != total_count: + raise BadRequest( + f"Array must contain all {total_count} middlewares" + ) + + for middleware_id in middleware_ids: + if not ObjectId.is_valid(middleware_id): + raise BadRequest(f"Invalid middleware ID: {middleware_id}") + + exists = collection.find_one({"_id": ObjectId(middleware_id)}) + if not exists: + raise MiddlewareNotFound( + f"Middleware with ID '{middleware_id}' not found" + ) + + now = datetime.utcnow().isoformat() + "Z" + for new_order, middleware_id in enumerate(middleware_ids): + collection.update_one( + {"_id": ObjectId(middleware_id)}, + {"$set": {"order": new_order, "updated_at": now}} + ) + + middlewares = list(collection.find({}).sort("order", 1)) + # Convert ObjectIds to strings + for mw in middlewares: + mw["_id"] = str(mw["_id"]) + + logger.info("Reordered middleware stack") + + return { + "message": "Middleware stack reordered successfully", + "middlewares": middlewares + } + + except (BadRequest, MiddlewareNotFound): + raise + except Exception as e: + logger.error(f"Error reordering middlewares: {e}") + raise InternalServerError("Failed to reorder middlewares") + + +# Note: ValidateMiddleware endpoint removed per PR #1 review comments +# The validation endpoint was removed from the OpenAPI spec and +# should not be implemented diff --git a/pro_tes/api/middlewares/models.py b/pro_tes/api/middlewares/models.py new file mode 100644 index 0000000..bd7bcaf --- /dev/null +++ b/pro_tes/api/middlewares/models.py @@ -0,0 +1,301 @@ +"""Data models for middleware management API. + +This module defines Pydantic models that match the OpenAPI specification +for the Middleware Management API. All models follow the finalized API +design from PR #1 (middleware-api-spec branch). +""" + +from typing import List, Optional, Union, Literal + +from pydantic import BaseModel, Field + + +# ============================================================================ +# Source Configuration Models (Discriminated Union) +# ============================================================================ + +class MiddlewareSourceLocal(BaseModel): + """Local package source configuration (deprecated - development only).""" + + type: Literal["local"] + entry_point: str = Field( + ..., + description=( + "Class path entry point (e.g., 'package.module.ClassName')" + ), + json_schema_extra={"example": ( + "pro_tes.plugins.middlewares.task_distribution.distance." + "TaskDistributionDistance" + )} + ) + + +class MiddlewareSourceGithub(BaseModel): + """GitHub repository source configuration (recommended for production).""" + + type: Literal["github"] + entry_point: str = Field( + ..., + description=( + "Class path entry point (e.g., 'package.module.ClassName')" + ), + json_schema_extra={"example": "custom_middleware.LoadBalancer"} + ) + repository: str = Field( + ..., + description="Git repository URL", + pattern=r'^https://github\.com/.+\.git$', + json_schema_extra={"example": "https://github.com/user/repo.git"} + ) + version: Optional[str] = Field( + None, + description="Git tag or branch name", + json_schema_extra={"example": "v1.0.0"} + ) + + +class MiddlewareSourcePypi(BaseModel): + """PyPI package source configuration (recommended for production).""" + + type: Literal["pypi"] + entry_point: str = Field( + ..., + description=( + "Class path entry point (e.g., 'package.module.ClassName')" + ), + json_schema_extra={"example": "custom.Middleware"} + ) + package: str = Field( + ..., + description="Package name from PyPI", + json_schema_extra={"example": "protes-middleware-custom"} + ) + version: Optional[str] = Field( + None, + description="Package version", + json_schema_extra={"example": "1.0.0"} + ) + + +# Discriminated union of all source types +MiddlewareSource = Union[ + MiddlewareSourceLocal, + MiddlewareSourceGithub, + MiddlewareSourcePypi +] + + +# ============================================================================ +# Request/Response Models +# ============================================================================ + +class MiddlewareCreate(BaseModel): + """Request model for creating middleware.""" + + name: Optional[str] = Field( + None, + min_length=1, + max_length=255, + description=( + "Human-readable name. If not provided, " + "derived from package/repo name." + ), + json_schema_extra={"example": "Distance-based Router"} + ) + source: Union[MiddlewareSource, List[MiddlewareSource]] = Field( + ..., + description="Single source or array of sources for fallback groups" + ) + order: Optional[int] = Field( + 0, + ge=0, + description=( + "Execution order (0 = first). " + "If not provided, defaults to 0." + ), + json_schema_extra={"example": 0} + ) + config: Optional[dict] = Field( + None, + description="Middleware-specific configuration", + json_schema_extra={"example": {"timeout": 30}, "retries": 3} + ) + enabled: bool = Field( + True, + description="Whether the middleware should be active", + json_schema_extra={"example": True} + ) + + +class MiddlewareUpdate(BaseModel): + """Request model for updating middleware.""" + + name: Optional[str] = Field( + None, + min_length=1, + max_length=255, + description="Human-readable name for the middleware", + json_schema_extra={"example": "Distance-based Router v2"} + ) + order: Optional[int] = Field( + None, + ge=0, + description="Execution order", + json_schema_extra={"example": 1} + ) + config: Optional[dict] = Field( + None, + description="Middleware-specific configuration", + json_schema_extra={"example": {"timeout": 60}, "retries": 5} + ) + enabled: Optional[bool] = Field( + None, + description="Whether the middleware is active", + json_schema_extra={"example": False} + ) + + +class MiddlewareConfig(BaseModel): + """Complete middleware configuration (response model).""" + + id: str = Field( + ..., + alias="_id", + description="Unique identifier (MongoDB ObjectId)", + json_schema_extra={"example": "507f1f77bcf86cd799439011"} + ) + name: Optional[str] = Field( + None, + description="Human-readable name for the middleware", + json_schema_extra={"example": "Distance-based Router"} + ) + source: Union[MiddlewareSource, List[MiddlewareSource]] = Field( + ..., + description="Single source or array of sources for fallback groups" + ) + order: int = Field( + ..., + description="Execution order (0 = first)", + json_schema_extra={"example": 0} + ) + config: Optional[dict] = Field( + None, + description="Middleware-specific configuration", + json_schema_extra={"example": {"timeout": 30}, "retries": 3} + ) + enabled: bool = Field( + ..., + description="Whether the middleware is active", + json_schema_extra={"example": True} + ) + created_at: str = Field( + ..., + description="Creation timestamp", + json_schema_extra={"example": "2026-01-24T10:30:00Z"} + ) + updated_at: str = Field( + ..., + description="Last update timestamp", + json_schema_extra={"example": "2026-01-24T10:30:00Z"} + ) + + class Config: + """Pydantic model configuration.""" + + populate_by_name = True + + +class PaginationInfo(BaseModel): + """Pagination information following GA4GH guidelines.""" + + page: int = Field( + ..., + description="Current page number (0-indexed)", + json_schema_extra={"example": 0} + ) + page_size: int = Field( + ..., + description="Number of results per page", + json_schema_extra={"example": 50} + ) + total: int = Field( + ..., + description="Total number of middlewares available", + json_schema_extra={"example": 5} + ) + total_pages: int = Field( + ..., + description="Total number of pages available", + json_schema_extra={"example": 1} + ) + + +class MiddlewareList(BaseModel): + """Response model for list of middlewares.""" + + middlewares: List[dict] = Field( + ..., + description="Array of middleware configurations" + ) + pagination: PaginationInfo = Field( + ..., + description="Pagination information" + ) + + +class MiddlewareCreateResponse(BaseModel): + """Response model for middleware creation.""" + + id: str = Field( + ..., + alias="_id", + description="Unique identifier of created middleware", + json_schema_extra={"example": "507f1f77bcf86cd799439011"} + ) + order: int = Field( + ..., + description="Assigned execution order", + json_schema_extra={"example": 0} + ) + message: str = Field( + ..., + description="Success message", + json_schema_extra={"example": "Middleware added successfully"} + ) + + class Config: + """Pydantic model configuration.""" + + populate_by_name = True + + +class MiddlewareOrder(BaseModel): + """Request model for reordering middlewares.""" + + ordered_ids: List[str] = Field( + ..., + min_length=1, + description="Array of middleware IDs in desired execution order", + json_schema_extra={ + "example": [ + "507f1f77bcf86cd799439011", + "507f1f77bcf86cd799439012" + ] + } + ) + + +# ============================================================================ +# MongoDB Document Model (Internal Use) +# ============================================================================ + +class MiddlewareDocument(BaseModel): + """MongoDB document structure for middleware storage (internal use).""" + + name: Optional[str] + source: Union[MiddlewareSource, List[MiddlewareSource]] + order: int + enabled: bool = True + config: Optional[dict] = None + created_at: str + updated_at: str diff --git a/pro_tes/exceptions.py b/pro_tes/exceptions.py index 116d895..5f30a49 100644 --- a/pro_tes/exceptions.py +++ b/pro_tes/exceptions.py @@ -49,6 +49,18 @@ class TesUriError(ValueError): """Raised when TES URI cannot be parsed.""" +class MiddlewareNotFound(NotFound): + """Raised when middleware with given ID was not found.""" + + +class MiddlewareDuplicateName(BadRequest): + """Raised when middleware name already exists.""" + + +class MiddlewareDuplicateEntryPoint(BadRequest): + """Raised when middleware entry_point already exists.""" + + exceptions = { Exception: { "message": "An unexpected error occurred.", @@ -118,4 +130,16 @@ class TesUriError(ValueError): "message": "IP distance calculation failed.", "code": "500", }, + MiddlewareNotFound: { + "message": "Middleware with given ID was not found.", + "code": "404", + }, + MiddlewareDuplicateName: { + "message": "Middleware name already exists.", + "code": "400", + }, + MiddlewareDuplicateEntryPoint: { + "message": "Middleware entry_point already exists.", + "code": "400", + }, }