From 79909b9cce80cc04758e2598124ab8d028f64f89 Mon Sep 17 00:00:00 2001 From: serhii lemeshko Date: Sat, 11 Apr 2026 14:21:47 +0300 Subject: [PATCH 01/52] feat: add API entry point and Vercel configuration for routing --- api/index.py | 1 + vercel.json | 9 +++++++++ 2 files changed, 10 insertions(+) create mode 100644 api/index.py create mode 100644 vercel.json diff --git a/api/index.py b/api/index.py new file mode 100644 index 0000000..7436e63 --- /dev/null +++ b/api/index.py @@ -0,0 +1 @@ +from app.main import app diff --git a/vercel.json b/vercel.json new file mode 100644 index 0000000..2738390 --- /dev/null +++ b/vercel.json @@ -0,0 +1,9 @@ +{ + "version": 2, + "rewrites": [ + { + "source": "/(.*)", + "destination": "/api/index" + } + ] +} From ca947fda6bf7f80fb7980643f89d1d65b5685cdb Mon Sep 17 00:00:00 2001 From: opsmanager1 Date: Sat, 11 Apr 2026 20:27:15 +0300 Subject: [PATCH 02/52] fix: update port mapping in docker-compose for app service --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 0c7cf46..5b4a3ed 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -8,7 +8,7 @@ services: env_file: - .env ports: - - "8000:8000" + - "8001:8001" healthcheck: test: [ From 256b19ed0292a902d040aff0edb54d82070fb24d Mon Sep 17 00:00:00 2001 From: opsmanager1 Date: Sat, 11 Apr 2026 20:37:51 +0300 Subject: [PATCH 03/52] fix: update port from 8000 to 8001 in Dockerfile, entrypoint, and healthcheck; remove unused API index --- Dockerfile | 4 ++-- api/index.py | 1 - docker-compose.yml | 2 +- entrypoint.sh | 2 +- vercel.json | 9 --------- 5 files changed, 4 insertions(+), 14 deletions(-) delete mode 100644 api/index.py delete mode 100644 vercel.json diff --git a/Dockerfile b/Dockerfile index 957ca20..f44b397 100644 --- a/Dockerfile +++ b/Dockerfile @@ -17,7 +17,7 @@ RUN sed -i 's/\r$//' /app/entrypoint.sh \ USER appuser -EXPOSE 8000 +EXPOSE 8001 ENTRYPOINT ["/app/entrypoint.sh"] -CMD ["python", "-m", "uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8000"] \ No newline at end of file +CMD ["python", "-m", "uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8001"] \ No newline at end of file diff --git a/api/index.py b/api/index.py deleted file mode 100644 index 7436e63..0000000 --- a/api/index.py +++ /dev/null @@ -1 +0,0 @@ -from app.main import app diff --git a/docker-compose.yml b/docker-compose.yml index 5b4a3ed..066f6d5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -15,7 +15,7 @@ services: "CMD", "python", "-c", - "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8000/health', timeout=2)", + "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8001/health', timeout=2)", ] interval: 30s timeout: 5s diff --git a/entrypoint.sh b/entrypoint.sh index 893b72d..b234644 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -8,7 +8,7 @@ echo "[start] Running admin bootstrap script..." python -m app.scripts.bootstrap_admin if [ "$#" -eq 0 ]; then - set -- python -m uvicorn app.main:app --host 0.0.0.0 --port 8000 + set -- python -m uvicorn app.main:app --host 0.0.0.0 --port 8001 fi echo "[start] Starting: $*" diff --git a/vercel.json b/vercel.json deleted file mode 100644 index 2738390..0000000 --- a/vercel.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "version": 2, - "rewrites": [ - { - "source": "/(.*)", - "destination": "/api/index" - } - ] -} From 804383bd0ac08c98fd5f0ae84498ad8c3224c505 Mon Sep 17 00:00:00 2001 From: opsmanager1 Date: Sat, 11 Apr 2026 21:03:33 +0300 Subject: [PATCH 04/52] feat: add SecurityMiddleware with content security policy to enhance security --- app/middlewares/security/setup.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/app/middlewares/security/setup.py b/app/middlewares/security/setup.py index 8ff2ace..5b1559a 100644 --- a/app/middlewares/security/setup.py +++ b/app/middlewares/security/setup.py @@ -1,5 +1,6 @@ from fastapi import FastAPI from starlette.middleware.cors import CORSMiddleware +from starlette.middleware.security import SecurityMiddleware from starlette.middleware.httpsredirect import HTTPSRedirectMiddleware from starlette.middleware.trustedhost import TrustedHostMiddleware @@ -11,6 +12,10 @@ def configure_security_middlewares(app: FastAPI) -> None: + app.add_middleware( + SecurityMiddleware, + content_security_policy="default-src 'self'; style-src 'self' https://cdn.jsdelivr.net; script-src 'self' https://cdn.jsdelivr.net; img-src 'self' https://fastapi.tiangolo.com", + ) if settings.SECURITY_ENABLE_HTTPS_REDIRECT: app.add_middleware(HTTPSRedirectMiddleware) From 2915d57844c7590efcc748fcc081e8318ba00c71 Mon Sep 17 00:00:00 2001 From: opsmanager1 Date: Sat, 11 Apr 2026 21:08:36 +0300 Subject: [PATCH 05/52] fix: format content security policy for SecurityMiddleware configuration --- app/middlewares/security/setup.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/app/middlewares/security/setup.py b/app/middlewares/security/setup.py index 5b1559a..aaef825 100644 --- a/app/middlewares/security/setup.py +++ b/app/middlewares/security/setup.py @@ -12,10 +12,17 @@ def configure_security_middlewares(app: FastAPI) -> None: - app.add_middleware( - SecurityMiddleware, - content_security_policy="default-src 'self'; style-src 'self' https://cdn.jsdelivr.net; script-src 'self' https://cdn.jsdelivr.net; img-src 'self' https://fastapi.tiangolo.com", - ) + + app.add_middleware( + SecurityMiddleware, + content_security_policy=( + "default-src 'self'; " + "style-src 'self' https://cdn.jsdelivr.net; " + "script-src 'self' https://cdn.jsdelivr.net; " + "img-src 'self' https://fastapi.tiangolo.com" + ), + ) + if settings.SECURITY_ENABLE_HTTPS_REDIRECT: app.add_middleware(HTTPSRedirectMiddleware) From 85404d3ad3fd2cecb8574e51d0a893dc93f11369 Mon Sep 17 00:00:00 2001 From: opsmanager1 Date: Sat, 11 Apr 2026 21:12:07 +0300 Subject: [PATCH 06/52] fix: remove SecurityMiddleware and its content security policy from configuration --- app/middlewares/security/setup.py | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/app/middlewares/security/setup.py b/app/middlewares/security/setup.py index aaef825..2916aa8 100644 --- a/app/middlewares/security/setup.py +++ b/app/middlewares/security/setup.py @@ -1,6 +1,5 @@ from fastapi import FastAPI from starlette.middleware.cors import CORSMiddleware -from starlette.middleware.security import SecurityMiddleware from starlette.middleware.httpsredirect import HTTPSRedirectMiddleware from starlette.middleware.trustedhost import TrustedHostMiddleware @@ -13,16 +12,6 @@ def configure_security_middlewares(app: FastAPI) -> None: - app.add_middleware( - SecurityMiddleware, - content_security_policy=( - "default-src 'self'; " - "style-src 'self' https://cdn.jsdelivr.net; " - "script-src 'self' https://cdn.jsdelivr.net; " - "img-src 'self' https://fastapi.tiangolo.com" - ), - ) - if settings.SECURITY_ENABLE_HTTPS_REDIRECT: app.add_middleware(HTTPSRedirectMiddleware) From baec2e086859be77c24fdf7f40d9fafed239a6e9 Mon Sep 17 00:00:00 2001 From: opsmanager1 Date: Sat, 11 Apr 2026 21:16:39 +0300 Subject: [PATCH 07/52] fix: clear content security policy in settings --- app/core/settings.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/app/core/settings.py b/app/core/settings.py index 613ae2e..80bdaac 100644 --- a/app/core/settings.py +++ b/app/core/settings.py @@ -31,9 +31,7 @@ class Settings(BaseSettings): ALLOWED_ORIGINS: list[str] = ["*"] SECURITY_TRUSTED_HOSTS: list[str] = ["*"] SECURITY_ENABLE_HTTPS_REDIRECT: bool = False - SECURITY_CSP: str = ( - "default-src 'self'; frame-ancestors 'none'; base-uri 'self'; form-action 'self'" - ) + SECURITY_CSP: str = "" SECURITY_REFERRER_POLICY: str = "strict-origin-when-cross-origin" SECURITY_CORS_ALLOW_METHODS: list[str] = [ "GET", From bc7d06c833d2b60b3de0f50df62c2c7357b2e498 Mon Sep 17 00:00:00 2001 From: opsmanager1 Date: Sat, 11 Apr 2026 21:21:34 +0300 Subject: [PATCH 08/52] fix: remove Content-Security-Policy handling from SecurityHeadersMiddleware --- app/middlewares/security/headers.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/app/middlewares/security/headers.py b/app/middlewares/security/headers.py index b80ee2f..8b18b04 100644 --- a/app/middlewares/security/headers.py +++ b/app/middlewares/security/headers.py @@ -1,4 +1,3 @@ -import uuid from starlette.datastructures import MutableHeaders from starlette.types import ASGIApp, Message, Receive, Scope, Send @@ -22,13 +21,6 @@ async def send_with_security_headers(message: Message) -> None: headers.setdefault("X-Frame-Options", "DENY") headers.setdefault("Referrer-Policy", settings.SECURITY_REFERRER_POLICY) - csp = settings.SECURITY_CSP - if "{nonce}" in csp: - nonce = uuid.uuid4().hex - scope["csp_nonce"] = nonce - csp = csp.replace("{nonce}", nonce) - headers.setdefault("Content-Security-Policy", csp) - if settings.ENV.lower() == "production": headers.setdefault( "Strict-Transport-Security", From c2c0ebbfad3549ddb4ea40ec98a1750e4eabc222 Mon Sep 17 00:00:00 2001 From: opsmanager1 Date: Sat, 11 Apr 2026 21:52:23 +0300 Subject: [PATCH 09/52] fix: set default value of is_verified to True in User model --- app/modules/auth/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/modules/auth/models.py b/app/modules/auth/models.py index 03c0222..5ffb030 100644 --- a/app/modules/auth/models.py +++ b/app/modules/auth/models.py @@ -17,7 +17,7 @@ class User(SQLAlchemyBaseUserTable[int], Base): hashed_password: Mapped[str] = mapped_column(String(1024), nullable=False) is_active: Mapped[bool] = mapped_column(Boolean, default=True, nullable=False) is_superuser: Mapped[bool] = mapped_column(Boolean, default=False, nullable=False) - is_verified: Mapped[bool] = mapped_column(Boolean, default=False, nullable=False) + is_verified: Mapped[bool] = mapped_column(Boolean, default=True, nullable=False) created_at: Mapped[datetime] = mapped_column( DateTime(timezone=True), server_default=func.now(), From 57a4a2fda9766fe75002647e060a4008ec807142 Mon Sep 17 00:00:00 2001 From: opsmanager1 Date: Sat, 11 Apr 2026 21:58:02 +0300 Subject: [PATCH 10/52] fix: update UserCreate schema to inherit from BaseModel and include email and password fields --- app/modules/auth/schemas.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/app/modules/auth/schemas.py b/app/modules/auth/schemas.py index 2273fb5..f9a6e68 100644 --- a/app/modules/auth/schemas.py +++ b/app/modules/auth/schemas.py @@ -1,12 +1,15 @@ + from fastapi_users import schemas -from pydantic import Field +from pydantic import Field, BaseModel, EmailStr class UserRead(schemas.BaseUser[int]): username: str -class UserCreate(schemas.BaseUserCreate): +class UserCreate(BaseModel): + email: EmailStr + password: str username: str = Field(min_length=3, max_length=100) From 27cd9f9780aaf3e35c81886ecead832381743262 Mon Sep 17 00:00:00 2001 From: opsmanager1 Date: Sat, 11 Apr 2026 22:15:41 +0300 Subject: [PATCH 11/52] fix: update error message for missing AUTH_PASSWORD in bootstrap process --- app/core/bootstrap.py | 4 +--- requirements.txt | 1 + 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/app/core/bootstrap.py b/app/core/bootstrap.py index 021592f..5f00ba4 100644 --- a/app/core/bootstrap.py +++ b/app/core/bootstrap.py @@ -11,11 +11,9 @@ def _resolve_bootstrap_hashed_password() -> str: helper = PasswordHelper() - if settings.AUTH_PASSWORD_HASH: - return settings.AUTH_PASSWORD_HASH if not settings.AUTH_PASSWORD: raise RuntimeError( - "AUTH_PASSWORD must be set when AUTH_PASSWORD_HASH is not provided" + "AUTH_PASSWORD must be set for admin bootstrap user creation" ) return helper.hash(settings.AUTH_PASSWORD) diff --git a/requirements.txt b/requirements.txt index e366f2b..4c89f19 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,6 +13,7 @@ alembic==1.13.1 pydantic==2.7.1 pydantic-settings==2.2.1 + # Auth / security python-multipart==0.0.20 fastapi-users==14.0.1 From c6ff9cd72705589e849abd5207f0578fb1b374c9 Mon Sep 17 00:00:00 2001 From: opsmanager1 Date: Sat, 11 Apr 2026 22:23:35 +0300 Subject: [PATCH 12/52] fix: add argon2-cffi to requirements for password hashing --- requirements.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/requirements.txt b/requirements.txt index 4c89f19..c55ea22 100644 --- a/requirements.txt +++ b/requirements.txt @@ -20,6 +20,9 @@ fastapi-users==14.0.1 fastapi-users-db-sqlalchemy[sqlalchemy]==7.0.0 asyncpg==0.30.0 + +argon2-cffi==23.1.0 + # Environment python-dotenv==1.0.1 From b5b29851728b787e07b12bbc579c8d7eb1d842eb Mon Sep 17 00:00:00 2001 From: opsmanager1 Date: Sat, 11 Apr 2026 22:32:01 +0300 Subject: [PATCH 13/52] fix: add Alembic schema sync check before migrations --- entrypoint.sh | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/entrypoint.sh b/entrypoint.sh index b234644..2309a6d 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -2,6 +2,14 @@ set -eu echo "[start] Running Alembic migrations..." + + +echo "[start] Checking Alembic schema sync..." +python -m alembic revision --autogenerate --check > /dev/null 2>&1 || { + echo "[error] Alembic schema is out of sync! Please generate and apply migrations." >&2 + exit 1 +} + python -m alembic upgrade head echo "[start] Running admin bootstrap script..." From aa3455543ba560d69aae89e53ea10bb1538f1e17 Mon Sep 17 00:00:00 2001 From: opsmanager1 Date: Sat, 11 Apr 2026 22:34:11 +0300 Subject: [PATCH 14/52] fix: improve Alembic schema sync check and migration handling --- entrypoint.sh | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/entrypoint.sh b/entrypoint.sh index 2309a6d..a89f7a0 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -2,15 +2,17 @@ set -eu echo "[start] Running Alembic migrations..." - - echo "[start] Checking Alembic schema sync..." -python -m alembic revision --autogenerate --check > /dev/null 2>&1 || { - echo "[error] Alembic schema is out of sync! Please generate and apply migrations." >&2 - exit 1 -} -python -m alembic upgrade head +echo "[start] Checking Alembic schema sync..." +if ! python -m alembic revision --autogenerate --check > /dev/null 2>&1; then + echo "[info] Alembic schema is out of sync. Generating migration..." + python -m alembic revision --autogenerate -m "auto sync migration" + echo "[info] Applying new migration..." + python -m alembic upgrade head +else + python -m alembic upgrade head +fi echo "[start] Running admin bootstrap script..." python -m app.scripts.bootstrap_admin From c0f81deeca5564315722f4ae8c16d0164b108aa2 Mon Sep 17 00:00:00 2001 From: opsmanager1 Date: Sat, 11 Apr 2026 22:41:47 +0300 Subject: [PATCH 15/52] fix: enhance user authentication by adding username awareness and updating schemas --- README.md | 5 ++++ .../infrastructure/fastapi_users_adapter.py | 27 ++++++++++++++++++- app/modules/auth/schemas.py | 6 ++--- 3 files changed, 33 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 0650822..bd5fb06 100644 --- a/README.md +++ b/README.md @@ -141,6 +141,11 @@ Base API prefix: /api/v1 - PATCH /api/v1/auth/users/{id} - DELETE /api/v1/auth/users/{id} +Login note: + +- Endpoint `/api/v1/auth/jwt/login` uses `application/x-www-form-urlencoded`. +- In form field `username`, pass only `AUTH_USERNAME`. + ### LLM - POST /api/v1/llm/generate diff --git a/app/modules/auth/infrastructure/fastapi_users_adapter.py b/app/modules/auth/infrastructure/fastapi_users_adapter.py index d862c79..b57a5a9 100644 --- a/app/modules/auth/infrastructure/fastapi_users_adapter.py +++ b/app/modules/auth/infrastructure/fastapi_users_adapter.py @@ -1,10 +1,12 @@ from collections.abc import AsyncGenerator from fastapi import Depends +from fastapi.security import OAuth2PasswordRequestForm from fastapi_users import BaseUserManager, FastAPIUsers, IntegerIDMixin from fastapi_users.authentication import (AuthenticationBackend, BearerTransport, JWTStrategy) from fastapi_users_db_sqlalchemy import SQLAlchemyUserDatabase +from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession from app.core.database import get_async_db @@ -12,15 +14,38 @@ from app.modules.auth.models import User +class UsernameAwareUserDatabase(SQLAlchemyUserDatabase): + async def get_by_username(self, username: str) -> User | None: + statement = select(User).where(User.username == username) + return await self._get_user(statement) + + class UserManager(IntegerIDMixin, BaseUserManager[User, int]): reset_password_token_secret = settings.SECRET_KEY verification_token_secret = settings.SECRET_KEY + async def authenticate( + self, credentials: OAuth2PasswordRequestForm + ) -> User | None: + if (user := await self.user_db.get_by_username(credentials.username)) is None: + return None + + verified, updated_password_hash = self.password_helper.verify_and_update( + credentials.password, user.hashed_password + ) + if not verified: + return None + + if updated_password_hash is not None: + await self.user_db.update(user, {"hashed_password": updated_password_hash}) + + return user + async def get_user_db( session: AsyncSession = Depends(get_async_db), ) -> AsyncGenerator[SQLAlchemyUserDatabase, None]: - yield SQLAlchemyUserDatabase(session, User) + yield UsernameAwareUserDatabase(session, User) async def get_user_manager( diff --git a/app/modules/auth/schemas.py b/app/modules/auth/schemas.py index f9a6e68..db78b2f 100644 --- a/app/modules/auth/schemas.py +++ b/app/modules/auth/schemas.py @@ -1,15 +1,13 @@ from fastapi_users import schemas -from pydantic import Field, BaseModel, EmailStr +from pydantic import Field class UserRead(schemas.BaseUser[int]): username: str -class UserCreate(BaseModel): - email: EmailStr - password: str +class UserCreate(schemas.BaseUserCreate): username: str = Field(min_length=3, max_length=100) From 5703c92231a0f0960333be234250e0375a138b6b Mon Sep 17 00:00:00 2001 From: opsmanager1 Date: Sat, 11 Apr 2026 23:05:27 +0300 Subject: [PATCH 16/52] feat: implement JWT refresh token functionality and update authentication endpoints --- README.md | 4 + .../6f4e8b8f4b11_add_refresh_tokens.py | 54 +++++++ app/core/settings.py | 10 ++ .../infrastructure/fastapi_users_adapter.py | 7 + app/modules/auth/jwt_router.py | 103 ++++++++++++++ app/modules/auth/models.py | 40 +++++- app/modules/auth/router.py | 6 +- app/modules/auth/schemas.py | 13 +- app/modules/auth/service.py | 62 ++++++++ requirements-dev.txt | 1 + tests/modules/api/test_auth_api.py | 134 ++++++++++++++++++ 11 files changed, 427 insertions(+), 7 deletions(-) create mode 100644 alembic/versions/6f4e8b8f4b11_add_refresh_tokens.py create mode 100644 app/modules/auth/jwt_router.py diff --git a/README.md b/README.md index bd5fb06..9f501ba 100644 --- a/README.md +++ b/README.md @@ -133,6 +133,7 @@ Base API prefix: /api/v1 ### Auth (fastapi-users) - POST /api/v1/auth/jwt/login +- POST /api/v1/auth/jwt/refresh - POST /api/v1/auth/jwt/logout - POST /api/v1/auth/register - GET /api/v1/auth/users/me @@ -145,6 +146,9 @@ Login note: - Endpoint `/api/v1/auth/jwt/login` uses `application/x-www-form-urlencoded`. - In form field `username`, pass only `AUTH_USERNAME`. +- Login response returns `access_token` in body and sets HttpOnly refresh cookie. +- Use `/api/v1/auth/jwt/refresh` to get a new access token and refresh cookie. +- `/api/v1/auth/jwt/logout` clears refresh cookie on client side. ### LLM diff --git a/alembic/versions/6f4e8b8f4b11_add_refresh_tokens.py b/alembic/versions/6f4e8b8f4b11_add_refresh_tokens.py new file mode 100644 index 0000000..d0079cc --- /dev/null +++ b/alembic/versions/6f4e8b8f4b11_add_refresh_tokens.py @@ -0,0 +1,54 @@ +"""add refresh tokens + +Revision ID: 6f4e8b8f4b11 +Revises: 2e5bd3a1eb88 +Create Date: 2026-04-11 22:30:00.000000 +""" + +import sqlalchemy as sa + +from alembic import op + +revision = "6f4e8b8f4b11" +down_revision = "2e5bd3a1eb88" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.create_table( + "refresh_tokens", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("user_id", sa.Integer(), nullable=False), + sa.Column("token_hash", sa.String(length=128), nullable=False), + sa.Column("expires_at", sa.DateTime(timezone=True), nullable=False), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.Column("revoked_at", sa.DateTime(timezone=True), nullable=True), + sa.ForeignKeyConstraint(["user_id"], ["users.id"], ondelete="CASCADE"), + sa.PrimaryKeyConstraint("id"), + ) + op.create_index(op.f("ix_refresh_tokens_id"), "refresh_tokens", ["id"], unique=False) + op.create_index( + op.f("ix_refresh_tokens_token_hash"), + "refresh_tokens", + ["token_hash"], + unique=True, + ) + op.create_index( + op.f("ix_refresh_tokens_user_id"), + "refresh_tokens", + ["user_id"], + unique=False, + ) + + +def downgrade() -> None: + op.drop_index(op.f("ix_refresh_tokens_user_id"), table_name="refresh_tokens") + op.drop_index(op.f("ix_refresh_tokens_token_hash"), table_name="refresh_tokens") + op.drop_index(op.f("ix_refresh_tokens_id"), table_name="refresh_tokens") + op.drop_table("refresh_tokens") diff --git a/app/core/settings.py b/app/core/settings.py index 80bdaac..98d4676 100644 --- a/app/core/settings.py +++ b/app/core/settings.py @@ -27,6 +27,16 @@ class Settings(BaseSettings): AUTH_EMAIL: str | None = None AUTH_PASSWORD: str | None = None AUTH_PASSWORD_HASH: str | None = None + AUTH_USERNAME_MIN_LENGTH: int = 3 + AUTH_USERNAME_MAX_LENGTH: int = 100 + AUTH_HASHED_PASSWORD_MAX_LENGTH: int = 1024 + AUTH_REFRESH_TOKEN_HASH_LENGTH: int = 128 + AUTH_DAY_IN_SECONDS: int = 24 * 60 * 60 + AUTH_REFRESH_TOKEN_EXPIRE_DAYS: int = 14 + AUTH_REFRESH_TOKEN_EXPIRE_SECONDS: int = 14 * 24 * 60 * 60 + AUTH_REFRESH_COOKIE_NAME: str = "refresh_token" + AUTH_REFRESH_COOKIE_SECURE: bool = True + AUTH_REFRESH_COOKIE_SAMESITE: str = "lax" ALLOWED_ORIGINS: list[str] = ["*"] SECURITY_TRUSTED_HOSTS: list[str] = ["*"] diff --git a/app/modules/auth/infrastructure/fastapi_users_adapter.py b/app/modules/auth/infrastructure/fastapi_users_adapter.py index b57a5a9..daa8e27 100644 --- a/app/modules/auth/infrastructure/fastapi_users_adapter.py +++ b/app/modules/auth/infrastructure/fastapi_users_adapter.py @@ -67,6 +67,13 @@ def get_jwt_strategy() -> JWTStrategy: ) +def get_refresh_jwt_strategy() -> JWTStrategy: + return JWTStrategy( + secret=settings.SECRET_KEY, + lifetime_seconds=settings.AUTH_REFRESH_TOKEN_EXPIRE_SECONDS, + ) + + auth_backend = AuthenticationBackend( name="jwt", transport=bearer_transport, diff --git a/app/modules/auth/jwt_router.py b/app/modules/auth/jwt_router.py new file mode 100644 index 0000000..f02c711 --- /dev/null +++ b/app/modules/auth/jwt_router.py @@ -0,0 +1,103 @@ +from fastapi import APIRouter, Depends, HTTPException, Request, Response, status +from fastapi.responses import JSONResponse +from fastapi.security import OAuth2PasswordRequestForm + +from app.core.settings import settings +from app.modules.auth.infrastructure.fastapi_users_adapter import ( + get_jwt_strategy, + get_refresh_jwt_strategy, + get_user_manager, +) +from app.modules.auth.service import AuthSessionService + +router = APIRouter(prefix="/jwt") + + +def _build_refresh_cookie_path() -> str: + return f"{settings.API_V1_PREFIX}{settings.AUTH_PREFIX}/jwt" + + +def _set_refresh_cookie(response: Response, refresh_token: str) -> None: + response.set_cookie( + key=settings.AUTH_REFRESH_COOKIE_NAME, + value=refresh_token, + httponly=True, + secure=settings.AUTH_REFRESH_COOKIE_SECURE, + samesite=settings.AUTH_REFRESH_COOKIE_SAMESITE, + max_age=settings.AUTH_REFRESH_TOKEN_EXPIRE_SECONDS, + path=_build_refresh_cookie_path(), + ) + + +def _clear_refresh_cookie(response: Response) -> None: + response.delete_cookie( + key=settings.AUTH_REFRESH_COOKIE_NAME, + path=_build_refresh_cookie_path(), + ) + + +async def get_auth_session_service( + user_manager=Depends(get_user_manager), +) -> AuthSessionService: + return AuthSessionService( + user_authentication=user_manager, + access_token_strategy=get_jwt_strategy(), + refresh_token_strategy=get_refresh_jwt_strategy(), + ) + + +@router.post("/login", name="auth:jwt.login") +async def login( + credentials: OAuth2PasswordRequestForm = Depends(), + auth_session_service: AuthSessionService = Depends(get_auth_session_service), +) -> JSONResponse: + tokens = await auth_session_service.login(credentials) + if tokens is None: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="LOGIN_BAD_CREDENTIALS", + ) + + response = JSONResponse( + {"access_token": tokens.access_token, "token_type": "bearer"} + ) + _set_refresh_cookie(response, tokens.refresh_token) + return response + + +@router.post("/refresh", name="auth:jwt.refresh") +async def refresh( + request: Request, + auth_session_service: AuthSessionService = Depends(get_auth_session_service), +) -> JSONResponse: + refresh_token = request.cookies.get(settings.AUTH_REFRESH_COOKIE_NAME) + if not refresh_token: + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="Unauthorized", + ) + + tokens = await auth_session_service.refresh(refresh_token) + if tokens is None: + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="Unauthorized", + ) + + response = JSONResponse( + {"access_token": tokens.access_token, "token_type": "bearer"} + ) + _set_refresh_cookie(response, tokens.refresh_token) + return response + + +@router.post("/logout", name="auth:jwt.logout", status_code=status.HTTP_204_NO_CONTENT) +async def logout( + request: Request, + auth_session_service: AuthSessionService = Depends(get_auth_session_service), +) -> Response: + await auth_session_service.logout(request.cookies.get(settings.AUTH_REFRESH_COOKIE_NAME)) + + response = Response(status_code=status.HTTP_204_NO_CONTENT) + _clear_refresh_cookie(response) + return response diff --git a/app/modules/auth/models.py b/app/modules/auth/models.py index 5ffb030..c688d0a 100644 --- a/app/modules/auth/models.py +++ b/app/modules/auth/models.py @@ -1,10 +1,11 @@ from datetime import datetime from fastapi_users_db_sqlalchemy import SQLAlchemyBaseUserTable -from sqlalchemy import Boolean, DateTime, Integer, String, func +from sqlalchemy import Boolean, DateTime, ForeignKey, Integer, String, func from sqlalchemy.orm import Mapped, mapped_column from app.core.database import Base +from app.core.settings import settings class User(SQLAlchemyBaseUserTable[int], Base): @@ -12,9 +13,15 @@ class User(SQLAlchemyBaseUserTable[int], Base): id: Mapped[int] = mapped_column(Integer, primary_key=True, index=True) username: Mapped[str] = mapped_column( - String(100), unique=True, index=True, nullable=False + String(settings.AUTH_USERNAME_MAX_LENGTH), + unique=True, + index=True, + nullable=False, + ) + hashed_password: Mapped[str] = mapped_column( + String(settings.AUTH_HASHED_PASSWORD_MAX_LENGTH), + nullable=False, ) - hashed_password: Mapped[str] = mapped_column(String(1024), nullable=False) is_active: Mapped[bool] = mapped_column(Boolean, default=True, nullable=False) is_superuser: Mapped[bool] = mapped_column(Boolean, default=False, nullable=False) is_verified: Mapped[bool] = mapped_column(Boolean, default=True, nullable=False) @@ -23,3 +30,30 @@ class User(SQLAlchemyBaseUserTable[int], Base): server_default=func.now(), nullable=False, ) + + +class RefreshToken(Base): + __tablename__ = "refresh_tokens" + + id: Mapped[int] = mapped_column(Integer, primary_key=True, index=True) + user_id: Mapped[int] = mapped_column( + Integer, + ForeignKey("users.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) + token_hash: Mapped[str] = mapped_column( + String(settings.AUTH_REFRESH_TOKEN_HASH_LENGTH), + unique=True, + nullable=False, + ) + expires_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), + server_default=func.now(), + nullable=False, + ) + revoked_at: Mapped[datetime | None] = mapped_column( + DateTime(timezone=True), + nullable=True, + ) diff --git a/app/modules/auth/router.py b/app/modules/auth/router.py index 29fe2fa..e52765a 100644 --- a/app/modules/auth/router.py +++ b/app/modules/auth/router.py @@ -2,12 +2,14 @@ from app.core.settings import settings from app.modules.auth.infrastructure.fastapi_users_adapter import ( - auth_backend, fastapi_users) + fastapi_users, +) +from app.modules.auth.jwt_router import router as jwt_router from app.modules.auth.schemas import UserCreate, UserRead, UserUpdate router = APIRouter(prefix=settings.AUTH_PREFIX, tags=[settings.AUTH_TAG]) -router.include_router(fastapi_users.get_auth_router(auth_backend), prefix="/jwt") +router.include_router(jwt_router) router.include_router( fastapi_users.get_register_router(UserRead, UserCreate), prefix="" ) diff --git a/app/modules/auth/schemas.py b/app/modules/auth/schemas.py index db78b2f..8c03e69 100644 --- a/app/modules/auth/schemas.py +++ b/app/modules/auth/schemas.py @@ -2,14 +2,23 @@ from fastapi_users import schemas from pydantic import Field +from app.core.settings import settings + class UserRead(schemas.BaseUser[int]): username: str class UserCreate(schemas.BaseUserCreate): - username: str = Field(min_length=3, max_length=100) + username: str = Field( + min_length=settings.AUTH_USERNAME_MIN_LENGTH, + max_length=settings.AUTH_USERNAME_MAX_LENGTH, + ) class UserUpdate(schemas.BaseUserUpdate): - username: str | None = Field(default=None, min_length=3, max_length=100) + username: str | None = Field( + default=None, + min_length=settings.AUTH_USERNAME_MIN_LENGTH, + max_length=settings.AUTH_USERNAME_MAX_LENGTH, + ) diff --git a/app/modules/auth/service.py b/app/modules/auth/service.py index 184038a..0590802 100644 --- a/app/modules/auth/service.py +++ b/app/modules/auth/service.py @@ -1,8 +1,35 @@ +from dataclasses import dataclass +from typing import Protocol + +from fastapi.security import OAuth2PasswordRequestForm + from app.modules.auth.errors import PermissionDeniedError from app.modules.auth.models import User from app.modules.auth.repository.user_repository import UserRepositoryPort +class UserAuthenticationPort(Protocol): + async def authenticate(self, credentials: OAuth2PasswordRequestForm) -> User | None: ... + + async def get(self, id): ... + + +class AccessTokenStrategyPort(Protocol): + async def write_token(self, user: User) -> str: ... + + +class RefreshTokenStrategyPort(Protocol): + async def write_token(self, user: User) -> str: ... + + async def read_token(self, token: str, user_manager: UserAuthenticationPort) -> User | None: ... + + +@dataclass(frozen=True) +class AuthTokens: + access_token: str + refresh_token: str + + class AuthorizationService: def ensure_superuser(self, user: User) -> User: if not user.is_superuser: @@ -40,3 +67,38 @@ def __init__(self, repository: UserRepositoryPort) -> None: async def user_exists_by_email(self, email: str) -> bool: return await self._repository.get_by_email(email) is not None + + +class AuthSessionService: + def __init__( + self, + user_authentication: UserAuthenticationPort, + access_token_strategy: AccessTokenStrategyPort, + refresh_token_strategy: RefreshTokenStrategyPort, + ) -> None: + self._user_authentication = user_authentication + self._access_token_strategy = access_token_strategy + self._refresh_token_strategy = refresh_token_strategy + + async def login(self, credentials: OAuth2PasswordRequestForm) -> AuthTokens | None: + user = await self._user_authentication.authenticate(credentials) + if user is None or not user.is_active: + return None + + access_token = await self._access_token_strategy.write_token(user) + refresh_token = await self._refresh_token_strategy.write_token(user) + return AuthTokens(access_token=access_token, refresh_token=refresh_token) + + async def refresh(self, refresh_token: str) -> AuthTokens | None: + user = await self._refresh_token_strategy.read_token( + refresh_token, self._user_authentication + ) + if user is None or not user.is_active: + return None + + access_token = await self._access_token_strategy.write_token(user) + new_refresh_token = await self._refresh_token_strategy.write_token(user) + return AuthTokens(access_token=access_token, refresh_token=new_refresh_token) + + async def logout(self, refresh_token: str | None) -> None: + return None diff --git a/requirements-dev.txt b/requirements-dev.txt index 0a9aa75..c8babd2 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,5 +1,6 @@ pytest==8.2.0 pytest-asyncio==0.23.6 +freezegun==1.5.1 black==24.4.2 isort==5.13.2 flake8==7.0.0 diff --git a/tests/modules/api/test_auth_api.py b/tests/modules/api/test_auth_api.py index 32aaa37..d5d4361 100644 --- a/tests/modules/api/test_auth_api.py +++ b/tests/modules/api/test_auth_api.py @@ -1,7 +1,12 @@ +from datetime import datetime, timedelta, timezone + import pytest +from freezegun import freeze_time from fastapi.testclient import TestClient from app.main import app +from app.modules.auth.jwt_router import get_auth_session_service +from app.modules.auth.service import AuthTokens def _get_auth_me_dependency_callable(): @@ -61,3 +66,132 @@ def test_login_validates_form_payload(api_client: TestClient) -> None: response = api_client.post("/api/v1/auth/jwt/login", data={}) assert response.status_code == 422 + + +@pytest.mark.unit +def test_login_returns_tokens_and_sets_refresh_cookie(api_client: TestClient) -> None: + class FakeAuthSessionService: + async def login(self, credentials): + return AuthTokens(access_token="access-token", refresh_token="refresh-token") + + app.dependency_overrides[get_auth_session_service] = lambda: FakeAuthSessionService() + try: + response = api_client.post( + "/api/v1/auth/jwt/login", + data={"username": "admin", "password": "!QAZ1qaz"}, + ) + finally: + app.dependency_overrides.pop(get_auth_session_service, None) + + assert response.status_code == 200 + assert response.json()["token_type"] == "bearer" + assert response.json()["access_token"] == "access-token" + assert "refresh_token=" in response.headers.get("set-cookie", "") + + +@pytest.mark.unit +def test_refresh_requires_refresh_cookie(api_client: TestClient) -> None: + response = api_client.post("/api/v1/auth/jwt/refresh") + + assert response.status_code == 401 + assert response.json()["detail"] == "Unauthorized" + + +@pytest.mark.unit +def test_refresh_returns_new_access_token(api_client: TestClient) -> None: + class FakeAuthSessionService: + async def refresh(self, refresh_token): + if refresh_token != "valid-refresh-token": + return None + return AuthTokens( + access_token="new-access-token", + refresh_token="new-refresh-token", + ) + + app.dependency_overrides[get_auth_session_service] = lambda: FakeAuthSessionService() + try: + response = api_client.post( + "/api/v1/auth/jwt/refresh", + cookies={"refresh_token": "valid-refresh-token"}, + ) + finally: + app.dependency_overrides.pop(get_auth_session_service, None) + + assert response.status_code == 200 + assert response.json()["token_type"] == "bearer" + assert response.json()["access_token"] == "new-access-token" + assert "refresh_token=" in response.headers.get("set-cookie", "") + + +@pytest.mark.unit +def test_logout_clears_refresh_cookie(api_client: TestClient) -> None: + class FakeAuthSessionService: + async def logout(self, refresh_token): + return None + + app.dependency_overrides[get_auth_session_service] = lambda: FakeAuthSessionService() + try: + response = api_client.post( + "/api/v1/auth/jwt/logout", + cookies={"refresh_token": "any-token"}, + ) + finally: + app.dependency_overrides.pop(get_auth_session_service, None) + + assert response.status_code == 204 + set_cookie = response.headers.get("set-cookie", "") + assert "refresh_token=" in set_cookie + assert "Max-Age=0" in set_cookie or "expires=" in set_cookie.lower() + + +@pytest.mark.unit +def test_refresh_returns_401_when_mocked_token_expired_by_time( + api_client: TestClient, +) -> None: + class TimeAwareFakeAuthSessionService: + def __init__(self) -> None: + self._expires_at: datetime | None = None + + async def login(self, credentials): + self._expires_at = datetime.now(timezone.utc) + timedelta(seconds=60) + return AuthTokens( + access_token="access-token", + refresh_token="refresh-token", + ) + + async def refresh(self, refresh_token): + if self._expires_at is None: + return None + + if datetime.now(timezone.utc) >= self._expires_at: + return None + + return AuthTokens( + access_token="new-access-token", + refresh_token="refresh-token", + ) + + async def logout(self, refresh_token): + return None + + fake_service = TimeAwareFakeAuthSessionService() + app.dependency_overrides[get_auth_session_service] = lambda: fake_service + try: + with freeze_time("2026-04-11 19:00:00"): + login_response = api_client.post( + "/api/v1/auth/jwt/login", + data={"username": "admin", "password": "!QAZ1qaz"}, + ) + + assert login_response.status_code == 200 + + with freeze_time("2026-04-11 19:01:01"): + refresh_response = api_client.post( + "/api/v1/auth/jwt/refresh", + cookies={"refresh_token": "refresh-token"}, + ) + finally: + app.dependency_overrides.pop(get_auth_session_service, None) + + assert refresh_response.status_code == 401 + assert refresh_response.json()["detail"] == "Unauthorized" From 42e977413044bdbabf82873a4dde6ddf2ba7fade Mon Sep 17 00:00:00 2001 From: opsmanager1 Date: Sat, 11 Apr 2026 23:06:41 +0300 Subject: [PATCH 17/52] fix: streamline Alembic schema sync check and migration process --- entrypoint.sh | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/entrypoint.sh b/entrypoint.sh index a89f7a0..e70f91a 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -2,17 +2,13 @@ set -eu echo "[start] Running Alembic migrations..." -echo "[start] Checking Alembic schema sync..." +python -m alembic upgrade head echo "[start] Checking Alembic schema sync..." -if ! python -m alembic revision --autogenerate --check > /dev/null 2>&1; then - echo "[info] Alembic schema is out of sync. Generating migration..." - python -m alembic revision --autogenerate -m "auto sync migration" - echo "[info] Applying new migration..." - python -m alembic upgrade head -else - python -m alembic upgrade head -fi +python -m alembic revision --autogenerate --check > /dev/null 2>&1 || { + echo "[error] Alembic schema is out of sync! Create and commit migrations before deploy." >&2 + exit 1 +} echo "[start] Running admin bootstrap script..." python -m app.scripts.bootstrap_admin From 4af8acbb6c4d381beac9e4905db7b41459b14058 Mon Sep 17 00:00:00 2001 From: opsmanager1 Date: Sat, 11 Apr 2026 23:09:16 +0300 Subject: [PATCH 18/52] fix: add index to RefreshToken token_hash and improve Alembic sync check handling --- app/modules/auth/models.py | 1 + entrypoint.sh | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/app/modules/auth/models.py b/app/modules/auth/models.py index c688d0a..52c3197 100644 --- a/app/modules/auth/models.py +++ b/app/modules/auth/models.py @@ -44,6 +44,7 @@ class RefreshToken(Base): ) token_hash: Mapped[str] = mapped_column( String(settings.AUTH_REFRESH_TOKEN_HASH_LENGTH), + index=True, unique=True, nullable=False, ) diff --git a/entrypoint.sh b/entrypoint.sh index e70f91a..9113f4e 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -5,7 +5,7 @@ echo "[start] Running Alembic migrations..." python -m alembic upgrade head echo "[start] Checking Alembic schema sync..." -python -m alembic revision --autogenerate --check > /dev/null 2>&1 || { +python -m alembic revision --autogenerate --check || { echo "[error] Alembic schema is out of sync! Create and commit migrations before deploy." >&2 exit 1 } From 127f556e81edd2527dd66868f47f197171ee4460 Mon Sep 17 00:00:00 2001 From: opsmanager1 Date: Sat, 11 Apr 2026 23:12:37 +0300 Subject: [PATCH 19/52] fix: update Alembic schema sync check to use 'alembic check' command --- entrypoint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/entrypoint.sh b/entrypoint.sh index 9113f4e..7558bc3 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -5,7 +5,7 @@ echo "[start] Running Alembic migrations..." python -m alembic upgrade head echo "[start] Checking Alembic schema sync..." -python -m alembic revision --autogenerate --check || { +python -m alembic check || { echo "[error] Alembic schema is out of sync! Create and commit migrations before deploy." >&2 exit 1 } From 49736edfb9bc34e0960f3874aa842492ce4a059e Mon Sep 17 00:00:00 2001 From: opsmanager1 Date: Sat, 11 Apr 2026 23:16:48 +0300 Subject: [PATCH 20/52] feat: implement migration and schema sync check script --- app/scripts/migrate_and_check.py | 72 ++++++++++++++++++++++++++++++++ entrypoint.sh | 10 +---- 2 files changed, 74 insertions(+), 8 deletions(-) create mode 100644 app/scripts/migrate_and_check.py diff --git a/app/scripts/migrate_and_check.py b/app/scripts/migrate_and_check.py new file mode 100644 index 0000000..39f6f81 --- /dev/null +++ b/app/scripts/migrate_and_check.py @@ -0,0 +1,72 @@ +from pathlib import Path +import logging + +from alembic import command +from alembic.config import Config +from alembic.runtime.migration import MigrationContext +from alembic.script import ScriptDirectory +from sqlalchemy import create_engine + +from app.core.settings import settings + +logger = logging.getLogger(__name__) + + +def _normalize_database_url(url: str) -> str: + if url.startswith("postgres://"): + return url.replace("postgres://", "postgresql://", 1) + return url + + +def _project_root() -> Path: + return Path(__file__).resolve().parents[2] + + +def _build_alembic_config() -> Config: + config = Config(str(_project_root() / "alembic.ini")) + config.set_main_option( + "sqlalchemy.url", _normalize_database_url(settings.DATABASE_URL) + ) + return config + + +def migrate_and_check() -> None: + config = _build_alembic_config() + + logger.info("Running Alembic upgrade to head") + command.upgrade(config, "head") + + script = ScriptDirectory.from_config(config) + expected_heads = set(script.get_heads()) + + engine = create_engine( + _normalize_database_url(settings.DATABASE_URL), + pool_pre_ping=True, + ) + + try: + with engine.connect() as connection: + context = MigrationContext.configure(connection) + current_heads = set(context.get_current_heads()) + finally: + engine.dispose() + + if current_heads != expected_heads: + raise RuntimeError( + "Alembic DB revision is not at head: " + f"current={sorted(current_heads)} expected={sorted(expected_heads)}" + ) + + logger.info("Alembic migrations are applied and DB is in sync") + + +def main() -> None: + logging.basicConfig( + level=logging.INFO, + format="[migrations] %(message)s", + ) + migrate_and_check() + + +if __name__ == "__main__": + main() diff --git a/entrypoint.sh b/entrypoint.sh index 7558bc3..2116919 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -1,14 +1,8 @@ #!/usr/bin/env sh set -eu -echo "[start] Running Alembic migrations..." -python -m alembic upgrade head - -echo "[start] Checking Alembic schema sync..." -python -m alembic check || { - echo "[error] Alembic schema is out of sync! Create and commit migrations before deploy." >&2 - exit 1 -} +echo "[start] Running migrations and schema sync check..." +python -m app.scripts.migrate_and_check echo "[start] Running admin bootstrap script..." python -m app.scripts.bootstrap_admin From 5faf1e00a2fef93f09ad0b8b49d0dcbb47d28303 Mon Sep 17 00:00:00 2001 From: opsmanager1 Date: Sat, 11 Apr 2026 23:20:32 +0300 Subject: [PATCH 21/52] refactor: streamline Alembic migration process by removing unused imports and simplifying configuration --- app/scripts/migrate_and_check.py | 56 ++++++++++++++------------------ 1 file changed, 25 insertions(+), 31 deletions(-) diff --git a/app/scripts/migrate_and_check.py b/app/scripts/migrate_and_check.py index 39f6f81..561cde0 100644 --- a/app/scripts/migrate_and_check.py +++ b/app/scripts/migrate_and_check.py @@ -1,13 +1,8 @@ -from pathlib import Path import logging - -from alembic import command -from alembic.config import Config -from alembic.runtime.migration import MigrationContext -from alembic.script import ScriptDirectory -from sqlalchemy import create_engine - -from app.core.settings import settings +from pathlib import Path +import re +import subprocess +import sys logger = logging.getLogger(__name__) @@ -22,34 +17,33 @@ def _project_root() -> Path: return Path(__file__).resolve().parents[2] -def _build_alembic_config() -> Config: - config = Config(str(_project_root() / "alembic.ini")) - config.set_main_option( - "sqlalchemy.url", _normalize_database_url(settings.DATABASE_URL) +def _run_alembic(*args: str) -> str: + command = [sys.executable, "-m", "alembic", *args] + result = subprocess.run( + command, + cwd=_project_root(), + check=True, + capture_output=True, + text=True, ) - return config - + return result.stdout -def migrate_and_check() -> None: - config = _build_alembic_config() - logger.info("Running Alembic upgrade to head") - command.upgrade(config, "head") +def _extract_revisions(raw_output: str) -> set[str]: + revisions: set[str] = set() + for line in raw_output.splitlines(): + match = re.match(r"^([0-9a-f]+)\b", line.strip()) + if match: + revisions.add(match.group(1)) + return revisions - script = ScriptDirectory.from_config(config) - expected_heads = set(script.get_heads()) - engine = create_engine( - _normalize_database_url(settings.DATABASE_URL), - pool_pre_ping=True, - ) +def migrate_and_check() -> None: + logger.info("Running Alembic upgrade to head") + _run_alembic("upgrade", "head") - try: - with engine.connect() as connection: - context = MigrationContext.configure(connection) - current_heads = set(context.get_current_heads()) - finally: - engine.dispose() + current_heads = _extract_revisions(_run_alembic("current")) + expected_heads = _extract_revisions(_run_alembic("heads")) if current_heads != expected_heads: raise RuntimeError( From 4beb538ccf9c23a1ff90918f198b3ef83e8e104d Mon Sep 17 00:00:00 2001 From: opsmanager1 Date: Sat, 11 Apr 2026 23:25:23 +0300 Subject: [PATCH 22/52] refactor: simplify Alembic command execution and remove unused database URL normalization function --- app/scripts/migrate_and_check.py | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/app/scripts/migrate_and_check.py b/app/scripts/migrate_and_check.py index 561cde0..a32a596 100644 --- a/app/scripts/migrate_and_check.py +++ b/app/scripts/migrate_and_check.py @@ -7,25 +7,18 @@ logger = logging.getLogger(__name__) -def _normalize_database_url(url: str) -> str: - if url.startswith("postgres://"): - return url.replace("postgres://", "postgresql://", 1) - return url - - def _project_root() -> Path: return Path(__file__).resolve().parents[2] def _run_alembic(*args: str) -> str: command = [sys.executable, "-m", "alembic", *args] - result = subprocess.run( - command, - cwd=_project_root(), - check=True, - capture_output=True, - text=True, - ) + result = subprocess.run(command, cwd=_project_root(), capture_output=True, text=True) + if result.returncode != 0: + stderr = (result.stderr or "").strip() + stdout = (result.stdout or "").strip() + details = stderr or stdout or "unknown error" + raise RuntimeError(f"Alembic command failed: {' '.join(args)}; {details}") return result.stdout @@ -45,6 +38,9 @@ def migrate_and_check() -> None: current_heads = _extract_revisions(_run_alembic("current")) expected_heads = _extract_revisions(_run_alembic("heads")) + if not current_heads: + raise RuntimeError("No migration state found in DB after alembic upgrade") + if current_heads != expected_heads: raise RuntimeError( "Alembic DB revision is not at head: " From 96829655075c2495cec5a10f3e5822225bc30d43 Mon Sep 17 00:00:00 2001 From: opsmanager1 Date: Sat, 11 Apr 2026 23:30:19 +0300 Subject: [PATCH 23/52] fix: ensure transaction is committed in version number capacity check --- alembic/env.py | 48 +++++++++++++++--------------------------------- 1 file changed, 15 insertions(+), 33 deletions(-) diff --git a/alembic/env.py b/alembic/env.py index affedde..a97b863 100644 --- a/alembic/env.py +++ b/alembic/env.py @@ -1,15 +1,27 @@ import logging from logging.config import fileConfig +from importlib import import_module +import pkgutil -from sqlalchemy import engine_from_config, pool, text +from sqlalchemy import engine_from_config, pool from alembic import context -from importlib import import_module from app.core.database import Base, _normalize_database_url from app.core.settings import settings -import_module("app.modules.auth.models") + +def _import_all_models() -> None: + modules_pkg = import_module("app.modules") + for module in pkgutil.iter_modules(modules_pkg.__path__, "app.modules."): + model_module_name = f"{module.name}.models" + try: + import_module(model_module_name) + except ModuleNotFoundError: + continue + + +_import_all_models() config = context.config logger = logging.getLogger("alembic.runtime.migration") @@ -22,35 +34,6 @@ target_metadata = Base.metadata -def _ensure_version_num_capacity(connection) -> None: - """Prevent failures when custom revision IDs are longer than 32 chars.""" - result = connection.execute( - text( - """ - SELECT character_maximum_length - FROM information_schema.columns - WHERE table_schema = current_schema() - AND table_name = 'alembic_version' - AND column_name = 'version_num' - """ - ) - ).scalar_one_or_none() - - if result is None: - return - - if result < 255: - logger.warning( - "Expanding alembic_version.version_num from %s to 255 chars for revision-id safety", - result, - ) - connection.execute( - text( - "ALTER TABLE alembic_version ALTER COLUMN version_num TYPE VARCHAR(255)" - ) - ) - - def run_migrations_offline() -> None: url = config.get_main_option("sqlalchemy.url") context.configure( @@ -73,7 +56,6 @@ def run_migrations_online() -> None: ) with connectable.connect() as connection: - _ensure_version_num_capacity(connection) context.configure( connection=connection, target_metadata=target_metadata, From c3b53da45fec0141ddcf6fd444cd4128d89036fe Mon Sep 17 00:00:00 2001 From: opsmanager1 Date: Sat, 11 Apr 2026 23:40:32 +0300 Subject: [PATCH 24/52] feat: add ReDoc documentation endpoint and configure FastAPI initialization --- app/main.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/app/main.py b/app/main.py index ac6f427..8ffbdeb 100644 --- a/app/main.py +++ b/app/main.py @@ -1,4 +1,5 @@ from fastapi import FastAPI +from fastapi.openapi.docs import get_redoc_html from app.api.health import router as health_router from app.api.openapi import configure_openapi_bearer_auth @@ -8,7 +9,12 @@ from app.modules.auth.router import router as auth_router from app.modules.llm.router import router as llm_router -app = FastAPI(title=settings.APP_NAME, version=settings.VERSION, lifespan=lifespan) +app = FastAPI( + title=settings.APP_NAME, + version=settings.VERSION, + lifespan=lifespan, + redoc_url=None, +) configure_security_middlewares(app) @@ -17,3 +23,12 @@ app.include_router(health_router) configure_openapi_bearer_auth(app) + + +@app.get("/redoc", include_in_schema=False) +async def redoc_html(): + return get_redoc_html( + openapi_url=app.openapi_url, + title=f"{app.title} - ReDoc", + redoc_js_url="https://cdn.jsdelivr.net/npm/redoc@2.1.5/bundles/redoc.standalone.js", + ) From 7d5950f3691c2f3eca73a08ffa6bc47594e06bde Mon Sep 17 00:00:00 2001 From: opsmanager1 Date: Sat, 11 Apr 2026 23:41:40 +0300 Subject: [PATCH 25/52] docs: update README with migration command and ReDoc script details --- README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 9f501ba..aae4c45 100644 --- a/README.md +++ b/README.md @@ -109,7 +109,7 @@ docker compose up --build Notes: - Container entrypoint automatically runs: - - alembic upgrade head + - migration + DB head check (`python -m app.scripts.migrate_and_check`) - admin bootstrap script - uvicorn app startup @@ -118,7 +118,7 @@ Notes: When server is running: - Swagger UI: http://localhost:8000/docs -- ReDoc: http://localhost:8000/redoc +- ReDoc: http://localhost:8000/redoc (pinned ReDoc 2.x script) - OpenAPI JSON: http://localhost:8000/openapi.json ## API Endpoints @@ -149,6 +149,7 @@ Login note: - Login response returns `access_token` in body and sets HttpOnly refresh cookie. - Use `/api/v1/auth/jwt/refresh` to get a new access token and refresh cookie. - `/api/v1/auth/jwt/logout` clears refresh cookie on client side. +- Swagger `Authorize` value must contain only raw JWT token (without `Bearer ` prefix). ### LLM From 50f5d9b15d2a6acff9c509db22cd9818ab6ce141 Mon Sep 17 00:00:00 2001 From: opsmanager1 Date: Sat, 11 Apr 2026 23:43:56 +0300 Subject: [PATCH 26/52] style: format code for consistency in main.py --- app/main.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/app/main.py b/app/main.py index 8ffbdeb..9d2df52 100644 --- a/app/main.py +++ b/app/main.py @@ -10,10 +10,10 @@ from app.modules.llm.router import router as llm_router app = FastAPI( - title=settings.APP_NAME, - version=settings.VERSION, - lifespan=lifespan, - redoc_url=None, + title=settings.APP_NAME, + version=settings.VERSION, + lifespan=lifespan, + redoc_url=None, ) configure_security_middlewares(app) @@ -27,8 +27,8 @@ @app.get("/redoc", include_in_schema=False) async def redoc_html(): - return get_redoc_html( - openapi_url=app.openapi_url, - title=f"{app.title} - ReDoc", - redoc_js_url="https://cdn.jsdelivr.net/npm/redoc@2.1.5/bundles/redoc.standalone.js", - ) + return get_redoc_html( + openapi_url=app.openapi_url, + title=f"{app.title} - ReDoc", + redoc_js_url="https://cdn.jsdelivr.net/npm/redoc@2.1.5/bundles/redoc.standalone.js", + ) From 67c1bf2a8adbad29a316acfb9977bdee5c7d3584 Mon Sep 17 00:00:00 2001 From: opsmanager1 Date: Wed, 15 Apr 2026 23:07:51 +0300 Subject: [PATCH 27/52] Refactor LLM module and add feature specification functionality - Removed the OllamaClient and related LLM router, schemas, and service files. - Introduced a new feature specification module with models, orchestrator, and router. - Added prompt template management with bootstrap functionality. - Implemented database migrations for prompt templates and feature spec runs. - Enhanced error handling and logging in the new feature spec generation process. - Updated Docker Compose to include Ollama service and health checks. - Added unit tests for new feature spec functionality and authentication schemas. --- .env.example | 73 ++++----- README.md | 26 +++- .../a1f9c7de42b3_add_prompt_templates.py | 35 +++++ .../b3d1f4a90e2c_add_feature_spec_runs.py | 66 ++++++++ ...f6b1_enforce_single_prompt_template_row.py | 42 +++++ ...ompt_template_column_to_feature_summary.py | 29 ++++ app/core/bootstrap.py | 76 --------- app/core/settings.py | 5 + app/main.py | 4 +- .../infrastructure/fastapi_users_adapter.py | 8 + app/modules/auth/schemas.py | 22 ++- app/modules/feature_spec/__init__.py | 3 + app/modules/feature_spec/models.py | 61 ++++++++ app/modules/feature_spec/orchestrator.py | 104 +++++++++++++ app/modules/feature_spec/parser.py | 18 +++ app/modules/feature_spec/prompts/__init__.py | 13 ++ .../feature_spec/prompts/feature_summary.py | 47 ++++++ .../feature_spec/providers/__init__.py | 3 + app/modules/feature_spec/providers/ollama.py | 131 ++++++++++++++++ app/modules/feature_spec/router.py | 47 ++++++ app/modules/feature_spec/schemas.py | 56 +++++++ app/modules/llm/parser.py | 49 ------ app/modules/llm/providers/ollama.py | 70 --------- app/modules/llm/router.py | 23 --- app/modules/llm/schemas.py | 15 -- app/modules/llm/service.py | 26 ---- app/scripts/bootstrap_admin.py | 68 ++++++++- app/scripts/bootstrap_prompt_template.py | 144 ++++++++++++++++++ docker-compose.yml | 26 ++++ entrypoint.sh | 3 + tests/modules/api/test_llm_api.py | 73 --------- tests/modules/auth/test_auth_schemas.py | 25 +++ tests/modules/llm/test_ollama_provider.py | 100 ------------ 33 files changed, 1016 insertions(+), 475 deletions(-) create mode 100644 alembic/versions/a1f9c7de42b3_add_prompt_templates.py create mode 100644 alembic/versions/b3d1f4a90e2c_add_feature_spec_runs.py create mode 100644 alembic/versions/c8a4e9d2f6b1_enforce_single_prompt_template_row.py create mode 100644 alembic/versions/f1d2c3b4a5e6_rename_prompt_template_column_to_feature_summary.py delete mode 100644 app/core/bootstrap.py create mode 100644 app/modules/feature_spec/__init__.py create mode 100644 app/modules/feature_spec/models.py create mode 100644 app/modules/feature_spec/orchestrator.py create mode 100644 app/modules/feature_spec/parser.py create mode 100644 app/modules/feature_spec/prompts/__init__.py create mode 100644 app/modules/feature_spec/prompts/feature_summary.py create mode 100644 app/modules/feature_spec/providers/__init__.py create mode 100644 app/modules/feature_spec/providers/ollama.py create mode 100644 app/modules/feature_spec/router.py create mode 100644 app/modules/feature_spec/schemas.py delete mode 100644 app/modules/llm/parser.py delete mode 100644 app/modules/llm/providers/ollama.py delete mode 100644 app/modules/llm/router.py delete mode 100644 app/modules/llm/schemas.py delete mode 100644 app/modules/llm/service.py create mode 100644 app/scripts/bootstrap_prompt_template.py delete mode 100644 tests/modules/api/test_llm_api.py create mode 100644 tests/modules/auth/test_auth_schemas.py delete mode 100644 tests/modules/llm/test_ollama_provider.py diff --git a/.env.example b/.env.example index 5b52a4d..57ec2b3 100644 --- a/.env.example +++ b/.env.example @@ -1,51 +1,56 @@ -APP_NAME=Specification Generator -ENV=development -DEBUG=False -VERSION=1.0.0 +APP_NAME= +ENV= +DEBUG= +VERSION= -HOST=0.0.0.0 -PORT=8000 +HOST= +PORT= DATABASE_URL=postgresql://:@:5432/?sslmode=disable -SECRET_KEY= -ACCESS_TOKEN_EXPIRE_MINUTES=60 -ACCESS_TOKEN_MINUTE_IN_SECONDS=60 +SECRET_KEY= +ACCESS_TOKEN_EXPIRE_MINUTES= +ACCESS_TOKEN_MINUTE_IN_SECONDS= -AUTH_PREFIX=/auth -AUTH_ME_PATH=/me -AUTH_TAG=auth -AUTH_BOOTSTRAP_ENABLED=False -AUTH_BOOTSTRAP_SUPERUSER=True +AUTH_PREFIX= +AUTH_ME_PATH= +AUTH_TAG= +AUTH_BOOTSTRAP_ENABLED= +AUTH_BOOTSTRAP_SUPERUSER= AUTH_USERNAME= AUTH_EMAIL= AUTH_PASSWORD= -AUTH_PASSWORD_HASH= +AUTH_PASSWORD_HASH= -ALLOWED_ORIGINS=["*"] -SECURITY_TRUSTED_HOSTS=["*"] -SECURITY_ENABLE_HTTPS_REDIRECT=False -SECURITY_CSP=default-src 'self'; frame-ancestors 'none'; base-uri 'self'; form-action 'self' -SECURITY_REFERRER_POLICY=strict-origin-when-cross-origin -SECURITY_CORS_ALLOW_METHODS=["GET", "POST", "PUT", "PATCH", "DELETE", "OPTIONS"] -SECURITY_CORS_ALLOW_HEADERS=["Authorization", "Content-Type", "Accept", "Origin", "X-Requested-With", "X-Request-ID"] -SECURITY_REQUEST_ID_HEADER=X-Request-ID -SECURITY_LOG_SUSPICIOUS=True -SECURITY_RATE_LIMIT_ENABLED=True -SECURITY_RATE_LIMIT_REQUESTS=120 -SECURITY_RATE_LIMIT_WINDOW_SECONDS=60 -SECURITY_RATE_LIMIT_PATHS=["/api/v1/auth"] +ALLOWED_ORIGINS= +SECURITY_TRUSTED_HOSTS= +SECURITY_ENABLE_HTTPS_REDIRECT= +SECURITY_CSP= +SECURITY_REFERRER_POLICY= +SECURITY_CORS_ALLOW_METHODS= +SECURITY_CORS_ALLOW_HEADERS= +SECURITY_REQUEST_ID_HEADER= +SECURITY_LOG_SUSPICIOUS= +SECURITY_RATE_LIMIT_ENABLED= +SECURITY_RATE_LIMIT_REQUESTS= +SECURITY_RATE_LIMIT_WINDOW_SECONDS= +SECURITY_RATE_LIMIT_PATHS= -OLLAMA_BASE_URL=http://localhost:11434 -OLLAMA_MODEL=mistral -OLLAMA_TIMEOUT=120 -OLLAMA_SYSTEM_PROMPT=You are a helpful assistant that generates software specifications. +OLLAMA_BASE_URL= +OLLAMA_MODEL= +OLLAMA_TIMEOUT= +OLLAMA_CONNECT_TIMEOUT= +OLLAMA_MAX_RETRIES= +OLLAMA_RETRY_BACKOFF_SECONDS= +OLLAMA_SYSTEM_PROMPT= +LLM_PROMPT_MAX_LENGTH= +FEATURE_SPEC_HISTORY_DEFAULT_LIMIT= TEST_DB_HOST= -TEST_DB_PORT=5432 +TEST_DB_PORT= TEST_DB_NAME= TEST_DB_USER= TEST_DB_PASSWORD= -TEST_DB_SSLMODE=disable +TEST_DB_SSLMODE= TEST_DEFAULT_USERNAME= TEST_DEFAULT_EMAIL= TEST_DEFAULT_HASHED_PASSWORD= diff --git a/README.md b/README.md index aae4c45..527a2f3 100644 --- a/README.md +++ b/README.md @@ -60,8 +60,7 @@ Production-ready FastAPI backend for authentication, LLM-powered specification g - Python 3.10+ - PostgreSQL database -- Optional: Docker + Docker Compose -- Optional: local Ollama instance for LLM generation +- Optional: Docker + Docker Compose (recommended for VPS) ### 1) Configure environment @@ -83,6 +82,10 @@ LLM values: - OLLAMA_BASE_URL - OLLAMA_MODEL +For Docker Compose in this project use: + +- OLLAMA_BASE_URL=http://ollama:11434 + ### 2A) Run locally ```bash @@ -97,21 +100,35 @@ pip install -r requirements.txt python -m alembic upgrade head python -m app.scripts.bootstrap_admin +python -m app.scripts.bootstrap_prompt_template python -m uvicorn app.main:app --host 0.0.0.0 --port 8000 ``` ### 2B) Run with Docker ```bash -docker compose up --build +docker compose up -d --build +docker exec -it specification-generator-ollama ollama pull mistral ``` Notes: - Container entrypoint automatically runs: - migration + DB head check (`python -m app.scripts.migrate_and_check`) - - admin bootstrap script + - admin bootstrap script (`python -m app.scripts.bootstrap_admin`) + - prompt template bootstrap script (`python -m app.scripts.bootstrap_prompt_template`) - uvicorn app startup +- FastAPI container reaches Ollama via internal Docker network URL: http://ollama:11434 + +Verify Ollama API: + +```bash +curl http://localhost:11434/api/generate -d '{ + "model": "mistral", + "prompt": "hello", + "stream": false +}' +``` ## API Docs @@ -208,3 +225,4 @@ If LLM requests fail: - Verify OLLAMA_BASE_URL - Ensure Ollama is running and model is available +- For Docker deployment, ensure OLLAMA_BASE_URL is http://ollama:11434 diff --git a/alembic/versions/a1f9c7de42b3_add_prompt_templates.py b/alembic/versions/a1f9c7de42b3_add_prompt_templates.py new file mode 100644 index 0000000..9c5b47c --- /dev/null +++ b/alembic/versions/a1f9c7de42b3_add_prompt_templates.py @@ -0,0 +1,35 @@ +"""add prompt templates + +Revision ID: a1f9c7de42b3 +Revises: 6f4e8b8f4b11 +Create Date: 2026-04-15 12:15:00.000000 +""" + +import sqlalchemy as sa + +from alembic import op + +revision = "a1f9c7de42b3" +down_revision = "6f4e8b8f4b11" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.create_table( + "prompt_templates", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("feature_to_user_stories", sa.Text(), nullable=False), + sa.Column("is_active", sa.Boolean(), nullable=False, server_default=sa.true()), + sa.Column( + "updated_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.PrimaryKeyConstraint("id"), + ) + + +def downgrade() -> None: + op.drop_table("prompt_templates") \ No newline at end of file diff --git a/alembic/versions/b3d1f4a90e2c_add_feature_spec_runs.py b/alembic/versions/b3d1f4a90e2c_add_feature_spec_runs.py new file mode 100644 index 0000000..a9fc2e8 --- /dev/null +++ b/alembic/versions/b3d1f4a90e2c_add_feature_spec_runs.py @@ -0,0 +1,66 @@ +"""add feature spec runs + +Revision ID: b3d1f4a90e2c +Revises: a1f9c7de42b3 +Create Date: 2026-04-15 13:05:00.000000 +""" + +import sqlalchemy as sa + +from alembic import op + +revision = "b3d1f4a90e2c" +down_revision = "a1f9c7de42b3" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.create_table( + "feature_spec_runs", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("user_id", sa.Integer(), nullable=False), + sa.Column("feature_idea", sa.Text(), nullable=False), + sa.Column("status", sa.String(length=32), nullable=False), + sa.Column("response_json", sa.JSON(), nullable=True), + sa.Column("error_message", sa.Text(), nullable=True), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.Column( + "updated_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.ForeignKeyConstraint(["user_id"], ["users.id"], ondelete="CASCADE"), + sa.PrimaryKeyConstraint("id"), + ) + op.create_index( + op.f("ix_feature_spec_runs_user_id"), + "feature_spec_runs", + ["user_id"], + unique=False, + ) + op.create_index( + op.f("ix_feature_spec_runs_status"), + "feature_spec_runs", + ["status"], + unique=False, + ) + op.create_index( + op.f("ix_feature_spec_runs_created_at"), + "feature_spec_runs", + ["created_at"], + unique=False, + ) + + +def downgrade() -> None: + op.drop_index(op.f("ix_feature_spec_runs_created_at"), table_name="feature_spec_runs") + op.drop_index(op.f("ix_feature_spec_runs_status"), table_name="feature_spec_runs") + op.drop_index(op.f("ix_feature_spec_runs_user_id"), table_name="feature_spec_runs") + op.drop_table("feature_spec_runs") diff --git a/alembic/versions/c8a4e9d2f6b1_enforce_single_prompt_template_row.py b/alembic/versions/c8a4e9d2f6b1_enforce_single_prompt_template_row.py new file mode 100644 index 0000000..f3efada --- /dev/null +++ b/alembic/versions/c8a4e9d2f6b1_enforce_single_prompt_template_row.py @@ -0,0 +1,42 @@ +"""enforce single prompt template row + +Revision ID: c8a4e9d2f6b1 +Revises: b3d1f4a90e2c +Create Date: 2026-04-15 13:45:00.000000 +""" + +import sqlalchemy as sa + +from alembic import op + +revision = "c8a4e9d2f6b1" +down_revision = "b3d1f4a90e2c" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + conn = op.get_bind() + + row_count = conn.execute(sa.text("SELECT COUNT(*) FROM prompt_templates")).scalar_one() + if row_count > 1: + raise RuntimeError( + "Cannot enforce single-row constraint: prompt_templates has more than one row" + ) + + if row_count == 1: + conn.execute(sa.text("UPDATE prompt_templates SET id = 1 WHERE id <> 1")) + + op.create_check_constraint( + "ck_prompt_templates_single_row", + "prompt_templates", + "id = 1", + ) + + +def downgrade() -> None: + op.drop_constraint( + "ck_prompt_templates_single_row", + "prompt_templates", + type_="check", + ) diff --git a/alembic/versions/f1d2c3b4a5e6_rename_prompt_template_column_to_feature_summary.py b/alembic/versions/f1d2c3b4a5e6_rename_prompt_template_column_to_feature_summary.py new file mode 100644 index 0000000..e6531e8 --- /dev/null +++ b/alembic/versions/f1d2c3b4a5e6_rename_prompt_template_column_to_feature_summary.py @@ -0,0 +1,29 @@ +"""rename prompt template column to feature summary + +Revision ID: f1d2c3b4a5e6 +Revises: c8a4e9d2f6b1 +Create Date: 2026-04-15 16:30:00.000000 +""" + +from alembic import op + +revision = "f1d2c3b4a5e6" +down_revision = "c8a4e9d2f6b1" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.alter_column( + "prompt_templates", + "feature_to_user_stories", + new_column_name="feature_to_feature_summary", + ) + + +def downgrade() -> None: + op.alter_column( + "prompt_templates", + "feature_to_feature_summary", + new_column_name="feature_to_user_stories", + ) diff --git a/app/core/bootstrap.py b/app/core/bootstrap.py deleted file mode 100644 index 5f00ba4..0000000 --- a/app/core/bootstrap.py +++ /dev/null @@ -1,76 +0,0 @@ -import logging - -from fastapi_users.password import PasswordHelper -from sqlalchemy.orm import Session - -from app.core.settings import settings -from app.modules.auth.models import User - -logger = logging.getLogger(__name__) - - -def _resolve_bootstrap_hashed_password() -> str: - helper = PasswordHelper() - if not settings.AUTH_PASSWORD: - raise RuntimeError( - "AUTH_PASSWORD must be set for admin bootstrap user creation" - ) - return helper.hash(settings.AUTH_PASSWORD) - - -def _is_bootstrap_enabled() -> bool: - if settings.ENV.lower() == "production": - return settings.AUTH_BOOTSTRAP_ENABLED - return settings.AUTH_BOOTSTRAP_ENABLED or bool(settings.AUTH_EMAIL) - - -def bootstrap_auth(db: Session) -> None: - if not _is_bootstrap_enabled(): - logger.info("Auth bootstrap is disabled") - return - - if not settings.AUTH_EMAIL or not settings.AUTH_USERNAME: - raise RuntimeError( - "AUTH_EMAIL and AUTH_USERNAME must be set when bootstrap is enabled" - ) - - existing_user = ( - db.query(User).filter(User.email == settings.AUTH_EMAIL).one_or_none() - ) - if existing_user is not None: - try: - if not existing_user.hashed_password: - existing_user.hashed_password = _resolve_bootstrap_hashed_password() - db.add(existing_user) - db.commit() - except Exception: - logger.exception( - "Failed to ensure bootstrap state for existing user: email=%s", - settings.AUTH_EMAIL, - ) - db.rollback() - raise - return - - try: - user = User( - username=settings.AUTH_USERNAME, - email=settings.AUTH_EMAIL, - hashed_password=_resolve_bootstrap_hashed_password(), - is_active=True, - is_superuser=settings.AUTH_BOOTSTRAP_SUPERUSER, - is_verified=True, - ) - db.add(user) - db.commit() - except Exception: - logger.exception( - "Failed to create bootstrap user: email=%s", - settings.AUTH_EMAIL, - ) - db.rollback() - raise - - -def ensure_default_user(db: Session) -> None: - bootstrap_auth(db) diff --git a/app/core/settings.py b/app/core/settings.py index 98d4676..56e29d6 100644 --- a/app/core/settings.py +++ b/app/core/settings.py @@ -69,12 +69,17 @@ class Settings(BaseSettings): OLLAMA_BASE_URL: str = "http://localhost:11434" OLLAMA_MODEL: str = "mistral" OLLAMA_TIMEOUT: int = 120 + OLLAMA_CONNECT_TIMEOUT: int = 10 + OLLAMA_MAX_RETRIES: int = 2 + OLLAMA_RETRY_BACKOFF_SECONDS: float = 1.0 OLLAMA_SYSTEM_PROMPT: str = ( "You are a helpful assistant that generates software specifications." ) LLM_PREFIX: str = "/llm" LLM_GENERATE_PATH: str = "/generate" LLM_TAG: str = "llm" + LLM_PROMPT_MAX_LENGTH: int = 8000 + FEATURE_SPEC_HISTORY_DEFAULT_LIMIT: int = 10 model_config = SettingsConfigDict( env_file=".env", diff --git a/app/main.py b/app/main.py index 9d2df52..dd4a95f 100644 --- a/app/main.py +++ b/app/main.py @@ -7,7 +7,7 @@ from app.core.startup import lifespan from app.middlewares import configure_security_middlewares from app.modules.auth.router import router as auth_router -from app.modules.llm.router import router as llm_router +from app.modules.feature_spec.router import router as feature_spec_router app = FastAPI( title=settings.APP_NAME, @@ -19,7 +19,7 @@ configure_security_middlewares(app) app.include_router(auth_router, prefix=settings.API_V1_PREFIX) -app.include_router(llm_router, prefix=settings.API_V1_PREFIX) +app.include_router(feature_spec_router, prefix=settings.API_V1_PREFIX) app.include_router(health_router) configure_openapi_bearer_auth(app) diff --git a/app/modules/auth/infrastructure/fastapi_users_adapter.py b/app/modules/auth/infrastructure/fastapi_users_adapter.py index daa8e27..17102f6 100644 --- a/app/modules/auth/infrastructure/fastapi_users_adapter.py +++ b/app/modules/auth/infrastructure/fastapi_users_adapter.py @@ -1,4 +1,5 @@ from collections.abc import AsyncGenerator +from typing import Any from fastapi import Depends from fastapi.security import OAuth2PasswordRequestForm @@ -41,6 +42,13 @@ async def authenticate( return user + async def create(self, user_create: Any, safe: bool = False, request: Any = None): + if safe: + user_create = user_create.model_copy( + update={"is_superuser": False, "is_verified": False} + ) + return await super().create(user_create, safe=safe, request=request) + async def get_user_db( session: AsyncSession = Depends(get_async_db), diff --git a/app/modules/auth/schemas.py b/app/modules/auth/schemas.py index 8c03e69..96a4f36 100644 --- a/app/modules/auth/schemas.py +++ b/app/modules/auth/schemas.py @@ -1,6 +1,6 @@ from fastapi_users import schemas -from pydantic import Field +from pydantic import ConfigDict, Field, model_validator from app.core.settings import settings @@ -14,6 +14,26 @@ class UserCreate(schemas.BaseUserCreate): min_length=settings.AUTH_USERNAME_MIN_LENGTH, max_length=settings.AUTH_USERNAME_MAX_LENGTH, ) + is_superuser: bool = Field(default=False, exclude=True) + + model_config = ConfigDict(extra="forbid") + + @model_validator(mode="after") + def validate_superuser_registration(self): + if self.is_superuser: + raise ValueError("Setting is_superuser via public registration is forbidden") + return self + + @classmethod + def model_json_schema(cls, *args, **kwargs): + schema = super().model_json_schema(*args, **kwargs) + properties = schema.get("properties", {}) + properties.pop("is_superuser", None) + + required = schema.get("required") + if isinstance(required, list) and "is_superuser" in required: + required.remove("is_superuser") + return schema class UserUpdate(schemas.BaseUserUpdate): diff --git a/app/modules/feature_spec/__init__.py b/app/modules/feature_spec/__init__.py new file mode 100644 index 0000000..c57be5d --- /dev/null +++ b/app/modules/feature_spec/__init__.py @@ -0,0 +1,3 @@ +from app.modules.feature_spec.router import router + +__all__ = ["router"] diff --git a/app/modules/feature_spec/models.py b/app/modules/feature_spec/models.py new file mode 100644 index 0000000..b8eeca5 --- /dev/null +++ b/app/modules/feature_spec/models.py @@ -0,0 +1,61 @@ +from datetime import datetime + +from sqlalchemy import ( + JSON, + Boolean, + CheckConstraint, + DateTime, + ForeignKey, + Integer, + String, + Text, + func, +) +from sqlalchemy.orm import Mapped, mapped_column + +from app.core.database import Base + + +class PromptTemplate(Base): + __tablename__ = "prompt_templates" + __table_args__ = ( + CheckConstraint("id = 1", name="ck_prompt_templates_single_row"), + ) + + id: Mapped[int] = mapped_column(Integer, primary_key=True) + feature_to_feature_summary: Mapped[str] = mapped_column(Text, nullable=False) + is_active: Mapped[bool] = mapped_column(Boolean, default=True, nullable=False) + updated_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), + server_default=func.now(), + onupdate=func.now(), + nullable=False, + ) + + +class FeatureSpecRun(Base): + __tablename__ = "feature_spec_runs" + + id: Mapped[int] = mapped_column(Integer, primary_key=True) + user_id: Mapped[int] = mapped_column( + Integer, + ForeignKey("users.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) + feature_idea: Mapped[str] = mapped_column(Text, nullable=False) + status: Mapped[str] = mapped_column(String(32), nullable=False, index=True) + response_json: Mapped[dict | list | None] = mapped_column(JSON, nullable=True) + error_message: Mapped[str | None] = mapped_column(Text, nullable=True) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), + server_default=func.now(), + nullable=False, + index=True, + ) + updated_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), + server_default=func.now(), + onupdate=func.now(), + nullable=False, + ) diff --git a/app/modules/feature_spec/orchestrator.py b/app/modules/feature_spec/orchestrator.py new file mode 100644 index 0000000..79f7f8b --- /dev/null +++ b/app/modules/feature_spec/orchestrator.py @@ -0,0 +1,104 @@ +from sqlalchemy.orm import Session +from sqlalchemy import select + +from app.core.settings import settings +from app.modules.feature_spec.models import FeatureSpecRun +from app.modules.feature_spec.prompts.feature_summary import ( + build_feature_summary_prompt_from_db, + parse_feature_summary_response, +) +from app.modules.feature_spec.providers.ollama import ollama_client +from app.modules.feature_spec.schemas import ( + FeatureSummaryResult, + FeatureSpecGenerateResponse, + FeatureSpecHistoryItem, + FeatureSpecHistoryResponse, +) + + +class FeatureSpecOrchestrator: + def __init__(self, llm_client) -> None: + self.llm = llm_client + + async def generate( + self, + idea: str, + db: Session, + user_id: int, + ) -> FeatureSpecGenerateResponse: + run = FeatureSpecRun( + user_id=user_id, + feature_idea=idea, + status="pending", + ) + db.add(run) + db.commit() + db.refresh(run) + + feature_summary_prompt = build_feature_summary_prompt_from_db(idea, db) + try: + feature_summary_raw = await self.llm.generate(feature_summary_prompt) + feature_summary_json = parse_feature_summary_response(feature_summary_raw) + feature_summary_typed = FeatureSummaryResult.model_validate( + feature_summary_json + ) + + run.status = "success" + run.response_json = feature_summary_typed.model_dump(mode="json") + run.error_message = None + db.add(run) + db.commit() + except Exception as exc: + run.status = "error" + run.error_message = str(exc) + db.add(run) + db.commit() + raise + + return FeatureSpecGenerateResponse( + feature_idea=idea, + feature_summary=feature_summary_typed, + ) + + +orchestrator = FeatureSpecOrchestrator(ollama_client) + + +async def generate_feature_spec( + feature_idea: str, + db: Session, + user_id: int, +) -> FeatureSpecGenerateResponse: + return await orchestrator.generate(feature_idea, db, user_id) + + +def get_feature_spec_history( + db: Session, + user_id: int, + limit: int = settings.FEATURE_SPEC_HISTORY_DEFAULT_LIMIT, +) -> FeatureSpecHistoryResponse: + safe_limit = max(1, min(limit, 100)) + statement = ( + select(FeatureSpecRun) + .where(FeatureSpecRun.user_id == user_id) + .order_by(FeatureSpecRun.created_at.desc()) + .limit(safe_limit) + ) + rows = db.execute(statement).scalars().all() + + items = [ + FeatureSpecHistoryItem( + id=row.id, + feature_idea=row.feature_idea, + status=row.status, + response_json=( + FeatureSummaryResult.model_validate(row.response_json) + if row.response_json is not None + else None + ), + error_message=row.error_message, + created_at=row.created_at, + ) + for row in rows + ] + return FeatureSpecHistoryResponse(items=items) diff --git a/app/modules/feature_spec/parser.py b/app/modules/feature_spec/parser.py new file mode 100644 index 0000000..4f52844 --- /dev/null +++ b/app/modules/feature_spec/parser.py @@ -0,0 +1,18 @@ +import json +import re + + +def extract_json(text: str) -> dict | list: + match = re.search(r"(\{.*\}|\[.*\])", text, re.DOTALL) + if not match: + raise ValueError("No JSON found in LLM response") + return json.loads(match.group(1)) + + +def strip_markdown(text: str) -> str: + text = re.sub(r"```[\w]*\n?", "", text) + return text.strip() + + +def normalize_whitespace(text: str) -> str: + return re.sub(r"\n{3,}", "\n\n", text).strip() diff --git a/app/modules/feature_spec/prompts/__init__.py b/app/modules/feature_spec/prompts/__init__.py new file mode 100644 index 0000000..891cfab --- /dev/null +++ b/app/modules/feature_spec/prompts/__init__.py @@ -0,0 +1,13 @@ +from app.modules.feature_spec.prompts.feature_summary import ( + build_feature_summary_prompt_from_db, + build_feature_summary_prompt_from_template, + load_feature_summary_template, + parse_feature_summary_response, +) + +__all__ = [ + "load_feature_summary_template", + "build_feature_summary_prompt_from_template", + "build_feature_summary_prompt_from_db", + "parse_feature_summary_response", +] diff --git a/app/modules/feature_spec/prompts/feature_summary.py b/app/modules/feature_spec/prompts/feature_summary.py new file mode 100644 index 0000000..d674804 --- /dev/null +++ b/app/modules/feature_spec/prompts/feature_summary.py @@ -0,0 +1,47 @@ +import html + +from sqlalchemy import select +from sqlalchemy.orm import Session + +from app.modules.feature_spec.models import PromptTemplate +from app.modules.feature_spec.parser import extract_json, normalize_whitespace, strip_markdown + + +def build_feature_summary_prompt_from_template(template: str, feature_idea: str) -> str: + safe_template = template.strip() + escaped_feature_idea = html.escape(feature_idea.strip(), quote=True) + try: + return safe_template.format( + feature_idea=escaped_feature_idea, + input=escaped_feature_idea, + ) + except KeyError as exc: + missing_key = exc.args[0] if exc.args else "unknown" + raise ValueError( + f"Prompt template error: missing placeholder '{missing_key}'" + ) from exc + + +def load_feature_summary_template(db: Session) -> str: + statement = ( + select(PromptTemplate.feature_to_feature_summary) + .where(PromptTemplate.is_active.is_(True)) + .order_by(PromptTemplate.updated_at.desc()) + .limit(1) + ) + template = db.execute(statement).scalar_one_or_none() + if template and template.strip(): + return template + raise ValueError( + "Active feature summary prompt template is not configured in database" + ) + + +def build_feature_summary_prompt_from_db(feature_idea: str, db: Session) -> str: + template = load_feature_summary_template(db) + return build_feature_summary_prompt_from_template(template, feature_idea) + + +def parse_feature_summary_response(raw_response: str) -> dict | list: + normalized_content = normalize_whitespace(strip_markdown(raw_response)) + return extract_json(normalized_content) diff --git a/app/modules/feature_spec/providers/__init__.py b/app/modules/feature_spec/providers/__init__.py new file mode 100644 index 0000000..3176b21 --- /dev/null +++ b/app/modules/feature_spec/providers/__init__.py @@ -0,0 +1,3 @@ +from app.modules.feature_spec.providers.ollama import OllamaClient, ollama_client + +__all__ = ["OllamaClient", "ollama_client"] diff --git a/app/modules/feature_spec/providers/ollama.py b/app/modules/feature_spec/providers/ollama.py new file mode 100644 index 0000000..8caa8ca --- /dev/null +++ b/app/modules/feature_spec/providers/ollama.py @@ -0,0 +1,131 @@ +import asyncio +import json +import logging +from typing import Any, AsyncGenerator + +import httpx + +from app.core.settings import settings + +logger = logging.getLogger(__name__) + + +class OllamaClient: + def __init__(self) -> None: + self._base_url = settings.OLLAMA_BASE_URL.rstrip("/") + self._model = settings.OLLAMA_MODEL + self._timeout = settings.OLLAMA_TIMEOUT + self._connect_timeout = settings.OLLAMA_CONNECT_TIMEOUT + self._max_retries = settings.OLLAMA_MAX_RETRIES + self._retry_backoff_seconds = settings.OLLAMA_RETRY_BACKOFF_SECONDS + self._system_prompt = settings.OLLAMA_SYSTEM_PROMPT + + def _build_payload(self, user_prompt: str, stream: bool = False) -> dict: + messages = [] + if self._system_prompt.strip(): + messages.append({"role": "system", "content": self._system_prompt}) + messages.append({"role": "user", "content": user_prompt}) + return {"model": self._model, "stream": stream, "messages": messages} + + def _http_timeout(self) -> httpx.Timeout: + return httpx.Timeout(float(self._timeout), connect=float(self._connect_timeout)) + + def _should_retry_status(self, status_code: int) -> bool: + return 500 <= status_code <= 599 + + async def _backoff(self, attempt: int) -> None: + await asyncio.sleep(self._retry_backoff_seconds * (attempt + 1)) + + async def generate(self, user_prompt: str) -> str: + url = f"{self._base_url}/api/chat" + payload = self._build_payload(user_prompt, stream=False) + + async with httpx.AsyncClient(timeout=self._http_timeout()) as client: + for attempt in range(self._max_retries + 1): + try: + response = await client.post(url, json=payload) + response.raise_for_status() + data = response.json() + message = data.get("message", {}) + content = message.get("content", "") + return content if isinstance(content, str) else str(content) + except httpx.TimeoutException as exc: + if attempt < self._max_retries: + await self._backoff(attempt) + continue + raise RuntimeError( + f"Ollama request timed out after {self._timeout}s" + ) from exc + except httpx.HTTPStatusError as exc: + status_code = exc.response.status_code + if ( + self._should_retry_status(status_code) + and attempt < self._max_retries + ): + await self._backoff(attempt) + continue + raise RuntimeError( + f"Ollama returned HTTP {status_code}: {exc.response.text}" + ) from exc + except httpx.RequestError as exc: + if attempt < self._max_retries: + await self._backoff(attempt) + continue + raise RuntimeError("Ollama request failed") from exc + + raise RuntimeError("Ollama request failed after retries") + + async def generate_stream(self, user_prompt: str) -> AsyncGenerator[str, None]: + url = f"{self._base_url}/api/chat" + payload = self._build_payload(user_prompt, stream=True) + + yielded_any = False + async with httpx.AsyncClient(timeout=self._http_timeout()) as client: + for attempt in range(self._max_retries + 1): + try: + async with client.stream("POST", url, json=payload) as response: + response.raise_for_status() + async for line in response.aiter_lines(): + if not line: + continue + try: + chunk: Any = json.loads(line) + except Exception: + logger.warning("Skipping invalid Ollama stream chunk") + continue + token = chunk.get("message", {}).get("content", "") + if token: + yielded_any = True + yield token + if chunk.get("done"): + return + return + except httpx.TimeoutException as exc: + if not yielded_any and attempt < self._max_retries: + await self._backoff(attempt) + continue + raise RuntimeError( + f"Ollama stream timed out after {self._timeout}s" + ) from exc + except httpx.HTTPStatusError as exc: + status_code = exc.response.status_code + if ( + not yielded_any + and self._should_retry_status(status_code) + and attempt < self._max_retries + ): + await self._backoff(attempt) + continue + raise RuntimeError( + f"Ollama stream returned HTTP {status_code}: {exc.response.text}" + ) from exc + except httpx.RequestError as exc: + if not yielded_any and attempt < self._max_retries: + await self._backoff(attempt) + continue + raise RuntimeError("Ollama stream request failed") from exc + + raise RuntimeError("Ollama stream failed after retries") + + +ollama_client = OllamaClient() diff --git a/app/modules/feature_spec/router.py b/app/modules/feature_spec/router.py new file mode 100644 index 0000000..de4d668 --- /dev/null +++ b/app/modules/feature_spec/router.py @@ -0,0 +1,47 @@ +from fastapi import APIRouter, Depends, HTTPException, status +from sqlalchemy.orm import Session + +from app.core.database import get_db +from app.core.settings import settings +from app.modules.auth.dependencies import get_current_user +from app.modules.auth.models import User +from app.modules.feature_spec.orchestrator import ( + generate_feature_spec, + get_feature_spec_history, +) +from app.modules.feature_spec.schemas import ( + FeatureSpecGenerateRequest, + FeatureSpecGenerateResponse, + FeatureSpecHistoryResponse, +) + +router = APIRouter(prefix="/feature-spec", tags=["feature-spec"]) + + +@router.post("/generate", response_model=FeatureSpecGenerateResponse) +async def generate_feature_spec_endpoint( + payload: FeatureSpecGenerateRequest, + current_user: User = Depends(get_current_user), + db: Session = Depends(get_db), +) -> FeatureSpecGenerateResponse: + try: + return await generate_feature_spec(payload.feature_idea, db, current_user.id) + except ValueError as exc: + raise HTTPException( + status_code=status.HTTP_502_BAD_GATEWAY, + detail=str(exc), + ) from exc + except RuntimeError as exc: + raise HTTPException( + status_code=status.HTTP_502_BAD_GATEWAY, + detail=str(exc), + ) from exc + + +@router.get("/history", response_model=FeatureSpecHistoryResponse) +async def get_feature_spec_history_endpoint( + limit: int = settings.FEATURE_SPEC_HISTORY_DEFAULT_LIMIT, + current_user: User = Depends(get_current_user), + db: Session = Depends(get_db), +) -> FeatureSpecHistoryResponse: + return get_feature_spec_history(db, current_user.id, limit) diff --git a/app/modules/feature_spec/schemas.py b/app/modules/feature_spec/schemas.py new file mode 100644 index 0000000..3072af1 --- /dev/null +++ b/app/modules/feature_spec/schemas.py @@ -0,0 +1,56 @@ +from datetime import datetime +from typing import Literal + +from pydantic import BaseModel, Field, field_validator, model_validator + +from app.core.settings import settings + + +class FeatureSpecGenerateRequest(BaseModel): + feature_idea: str = Field(min_length=1, max_length=settings.LLM_PROMPT_MAX_LENGTH) + + @field_validator("feature_idea") + @classmethod + def validate_feature_idea(cls, value: str) -> str: + normalized = value.strip() + if not normalized: + raise ValueError("Feature idea must not be empty") + return normalized + + +class FeatureSummaryItem(BaseModel): + title: str + as_a: str + i_want: str + so_that: str + + +class FeatureSummaryResult(BaseModel): + feature_summary: str + feature_summary_items: list[FeatureSummaryItem] + + @model_validator(mode="before") + @classmethod + def normalize_legacy_user_stories(cls, data): + if isinstance(data, dict) and "feature_summary_items" not in data: + if "user_stories" in data: + data = {**data, "feature_summary_items": data["user_stories"]} + return data + + +class FeatureSpecGenerateResponse(BaseModel): + feature_idea: str + feature_summary: FeatureSummaryResult + + +class FeatureSpecHistoryItem(BaseModel): + id: int + feature_idea: str + status: Literal["pending", "success", "error"] + response_json: FeatureSummaryResult | None = None + error_message: str | None = None + created_at: datetime + + +class FeatureSpecHistoryResponse(BaseModel): + items: list[FeatureSpecHistoryItem] diff --git a/app/modules/llm/parser.py b/app/modules/llm/parser.py deleted file mode 100644 index 4d71793..0000000 --- a/app/modules/llm/parser.py +++ /dev/null @@ -1,49 +0,0 @@ -import json -import re - - -def extract_json(text: str) -> dict | list: - """Extract the first JSON object or array from a raw LLM response.""" - match = re.search(r"(\{.*\}|\[.*\])", text, re.DOTALL) - if not match: - raise ValueError("No JSON found in LLM response") - return json.loads(match.group(1)) - - -def strip_markdown(text: str) -> str: - """Remove Markdown code fences and trim whitespace.""" - text = re.sub(r"```[\w]*\n?", "", text) - return text.strip() - - -def extract_sections(text: str) -> dict[str, str]: - """ - Split a Markdown-formatted LLM response into {heading: content} sections. - - Example input: - ## Overview - This is a spec... - ## Requirements - 1. ... - """ - sections: dict[str, str] = {} - current_heading = "_preamble" - current_lines: list[str] = [] - - for line in text.splitlines(): - heading_match = re.match(r"^#{1,3}\s+(.+)", line) - if heading_match: - sections[current_heading] = "\n".join(current_lines).strip() - current_heading = heading_match.group(1).strip() - current_lines = [] - else: - current_lines.append(line) - - sections[current_heading] = "\n".join(current_lines).strip() - sections.pop("_preamble", None) - return sections - - -def normalize_whitespace(text: str) -> str: - """Collapse multiple blank lines to a single blank line.""" - return re.sub(r"\n{3,}", "\n\n", text).strip() diff --git a/app/modules/llm/providers/ollama.py b/app/modules/llm/providers/ollama.py deleted file mode 100644 index 743c6d4..0000000 --- a/app/modules/llm/providers/ollama.py +++ /dev/null @@ -1,70 +0,0 @@ -import logging -from typing import AsyncGenerator - -import httpx - -from app.core.settings import settings - -logger = logging.getLogger(__name__) - - -class OllamaClient: - def __init__(self) -> None: - self._base_url = settings.OLLAMA_BASE_URL.rstrip("/") - self._model = settings.OLLAMA_MODEL - self._timeout = settings.OLLAMA_TIMEOUT - self._system_prompt = settings.OLLAMA_SYSTEM_PROMPT - - def _build_payload(self, user_prompt: str, stream: bool = False) -> dict: - return { - "model": self._model, - "stream": stream, - "messages": [ - {"role": "system", "content": self._system_prompt}, - {"role": "user", "content": user_prompt}, - ], - } - - async def generate(self, user_prompt: str) -> str: - url = f"{self._base_url}/api/chat" - payload = self._build_payload(user_prompt, stream=False) - - async with httpx.AsyncClient(timeout=self._timeout) as client: - try: - response = await client.post(url, json=payload) - response.raise_for_status() - except httpx.TimeoutException as exc: - logger.exception("Ollama request timed out") - raise RuntimeError( - f"Ollama request timed out after {self._timeout}s" - ) from exc - except httpx.HTTPStatusError as exc: - logger.exception("Ollama returned error response") - raise RuntimeError( - f"Ollama returned HTTP {exc.response.status_code}: {exc.response.text}" - ) from exc - - data = response.json() - return data["message"]["content"] - - async def generate_stream(self, user_prompt: str) -> AsyncGenerator[str, None]: - url = f"{self._base_url}/api/chat" - payload = self._build_payload(user_prompt, stream=True) - - async with httpx.AsyncClient(timeout=self._timeout) as client: - async with client.stream("POST", url, json=payload) as response: - response.raise_for_status() - async for line in response.aiter_lines(): - if not line: - continue - import json as _json - - chunk = _json.loads(line) - token = chunk.get("message", {}).get("content", "") - if token: - yield token - if chunk.get("done"): - break - - -ollama_client = OllamaClient() diff --git a/app/modules/llm/router.py b/app/modules/llm/router.py deleted file mode 100644 index e868011..0000000 --- a/app/modules/llm/router.py +++ /dev/null @@ -1,23 +0,0 @@ -from fastapi import APIRouter, HTTPException, status - -from app.core.settings import settings -from app.modules.llm.schemas import LlmGenerateRequest, LlmGenerateResponse -from app.modules.llm.service import generate_completion - -router = APIRouter(prefix=settings.LLM_PREFIX, tags=[settings.LLM_TAG]) - - -@router.post(settings.LLM_GENERATE_PATH, response_model=LlmGenerateResponse) -async def generate(payload: LlmGenerateRequest) -> LlmGenerateResponse: - try: - return await generate_completion(payload) - except ValueError as exc: - raise HTTPException( - status_code=status.HTTP_502_BAD_GATEWAY, - detail=str(exc), - ) from exc - except RuntimeError as exc: - raise HTTPException( - status_code=status.HTTP_502_BAD_GATEWAY, - detail=str(exc), - ) from exc diff --git a/app/modules/llm/schemas.py b/app/modules/llm/schemas.py deleted file mode 100644 index 0d31e9f..0000000 --- a/app/modules/llm/schemas.py +++ /dev/null @@ -1,15 +0,0 @@ -from typing import Literal - -from pydantic import BaseModel, Field - - -class LlmGenerateRequest(BaseModel): - prompt: str = Field(min_length=1) - response_format: Literal["text", "sections", "json"] = "text" - - -class LlmGenerateResponse(BaseModel): - raw_content: str - content: str | None = None - sections: dict[str, str] | None = None - data: dict | list | None = None diff --git a/app/modules/llm/service.py b/app/modules/llm/service.py deleted file mode 100644 index e8c347b..0000000 --- a/app/modules/llm/service.py +++ /dev/null @@ -1,26 +0,0 @@ -from app.modules.llm.parser import (extract_json, extract_sections, - normalize_whitespace, strip_markdown) -from app.modules.llm.providers.ollama import ollama_client -from app.modules.llm.schemas import LlmGenerateRequest, LlmGenerateResponse - - -async def generate_completion(payload: LlmGenerateRequest) -> LlmGenerateResponse: - raw_content = await ollama_client.generate(payload.prompt) - normalized_content = normalize_whitespace(strip_markdown(raw_content)) - - if payload.response_format == "json": - return LlmGenerateResponse( - raw_content=raw_content, - data=extract_json(normalized_content), - ) - - if payload.response_format == "sections": - return LlmGenerateResponse( - raw_content=raw_content, - sections=extract_sections(normalized_content), - ) - - return LlmGenerateResponse( - raw_content=raw_content, - content=normalized_content, - ) diff --git a/app/scripts/bootstrap_admin.py b/app/scripts/bootstrap_admin.py index 9c36452..785782d 100644 --- a/app/scripts/bootstrap_admin.py +++ b/app/scripts/bootstrap_admin.py @@ -1,15 +1,79 @@ import logging -from app.core.bootstrap import bootstrap_auth +from fastapi_users.password import PasswordHelper + from app.core.database import SessionLocal +from app.core.settings import settings +from app.modules.auth.models import User logger = logging.getLogger(__name__) +def _resolve_bootstrap_hashed_password() -> str: + helper = PasswordHelper() + if not settings.AUTH_PASSWORD: + raise RuntimeError( + "AUTH_PASSWORD must be set for admin bootstrap user creation" + ) + return helper.hash(settings.AUTH_PASSWORD) + + +def _is_bootstrap_enabled() -> bool: + if settings.ENV.lower() == "production": + return settings.AUTH_BOOTSTRAP_ENABLED + return settings.AUTH_BOOTSTRAP_ENABLED or bool(settings.AUTH_EMAIL) + + +def bootstrap_admin_user(db) -> None: + if not _is_bootstrap_enabled(): + logger.info("Auth bootstrap is disabled") + return + + if not settings.AUTH_EMAIL or not settings.AUTH_USERNAME: + raise RuntimeError( + "AUTH_EMAIL and AUTH_USERNAME must be set when bootstrap is enabled" + ) + + existing_user = db.query(User).filter(User.email == settings.AUTH_EMAIL).one_or_none() + if existing_user is not None: + try: + if not existing_user.hashed_password: + existing_user.hashed_password = _resolve_bootstrap_hashed_password() + db.add(existing_user) + db.commit() + except Exception: + logger.exception( + "Failed to ensure bootstrap state for existing user: email=%s", + settings.AUTH_EMAIL, + ) + db.rollback() + raise + return + + try: + user = User( + username=settings.AUTH_USERNAME, + email=settings.AUTH_EMAIL, + hashed_password=_resolve_bootstrap_hashed_password(), + is_active=True, + is_superuser=settings.AUTH_BOOTSTRAP_SUPERUSER, + is_verified=True, + ) + db.add(user) + db.commit() + except Exception: + logger.exception( + "Failed to create bootstrap user: email=%s", + settings.AUTH_EMAIL, + ) + db.rollback() + raise + + def main() -> None: db = SessionLocal() try: - bootstrap_auth(db) + bootstrap_admin_user(db) logger.info("Admin bootstrap script completed") finally: db.close() diff --git a/app/scripts/bootstrap_prompt_template.py b/app/scripts/bootstrap_prompt_template.py new file mode 100644 index 0000000..d7d7358 --- /dev/null +++ b/app/scripts/bootstrap_prompt_template.py @@ -0,0 +1,144 @@ +import logging + +from app.core.database import SessionLocal +from app.modules.feature_spec.models import PromptTemplate + +logger = logging.getLogger(__name__) + + +DEFAULT_FEATURE_SUMMARY_PROMPT_TEMPLATE = """You are a senior staff-level product engineer +and system architect working on production-grade software systems. + +Your job is to convert a high-level feature idea into a precise, +implementation-ready technical specification. + +You must think in terms of real software delivery: backend systems, APIs, +databases, edge cases, and scalability. + +--- + +## Core rules: +- Be extremely structured and deterministic +- Do NOT invent product requirements that were not implied or provided +- If information is missing, explicitly list assumptions in a separate section +- Prefer simplicity over over-engineering +- Think like you are writing a specification for a real engineering team +- All outputs must be actionable and implementation-oriented + +--- + +## Output format (strict structure): + +1. FEATURE OVERVIEW +- Short description of the feature +- Problem it solves +- Target users + +--- + +2. USER STORIES +- List of user stories in format: + As a [user type], I want [goal], so that [benefit] + +--- + +3. ACCEPTANCE CRITERIA +- Each user story must have clear, testable conditions +- Use bullet points +- Must be unambiguous and QA-ready + +--- + +4. API DESIGN (REST) +- List endpoints with: + - method (GET, POST, etc.) + - path + - request body (if applicable) + - response schema (high-level) + - purpose + +--- + +5. DATABASE DESIGN +- Define tables with: + - table name + - fields with types + - relationships +- Keep it normalized and production-oriented + +--- + +6. EDGE CASES & RISKS +- Identify failure scenarios +- Data consistency risks +- Security concerns +- Performance concerns + +--- + +7. FUTURE IMPROVEMENTS +- Optional enhancements +- Scalability improvements +- UX improvements + +--- + +## Important constraints: +- Do not add irrelevant features or expand scope +- Keep all assumptions explicitly labeled as "Assumption" +- Prefer backend clarity over marketing language +- Avoid vague phrases like "user-friendly", "robust", "fast" +- Be precise, technical, and engineering-focused + +Feature idea: +{feature_idea} +""" + + +def bootstrap_prompt_template(db) -> None: + existing_template = db.get(PromptTemplate, 1) + + if existing_template is None: + try: + db.add( + PromptTemplate( + id=1, + feature_to_feature_summary=DEFAULT_FEATURE_SUMMARY_PROMPT_TEMPLATE, + is_active=True, + ) + ) + db.commit() + logger.info("Created default feature summary prompt template") + except Exception: + logger.exception("Failed to create default feature summary prompt template") + db.rollback() + raise + return + + if existing_template.feature_to_feature_summary.strip(): + return + + try: + existing_template.feature_to_feature_summary = ( + DEFAULT_FEATURE_SUMMARY_PROMPT_TEMPLATE + ) + db.add(existing_template) + db.commit() + logger.info("Filled empty feature summary prompt template with default value") + except Exception: + logger.exception("Failed to fill default feature summary prompt template") + db.rollback() + raise + + +def main() -> None: + db = SessionLocal() + try: + bootstrap_prompt_template(db) + logger.info("Prompt template bootstrap script completed") + finally: + db.close() + + +if __name__ == "__main__": + main() diff --git a/docker-compose.yml b/docker-compose.yml index 066f6d5..b72b652 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -9,6 +9,8 @@ services: - .env ports: - "8001:8001" + depends_on: + - ollama healthcheck: test: [ @@ -23,3 +25,27 @@ services: start_period: 20s stop_grace_period: 20s restart: unless-stopped + + ollama: + image: ollama/ollama:latest + container_name: specification-generator-ollama + init: true + ports: + - "11434:11434" + volumes: + - ollama:/root/.ollama + healthcheck: + test: + [ + "CMD", + "ollama", + "list", + ] + interval: 30s + timeout: 10s + retries: 5 + start_period: 30s + restart: unless-stopped + +volumes: + ollama: diff --git a/entrypoint.sh b/entrypoint.sh index 2116919..23bda5d 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -7,6 +7,9 @@ python -m app.scripts.migrate_and_check echo "[start] Running admin bootstrap script..." python -m app.scripts.bootstrap_admin +echo "[start] Running prompt template bootstrap script..." +python -m app.scripts.bootstrap_prompt_template + if [ "$#" -eq 0 ]; then set -- python -m uvicorn app.main:app --host 0.0.0.0 --port 8001 fi diff --git a/tests/modules/api/test_llm_api.py b/tests/modules/api/test_llm_api.py deleted file mode 100644 index 25dbf58..0000000 --- a/tests/modules/api/test_llm_api.py +++ /dev/null @@ -1,73 +0,0 @@ -from unittest.mock import AsyncMock - -import pytest -from fastapi.testclient import TestClient - - -@pytest.mark.unit -def test_generate_returns_service_result( - monkeypatch: pytest.MonkeyPatch, api_client: TestClient -) -> None: - mocked_generate = AsyncMock( - return_value={ - "raw_content": "# Spec", - "content": "Spec", - "sections": None, - "data": None, - } - ) - monkeypatch.setattr("app.modules.llm.router.generate_completion", mocked_generate) - - response = api_client.post( - "/api/v1/llm/generate", - json={"prompt": "Generate auth spec", "response_format": "text"}, - ) - - assert response.status_code == 200 - assert response.json()["content"] == "Spec" - - -@pytest.mark.unit -def test_generate_maps_value_error_to_502( - monkeypatch: pytest.MonkeyPatch, api_client: TestClient -) -> None: - monkeypatch.setattr( - "app.modules.llm.router.generate_completion", - AsyncMock(side_effect=ValueError("invalid provider response")), - ) - - response = api_client.post( - "/api/v1/llm/generate", - json={"prompt": "Generate auth spec", "response_format": "text"}, - ) - - assert response.status_code == 502 - assert response.json()["detail"] == "invalid provider response" - - -@pytest.mark.unit -def test_generate_maps_runtime_error_to_502( - monkeypatch: pytest.MonkeyPatch, api_client: TestClient -) -> None: - monkeypatch.setattr( - "app.modules.llm.router.generate_completion", - AsyncMock(side_effect=RuntimeError("provider timeout")), - ) - - response = api_client.post( - "/api/v1/llm/generate", - json={"prompt": "Generate auth spec", "response_format": "sections"}, - ) - - assert response.status_code == 502 - assert response.json()["detail"] == "provider timeout" - - -@pytest.mark.unit -def test_generate_validates_request_payload(api_client: TestClient) -> None: - response = api_client.post( - "/api/v1/llm/generate", - json={"prompt": "", "response_format": "text"}, - ) - - assert response.status_code == 422 diff --git a/tests/modules/auth/test_auth_schemas.py b/tests/modules/auth/test_auth_schemas.py new file mode 100644 index 0000000..3207c9b --- /dev/null +++ b/tests/modules/auth/test_auth_schemas.py @@ -0,0 +1,25 @@ +import pytest + +from app.modules.auth.schemas import UserCreate + + +@pytest.mark.unit +def test_user_create_rejects_superuser_flag() -> None: + with pytest.raises(ValueError): + UserCreate( + email="user@example.com", + password="StrongPass123!", + username="user", + is_superuser=True, + ) + + +@pytest.mark.unit +def test_user_create_defaults_to_non_superuser() -> None: + user = UserCreate( + email="user@example.com", + password="StrongPass123!", + username="user", + ) + + assert user.is_superuser is False diff --git a/tests/modules/llm/test_ollama_provider.py b/tests/modules/llm/test_ollama_provider.py deleted file mode 100644 index 98b7904..0000000 --- a/tests/modules/llm/test_ollama_provider.py +++ /dev/null @@ -1,100 +0,0 @@ -import httpx -import pytest - -from app.modules.llm.providers.ollama import OllamaClient - - -class FakeResponse: - def __init__(self, *, json_data=None, status_code: int = 200, text: str = ""): - self._json_data = json_data or {} - self.status_code = status_code - self.text = text - - def raise_for_status(self) -> None: - if self.status_code >= 400: - request = httpx.Request("POST", "http://mock/api/chat") - response = httpx.Response(self.status_code, request=request, text=self.text) - raise httpx.HTTPStatusError("error", request=request, response=response) - - def json(self): - return self._json_data - - -class FakeAsyncClient: - def __init__( - self, response: FakeResponse | None = None, error: Exception | None = None - ): - self._response = response - self._error = error - self.last_url = None - self.last_json = None - - async def __aenter__(self): - return self - - async def __aexit__(self, exc_type, exc, tb): - return False - - async def post(self, url, json): - self.last_url = url - self.last_json = json - if self._error: - raise self._error - return self._response - - -@pytest.mark.unit -@pytest.mark.asyncio -async def test_generate_returns_content(monkeypatch: pytest.MonkeyPatch) -> None: - fake_client = FakeAsyncClient( - response=FakeResponse( - json_data={"message": {"content": "Generated specification"}} - ) - ) - monkeypatch.setattr( - "app.modules.llm.providers.ollama.httpx.AsyncClient", - lambda *args, **kwargs: fake_client, - ) - - client = OllamaClient() - result = await client.generate("Build API spec") - - assert result == "Generated specification" - assert fake_client.last_url.endswith("/api/chat") - assert fake_client.last_json["messages"][1]["content"] == "Build API spec" - - -@pytest.mark.unit -@pytest.mark.asyncio -async def test_generate_maps_timeout_to_runtime_error( - monkeypatch: pytest.MonkeyPatch, -) -> None: - fake_client = FakeAsyncClient(error=httpx.TimeoutException("timeout")) - monkeypatch.setattr( - "app.modules.llm.providers.ollama.httpx.AsyncClient", - lambda *args, **kwargs: fake_client, - ) - - client = OllamaClient() - - with pytest.raises(RuntimeError, match="timed out"): - await client.generate("Build API spec") - - -@pytest.mark.unit -@pytest.mark.asyncio -async def test_generate_maps_http_status_error_to_runtime_error( - monkeypatch: pytest.MonkeyPatch, -) -> None: - fake_client = FakeAsyncClient( - response=FakeResponse(status_code=500, text="internal error") - ) - monkeypatch.setattr( - "app.modules.llm.providers.ollama.httpx.AsyncClient", - lambda *args, **kwargs: fake_client, - ) - - client = OllamaClient() - - with pytest.raises(RuntimeError, match="Ollama returned HTTP 500"): - await client.generate("Build API spec") From 0ddf4b5bc0501ce949a13eac10d527b41d4e152e Mon Sep 17 00:00:00 2001 From: opsmanager1 Date: Wed, 15 Apr 2026 23:23:25 +0300 Subject: [PATCH 28/52] feat: add Ollama model bootstrap script and update Docker Compose health check --- README.md | 3 +- app/scripts/ensure_ollama_model.py | 84 ++++++++++++++++++++++++++++++ docker-compose.yml | 3 +- entrypoint.sh | 3 ++ 4 files changed, 91 insertions(+), 2 deletions(-) create mode 100644 app/scripts/ensure_ollama_model.py diff --git a/README.md b/README.md index 527a2f3..fd6b720 100644 --- a/README.md +++ b/README.md @@ -108,7 +108,6 @@ python -m uvicorn app.main:app --host 0.0.0.0 --port 8000 ```bash docker compose up -d --build -docker exec -it specification-generator-ollama ollama pull mistral ``` Notes: @@ -117,7 +116,9 @@ Notes: - migration + DB head check (`python -m app.scripts.migrate_and_check`) - admin bootstrap script (`python -m app.scripts.bootstrap_admin`) - prompt template bootstrap script (`python -m app.scripts.bootstrap_prompt_template`) + - Ollama model bootstrap (`python -m app.scripts.ensure_ollama_model`) - uvicorn app startup +- On first deploy, startup may take longer while the configured `OLLAMA_MODEL` is downloaded. - FastAPI container reaches Ollama via internal Docker network URL: http://ollama:11434 Verify Ollama API: diff --git a/app/scripts/ensure_ollama_model.py b/app/scripts/ensure_ollama_model.py new file mode 100644 index 0000000..42b3651 --- /dev/null +++ b/app/scripts/ensure_ollama_model.py @@ -0,0 +1,84 @@ +import logging +import os +import time + +import httpx + +logger = logging.getLogger(__name__) + + +def _env_int(name: str, default: int) -> int: + raw_value = os.getenv(name) + if raw_value is None: + return default + try: + return int(raw_value) + except ValueError as exc: + raise RuntimeError(f"{name} must be an integer") from exc + + +def _is_model_available(client: httpx.Client, model: str) -> bool: + response = client.get("/api/tags") + response.raise_for_status() + models = response.json().get("models", []) + return any(item.get("name") == model for item in models if isinstance(item, dict)) + + +def wait_for_ollama(client: httpx.Client, wait_seconds: int) -> None: + deadline = time.monotonic() + wait_seconds + while time.monotonic() < deadline: + try: + client.get("/api/tags").raise_for_status() + logger.info("Ollama is ready") + return + except httpx.HTTPError: + logger.info("Waiting for Ollama to become ready...") + time.sleep(2) + + raise RuntimeError(f"Ollama did not become ready within {wait_seconds}s") + + +def ensure_model(client: httpx.Client, model: str, pull_timeout: int) -> None: + if _is_model_available(client, model): + logger.info("Ollama model '%s' already available", model) + return + + logger.info("Pulling missing Ollama model '%s'", model) + response = client.post( + "/api/pull", + json={"model": model, "stream": False}, + timeout=pull_timeout, + ) + response.raise_for_status() + + if not _is_model_available(client, model): + raise RuntimeError(f"Ollama model '{model}' is still unavailable after pull") + + logger.info("Ollama model '%s' is ready", model) + + +def ensure_ollama_model() -> None: + base_url = os.getenv("OLLAMA_BASE_URL", "http://localhost:11434").rstrip("/") + model = os.getenv("OLLAMA_MODEL", "mistral").strip() + connect_timeout = _env_int("OLLAMA_CONNECT_TIMEOUT", 10) + request_timeout = _env_int("OLLAMA_TIMEOUT", 120) + startup_wait = _env_int("OLLAMA_STARTUP_WAIT_SECONDS", 120) + + if not model: + raise RuntimeError("OLLAMA_MODEL must not be empty") + + logger.info("Checking Ollama availability at %s", base_url) + + timeout = httpx.Timeout(timeout=request_timeout, connect=connect_timeout) + with httpx.Client(base_url=base_url, timeout=timeout) as client: + wait_for_ollama(client, startup_wait) + ensure_model(client, model, pull_timeout=max(request_timeout, 300)) + + +def main() -> None: + logging.basicConfig(level=logging.INFO, format="[ollama-bootstrap] %(message)s") + ensure_ollama_model() + + +if __name__ == "__main__": + main() diff --git a/docker-compose.yml b/docker-compose.yml index b72b652..906423e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -10,7 +10,8 @@ services: ports: - "8001:8001" depends_on: - - ollama + ollama: + condition: service_healthy healthcheck: test: [ diff --git a/entrypoint.sh b/entrypoint.sh index 23bda5d..b6729c1 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -10,6 +10,9 @@ python -m app.scripts.bootstrap_admin echo "[start] Running prompt template bootstrap script..." python -m app.scripts.bootstrap_prompt_template +echo "[start] Ensuring Ollama model is available..." +python -m app.scripts.ensure_ollama_model + if [ "$#" -eq 0 ]; then set -- python -m uvicorn app.main:app --host 0.0.0.0 --port 8001 fi From 7bfcd45eb2ced3aebea7f338c5634999c930daf9 Mon Sep 17 00:00:00 2001 From: opsmanager1 Date: Wed, 15 Apr 2026 23:40:44 +0300 Subject: [PATCH 29/52] feat: enhance JSON extraction and update feature summary schema with new fields --- app/modules/feature_spec/parser.py | 15 ++++++--- app/modules/feature_spec/schemas.py | 39 ++++++++++++++++++++---- app/scripts/bootstrap_prompt_template.py | 33 +++++++++++++++++--- 3 files changed, 72 insertions(+), 15 deletions(-) diff --git a/app/modules/feature_spec/parser.py b/app/modules/feature_spec/parser.py index 4f52844..de7a156 100644 --- a/app/modules/feature_spec/parser.py +++ b/app/modules/feature_spec/parser.py @@ -3,10 +3,17 @@ def extract_json(text: str) -> dict | list: - match = re.search(r"(\{.*\}|\[.*\])", text, re.DOTALL) - if not match: - raise ValueError("No JSON found in LLM response") - return json.loads(match.group(1)) + decoder = json.JSONDecoder() + for index, char in enumerate(text): + if char not in "[{": + continue + try: + parsed, _ = decoder.raw_decode(text[index:]) + except json.JSONDecodeError: + continue + if isinstance(parsed, (dict, list)): + return parsed + raise ValueError("No JSON found in LLM response") def strip_markdown(text: str) -> str: diff --git a/app/modules/feature_spec/schemas.py b/app/modules/feature_spec/schemas.py index 3072af1..6021b0f 100644 --- a/app/modules/feature_spec/schemas.py +++ b/app/modules/feature_spec/schemas.py @@ -25,17 +25,44 @@ class FeatureSummaryItem(BaseModel): so_that: str +class DbModelsAndApiEndpoints(BaseModel): + db_models: list[str] + api_endpoints: list[str] + + class FeatureSummaryResult(BaseModel): - feature_summary: str - feature_summary_items: list[FeatureSummaryItem] + user_stories: list[FeatureSummaryItem] + acceptance_criteria: list[str] + db_models_and_api_endpoints: DbModelsAndApiEndpoints + risk_assessment: list[str] @model_validator(mode="before") @classmethod def normalize_legacy_user_stories(cls, data): - if isinstance(data, dict) and "feature_summary_items" not in data: - if "user_stories" in data: - data = {**data, "feature_summary_items": data["user_stories"]} - return data + if not isinstance(data, dict): + return data + + normalized = dict(data) + + if "user_stories" not in normalized and "feature_summary_items" in normalized: + normalized["user_stories"] = normalized["feature_summary_items"] + + if "acceptance_criteria" not in normalized: + if "acceptance" in normalized: + normalized["acceptance_criteria"] = normalized["acceptance"] + + if "db_models_and_api_endpoints" not in normalized: + db_models = normalized.get("db_models", []) + api_endpoints = normalized.get("api_endpoints", []) + normalized["db_models_and_api_endpoints"] = { + "db_models": db_models, + "api_endpoints": api_endpoints, + } + + if "risk_assessment" not in normalized and "risks" in normalized: + normalized["risk_assessment"] = normalized["risks"] + + return normalized class FeatureSpecGenerateResponse(BaseModel): diff --git a/app/scripts/bootstrap_prompt_template.py b/app/scripts/bootstrap_prompt_template.py index d7d7358..defca5e 100644 --- a/app/scripts/bootstrap_prompt_template.py +++ b/app/scripts/bootstrap_prompt_template.py @@ -92,18 +92,42 @@ Feature idea: {feature_idea} + +--- + +Return format requirements (strict): +- Return ONLY valid JSON. +- Do not include markdown, code fences, or any text before/after JSON. +- JSON object must match this structure exactly: +{ + "user_stories": [ + { + "title": "string", + "as_a": "string", + "i_want": "string", + "so_that": "string" + } + ], + "acceptance_criteria": ["string"], + "db_models_and_api_endpoints": { + "db_models": ["string"], + "api_endpoints": ["string"] + }, + "risk_assessment": ["string"] +} """ def bootstrap_prompt_template(db) -> None: existing_template = db.get(PromptTemplate, 1) + default_template = DEFAULT_FEATURE_SUMMARY_PROMPT_TEMPLATE if existing_template is None: try: db.add( PromptTemplate( id=1, - feature_to_feature_summary=DEFAULT_FEATURE_SUMMARY_PROMPT_TEMPLATE, + feature_to_feature_summary=default_template, is_active=True, ) ) @@ -115,13 +139,12 @@ def bootstrap_prompt_template(db) -> None: raise return - if existing_template.feature_to_feature_summary.strip(): + existing_value = existing_template.feature_to_feature_summary.strip() + if existing_value: return try: - existing_template.feature_to_feature_summary = ( - DEFAULT_FEATURE_SUMMARY_PROMPT_TEMPLATE - ) + existing_template.feature_to_feature_summary = default_template db.add(existing_template) db.commit() logger.info("Filled empty feature summary prompt template with default value") From c3793f807114e6c8207124c0c01792e02c25883b Mon Sep 17 00:00:00 2001 From: opsmanager1 Date: Wed, 15 Apr 2026 23:50:18 +0300 Subject: [PATCH 30/52] feat: include system prompt in payload for OllamaClient --- app/modules/feature_spec/providers/ollama.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/app/modules/feature_spec/providers/ollama.py b/app/modules/feature_spec/providers/ollama.py index 8caa8ca..08ff0d3 100644 --- a/app/modules/feature_spec/providers/ollama.py +++ b/app/modules/feature_spec/providers/ollama.py @@ -25,7 +25,12 @@ def _build_payload(self, user_prompt: str, stream: bool = False) -> dict: if self._system_prompt.strip(): messages.append({"role": "system", "content": self._system_prompt}) messages.append({"role": "user", "content": user_prompt}) - return {"model": self._model, "stream": stream, "messages": messages} + return { + "model": self._model, + "stream": stream, + "format": "json", + "messages": messages, + } def _http_timeout(self) -> httpx.Timeout: return httpx.Timeout(float(self._timeout), connect=float(self._connect_timeout)) From cb3b317a7af3f36a9c1362c96f18e3d76d0fc6c1 Mon Sep 17 00:00:00 2001 From: opsmanager1 Date: Wed, 15 Apr 2026 23:56:05 +0300 Subject: [PATCH 31/52] feat: remove obsolete migration scripts for database schema updates --- alembic/versions/2e5bd3a1eb88_initial.py | 45 ------------- .../6f4e8b8f4b11_add_refresh_tokens.py | 54 --------------- .../a1f9c7de42b3_add_prompt_templates.py | 35 ---------- .../b3d1f4a90e2c_add_feature_spec_runs.py | 66 ------------------- ...f6b1_enforce_single_prompt_template_row.py | 42 ------------ ...ompt_template_column_to_feature_summary.py | 29 -------- 6 files changed, 271 deletions(-) delete mode 100644 alembic/versions/2e5bd3a1eb88_initial.py delete mode 100644 alembic/versions/6f4e8b8f4b11_add_refresh_tokens.py delete mode 100644 alembic/versions/a1f9c7de42b3_add_prompt_templates.py delete mode 100644 alembic/versions/b3d1f4a90e2c_add_feature_spec_runs.py delete mode 100644 alembic/versions/c8a4e9d2f6b1_enforce_single_prompt_template_row.py delete mode 100644 alembic/versions/f1d2c3b4a5e6_rename_prompt_template_column_to_feature_summary.py diff --git a/alembic/versions/2e5bd3a1eb88_initial.py b/alembic/versions/2e5bd3a1eb88_initial.py deleted file mode 100644 index 9b120c8..0000000 --- a/alembic/versions/2e5bd3a1eb88_initial.py +++ /dev/null @@ -1,45 +0,0 @@ -"""initial - -Revision ID: 2e5bd3a1eb88 -Revises: -Create Date: 2026-04-11 10:30:05.432174 -""" - -import sqlalchemy as sa - -from alembic import op - -revision = "2e5bd3a1eb88" -down_revision = None -branch_labels = None -depends_on = None - - -def upgrade() -> None: - op.create_table( - "users", - sa.Column("id", sa.Integer(), nullable=False), - sa.Column("username", sa.String(length=100), nullable=False), - sa.Column("hashed_password", sa.String(length=1024), nullable=False), - sa.Column("is_active", sa.Boolean(), nullable=False), - sa.Column("is_superuser", sa.Boolean(), nullable=False), - sa.Column("is_verified", sa.Boolean(), nullable=False), - sa.Column( - "created_at", - sa.DateTime(timezone=True), - server_default=sa.text("now()"), - nullable=False, - ), - sa.Column("email", sa.String(length=320), nullable=False), - sa.PrimaryKeyConstraint("id"), - ) - op.create_index(op.f("ix_users_email"), "users", ["email"], unique=True) - op.create_index(op.f("ix_users_id"), "users", ["id"], unique=False) - op.create_index(op.f("ix_users_username"), "users", ["username"], unique=True) - - -def downgrade() -> None: - op.drop_index(op.f("ix_users_username"), table_name="users") - op.drop_index(op.f("ix_users_id"), table_name="users") - op.drop_index(op.f("ix_users_email"), table_name="users") - op.drop_table("users") diff --git a/alembic/versions/6f4e8b8f4b11_add_refresh_tokens.py b/alembic/versions/6f4e8b8f4b11_add_refresh_tokens.py deleted file mode 100644 index d0079cc..0000000 --- a/alembic/versions/6f4e8b8f4b11_add_refresh_tokens.py +++ /dev/null @@ -1,54 +0,0 @@ -"""add refresh tokens - -Revision ID: 6f4e8b8f4b11 -Revises: 2e5bd3a1eb88 -Create Date: 2026-04-11 22:30:00.000000 -""" - -import sqlalchemy as sa - -from alembic import op - -revision = "6f4e8b8f4b11" -down_revision = "2e5bd3a1eb88" -branch_labels = None -depends_on = None - - -def upgrade() -> None: - op.create_table( - "refresh_tokens", - sa.Column("id", sa.Integer(), nullable=False), - sa.Column("user_id", sa.Integer(), nullable=False), - sa.Column("token_hash", sa.String(length=128), nullable=False), - sa.Column("expires_at", sa.DateTime(timezone=True), nullable=False), - sa.Column( - "created_at", - sa.DateTime(timezone=True), - server_default=sa.text("now()"), - nullable=False, - ), - sa.Column("revoked_at", sa.DateTime(timezone=True), nullable=True), - sa.ForeignKeyConstraint(["user_id"], ["users.id"], ondelete="CASCADE"), - sa.PrimaryKeyConstraint("id"), - ) - op.create_index(op.f("ix_refresh_tokens_id"), "refresh_tokens", ["id"], unique=False) - op.create_index( - op.f("ix_refresh_tokens_token_hash"), - "refresh_tokens", - ["token_hash"], - unique=True, - ) - op.create_index( - op.f("ix_refresh_tokens_user_id"), - "refresh_tokens", - ["user_id"], - unique=False, - ) - - -def downgrade() -> None: - op.drop_index(op.f("ix_refresh_tokens_user_id"), table_name="refresh_tokens") - op.drop_index(op.f("ix_refresh_tokens_token_hash"), table_name="refresh_tokens") - op.drop_index(op.f("ix_refresh_tokens_id"), table_name="refresh_tokens") - op.drop_table("refresh_tokens") diff --git a/alembic/versions/a1f9c7de42b3_add_prompt_templates.py b/alembic/versions/a1f9c7de42b3_add_prompt_templates.py deleted file mode 100644 index 9c5b47c..0000000 --- a/alembic/versions/a1f9c7de42b3_add_prompt_templates.py +++ /dev/null @@ -1,35 +0,0 @@ -"""add prompt templates - -Revision ID: a1f9c7de42b3 -Revises: 6f4e8b8f4b11 -Create Date: 2026-04-15 12:15:00.000000 -""" - -import sqlalchemy as sa - -from alembic import op - -revision = "a1f9c7de42b3" -down_revision = "6f4e8b8f4b11" -branch_labels = None -depends_on = None - - -def upgrade() -> None: - op.create_table( - "prompt_templates", - sa.Column("id", sa.Integer(), nullable=False), - sa.Column("feature_to_user_stories", sa.Text(), nullable=False), - sa.Column("is_active", sa.Boolean(), nullable=False, server_default=sa.true()), - sa.Column( - "updated_at", - sa.DateTime(timezone=True), - server_default=sa.text("now()"), - nullable=False, - ), - sa.PrimaryKeyConstraint("id"), - ) - - -def downgrade() -> None: - op.drop_table("prompt_templates") \ No newline at end of file diff --git a/alembic/versions/b3d1f4a90e2c_add_feature_spec_runs.py b/alembic/versions/b3d1f4a90e2c_add_feature_spec_runs.py deleted file mode 100644 index a9fc2e8..0000000 --- a/alembic/versions/b3d1f4a90e2c_add_feature_spec_runs.py +++ /dev/null @@ -1,66 +0,0 @@ -"""add feature spec runs - -Revision ID: b3d1f4a90e2c -Revises: a1f9c7de42b3 -Create Date: 2026-04-15 13:05:00.000000 -""" - -import sqlalchemy as sa - -from alembic import op - -revision = "b3d1f4a90e2c" -down_revision = "a1f9c7de42b3" -branch_labels = None -depends_on = None - - -def upgrade() -> None: - op.create_table( - "feature_spec_runs", - sa.Column("id", sa.Integer(), nullable=False), - sa.Column("user_id", sa.Integer(), nullable=False), - sa.Column("feature_idea", sa.Text(), nullable=False), - sa.Column("status", sa.String(length=32), nullable=False), - sa.Column("response_json", sa.JSON(), nullable=True), - sa.Column("error_message", sa.Text(), nullable=True), - sa.Column( - "created_at", - sa.DateTime(timezone=True), - server_default=sa.text("now()"), - nullable=False, - ), - sa.Column( - "updated_at", - sa.DateTime(timezone=True), - server_default=sa.text("now()"), - nullable=False, - ), - sa.ForeignKeyConstraint(["user_id"], ["users.id"], ondelete="CASCADE"), - sa.PrimaryKeyConstraint("id"), - ) - op.create_index( - op.f("ix_feature_spec_runs_user_id"), - "feature_spec_runs", - ["user_id"], - unique=False, - ) - op.create_index( - op.f("ix_feature_spec_runs_status"), - "feature_spec_runs", - ["status"], - unique=False, - ) - op.create_index( - op.f("ix_feature_spec_runs_created_at"), - "feature_spec_runs", - ["created_at"], - unique=False, - ) - - -def downgrade() -> None: - op.drop_index(op.f("ix_feature_spec_runs_created_at"), table_name="feature_spec_runs") - op.drop_index(op.f("ix_feature_spec_runs_status"), table_name="feature_spec_runs") - op.drop_index(op.f("ix_feature_spec_runs_user_id"), table_name="feature_spec_runs") - op.drop_table("feature_spec_runs") diff --git a/alembic/versions/c8a4e9d2f6b1_enforce_single_prompt_template_row.py b/alembic/versions/c8a4e9d2f6b1_enforce_single_prompt_template_row.py deleted file mode 100644 index f3efada..0000000 --- a/alembic/versions/c8a4e9d2f6b1_enforce_single_prompt_template_row.py +++ /dev/null @@ -1,42 +0,0 @@ -"""enforce single prompt template row - -Revision ID: c8a4e9d2f6b1 -Revises: b3d1f4a90e2c -Create Date: 2026-04-15 13:45:00.000000 -""" - -import sqlalchemy as sa - -from alembic import op - -revision = "c8a4e9d2f6b1" -down_revision = "b3d1f4a90e2c" -branch_labels = None -depends_on = None - - -def upgrade() -> None: - conn = op.get_bind() - - row_count = conn.execute(sa.text("SELECT COUNT(*) FROM prompt_templates")).scalar_one() - if row_count > 1: - raise RuntimeError( - "Cannot enforce single-row constraint: prompt_templates has more than one row" - ) - - if row_count == 1: - conn.execute(sa.text("UPDATE prompt_templates SET id = 1 WHERE id <> 1")) - - op.create_check_constraint( - "ck_prompt_templates_single_row", - "prompt_templates", - "id = 1", - ) - - -def downgrade() -> None: - op.drop_constraint( - "ck_prompt_templates_single_row", - "prompt_templates", - type_="check", - ) diff --git a/alembic/versions/f1d2c3b4a5e6_rename_prompt_template_column_to_feature_summary.py b/alembic/versions/f1d2c3b4a5e6_rename_prompt_template_column_to_feature_summary.py deleted file mode 100644 index e6531e8..0000000 --- a/alembic/versions/f1d2c3b4a5e6_rename_prompt_template_column_to_feature_summary.py +++ /dev/null @@ -1,29 +0,0 @@ -"""rename prompt template column to feature summary - -Revision ID: f1d2c3b4a5e6 -Revises: c8a4e9d2f6b1 -Create Date: 2026-04-15 16:30:00.000000 -""" - -from alembic import op - -revision = "f1d2c3b4a5e6" -down_revision = "c8a4e9d2f6b1" -branch_labels = None -depends_on = None - - -def upgrade() -> None: - op.alter_column( - "prompt_templates", - "feature_to_user_stories", - new_column_name="feature_to_feature_summary", - ) - - -def downgrade() -> None: - op.alter_column( - "prompt_templates", - "feature_to_feature_summary", - new_column_name="feature_to_user_stories", - ) From 0cd0e2037117a7006c2bcb3f3ab49c8c5236671d Mon Sep 17 00:00:00 2001 From: opsmanager1 Date: Thu, 16 Apr 2026 00:00:07 +0300 Subject: [PATCH 32/52] feat: add schema bootstrap functionality when no migration files are found --- app/scripts/migrate_and_check.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/app/scripts/migrate_and_check.py b/app/scripts/migrate_and_check.py index a32a596..833fec6 100644 --- a/app/scripts/migrate_and_check.py +++ b/app/scripts/migrate_and_check.py @@ -31,7 +31,26 @@ def _extract_revisions(raw_output: str) -> set[str]: return revisions +def _has_migration_files() -> bool: + versions_dir = _project_root() / "alembic" / "versions" + return any(versions_dir.glob("*.py")) + + +def _bootstrap_schema_without_migrations() -> None: + from app.core.database import Base, engine + + logger.warning( + "No Alembic migration files found; creating schema via SQLAlchemy metadata" + ) + Base.metadata.create_all(bind=engine) + logger.info("Schema created via SQLAlchemy metadata") + + def migrate_and_check() -> None: + if not _has_migration_files(): + _bootstrap_schema_without_migrations() + return + logger.info("Running Alembic upgrade to head") _run_alembic("upgrade", "head") From 96119d9d458d77789d24bc2f294c506986522e8f Mon Sep 17 00:00:00 2001 From: opsmanager1 Date: Thu, 16 Apr 2026 00:03:26 +0300 Subject: [PATCH 33/52] feat: enhance schema bootstrapping to raise error if no SQLAlchemy models are registered --- app/scripts/migrate_and_check.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/app/scripts/migrate_and_check.py b/app/scripts/migrate_and_check.py index 833fec6..25e8da0 100644 --- a/app/scripts/migrate_and_check.py +++ b/app/scripts/migrate_and_check.py @@ -1,4 +1,5 @@ import logging +import importlib from pathlib import Path import re import subprocess @@ -37,13 +38,22 @@ def _has_migration_files() -> bool: def _bootstrap_schema_without_migrations() -> None: + importlib.import_module("app.modules.auth.models") + importlib.import_module("app.modules.feature_spec.models") from app.core.database import Base, engine logger.warning( "No Alembic migration files found; creating schema via SQLAlchemy metadata" ) + if not Base.metadata.tables: + raise RuntimeError( + "No SQLAlchemy models are registered; cannot bootstrap schema" + ) Base.metadata.create_all(bind=engine) - logger.info("Schema created via SQLAlchemy metadata") + logger.info( + "Schema created via SQLAlchemy metadata (tables=%s)", + sorted(Base.metadata.tables.keys()), + ) def migrate_and_check() -> None: From 5302657c90cb4b2117ba044ecb99cd6bbc30d55c Mon Sep 17 00:00:00 2001 From: opsmanager1 Date: Thu, 16 Apr 2026 00:08:39 +0300 Subject: [PATCH 34/52] feat: simplify feature summary prompt rendering by removing error handling for missing placeholders --- app/modules/feature_spec/prompts/feature_summary.py | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/app/modules/feature_spec/prompts/feature_summary.py b/app/modules/feature_spec/prompts/feature_summary.py index d674804..895889e 100644 --- a/app/modules/feature_spec/prompts/feature_summary.py +++ b/app/modules/feature_spec/prompts/feature_summary.py @@ -10,16 +10,9 @@ def build_feature_summary_prompt_from_template(template: str, feature_idea: str) -> str: safe_template = template.strip() escaped_feature_idea = html.escape(feature_idea.strip(), quote=True) - try: - return safe_template.format( - feature_idea=escaped_feature_idea, - input=escaped_feature_idea, - ) - except KeyError as exc: - missing_key = exc.args[0] if exc.args else "unknown" - raise ValueError( - f"Prompt template error: missing placeholder '{missing_key}'" - ) from exc + rendered = safe_template.replace("{feature_idea}", escaped_feature_idea) + rendered = rendered.replace("{input}", escaped_feature_idea) + return rendered def load_feature_summary_template(db: Session) -> str: From fe58b5a1cb0e72da739cf46bfdda80c58d245694 Mon Sep 17 00:00:00 2001 From: opsmanager1 Date: Thu, 16 Apr 2026 00:20:07 +0300 Subject: [PATCH 35/52] feat: update DbModelsAndApiEndpoints to allow mixed types for db_models and api_endpoints --- app/modules/feature_spec/schemas.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app/modules/feature_spec/schemas.py b/app/modules/feature_spec/schemas.py index 6021b0f..79af927 100644 --- a/app/modules/feature_spec/schemas.py +++ b/app/modules/feature_spec/schemas.py @@ -1,5 +1,5 @@ from datetime import datetime -from typing import Literal +from typing import Any, Literal from pydantic import BaseModel, Field, field_validator, model_validator @@ -26,8 +26,8 @@ class FeatureSummaryItem(BaseModel): class DbModelsAndApiEndpoints(BaseModel): - db_models: list[str] - api_endpoints: list[str] + db_models: list[str | dict[str, Any]] + api_endpoints: list[str | dict[str, Any]] class FeatureSummaryResult(BaseModel): From 656a3e3fa77948595a269cb216d7946bd35a276f Mon Sep 17 00:00:00 2001 From: serhii lemeshko Date: Thu, 16 Apr 2026 10:36:40 +0300 Subject: [PATCH 36/52] feat: refine default feature summary prompt template for clarity and structure --- app/scripts/bootstrap_prompt_template.py | 160 +++++++++++------------ 1 file changed, 76 insertions(+), 84 deletions(-) diff --git a/app/scripts/bootstrap_prompt_template.py b/app/scripts/bootstrap_prompt_template.py index defca5e..092b367 100644 --- a/app/scripts/bootstrap_prompt_template.py +++ b/app/scripts/bootstrap_prompt_template.py @@ -6,115 +6,107 @@ logger = logging.getLogger(__name__) -DEFAULT_FEATURE_SUMMARY_PROMPT_TEMPLATE = """You are a senior staff-level product engineer -and system architect working on production-grade software systems. +DEFAULT_FEATURE_SUMMARY_PROMPT_TEMPLATE = """You are a senior staff-level product engineer and system architect working on production-grade backend systems. -Your job is to convert a high-level feature idea into a precise, -implementation-ready technical specification. +Your task is to convert a high-level feature idea into a precise, implementation-ready technical specification. -You must think in terms of real software delivery: backend systems, APIs, -databases, edge cases, and scalability. +You must think in terms of real backend development: APIs, database schema, data consistency, edge cases, and scalability. --- ## Core rules: -- Be extremely structured and deterministic -- Do NOT invent product requirements that were not implied or provided -- If information is missing, explicitly list assumptions in a separate section -- Prefer simplicity over over-engineering -- Think like you are writing a specification for a real engineering team -- All outputs must be actionable and implementation-oriented ---- - -## Output format (strict structure): - -1. FEATURE OVERVIEW -- Short description of the feature -- Problem it solves -- Target users +- Be strictly structured and deterministic +- Do NOT invent product requirements beyond the given feature idea +- If information is missing, explicitly list it under "assumptions" +- Prefer simple and realistic solutions over over-engineering +- Every part of the output must be implementation-ready +- Avoid vague phrases like "etc", "additional fields", "and so on" +- If something is required — define it explicitly --- -2. USER STORIES -- List of user stories in format: - As a [user type], I want [goal], so that [benefit] - ---- +## Hard requirements: -3. ACCEPTANCE CRITERIA -- Each user story must have clear, testable conditions -- Use bullet points -- Must be unambiguous and QA-ready +- Database design MUST include full schema: + - each table must have 4–8 fields + - include types and constraints (PK, FK, unique, nullable) + - include relationships ---- +- API design MUST: + - include at least 4 endpoints (not just 1–2) + - include request validation + - include access control logic + - include error responses -4. API DESIGN (REST) -- List endpoints with: - - method (GET, POST, etc.) - - path - - request body (if applicable) - - response schema (high-level) - - purpose +- Acceptance criteria MUST: + - include positive and negative cases + - include authorization behavior + - be testable (QA-ready, no vague language) ---- +- Edge cases & risks MUST include: + - concurrency issues + - data consistency problems + - security concerns + - failure scenarios -5. DATABASE DESIGN -- Define tables with: - - table name - - fields with types - - relationships -- Keep it normalized and production-oriented +- If the feature involves payments: + - include payment provider interaction (e.g. Stripe-like flow) + - include idempotency handling + - include transaction status tracking --- -6. EDGE CASES & RISKS -- Identify failure scenarios -- Data consistency risks -- Security concerns -- Performance concerns +## Output format (STRICT JSON ONLY) ---- +Return ONLY valid JSON. No explanations, no markdown, no extra text. -7. FUTURE IMPROVEMENTS -- Optional enhancements -- Scalability improvements -- UX improvements +{ + "assumptions": ["string"], + + "user_stories": [ + { + "title": "string", + "as_a": "string", + "i_want": "string", + "so_that": "string" + } + ], + + "acceptance_criteria": ["string"], + + "db_models": [ + { + "table": "string", + "fields": [ + { + "name": "string", + "type": "string", + "constraints": "string" + } + ], + "relationships": ["string"] + } + ], + + "api_endpoints": [ + { + "method": "string", + "path": "string", + "request_body": "string", + "response": "string", + "errors": ["string"], + "purpose": "string" + } + ], + + "risk_assessment": ["string"] +} --- -## Important constraints: -- Do not add irrelevant features or expand scope -- Keep all assumptions explicitly labeled as "Assumption" -- Prefer backend clarity over marketing language -- Avoid vague phrases like "user-friendly", "robust", "fast" -- Be precise, technical, and engineering-focused - Feature idea: {feature_idea} - ---- - -Return format requirements (strict): -- Return ONLY valid JSON. -- Do not include markdown, code fences, or any text before/after JSON. -- JSON object must match this structure exactly: -{ - "user_stories": [ - { - "title": "string", - "as_a": "string", - "i_want": "string", - "so_that": "string" - } - ], - "acceptance_criteria": ["string"], - "db_models_and_api_endpoints": { - "db_models": ["string"], - "api_endpoints": ["string"] - }, - "risk_assessment": ["string"] -} """ From 28a1c74af5910ae307ba8d0b74af730a4a78ea33 Mon Sep 17 00:00:00 2001 From: serhii lemeshko Date: Thu, 16 Apr 2026 10:57:06 +0300 Subject: [PATCH 37/52] feat: improve readability of the default feature summary prompt template --- app/scripts/bootstrap_prompt_template.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/app/scripts/bootstrap_prompt_template.py b/app/scripts/bootstrap_prompt_template.py index 092b367..68626b0 100644 --- a/app/scripts/bootstrap_prompt_template.py +++ b/app/scripts/bootstrap_prompt_template.py @@ -6,11 +6,14 @@ logger = logging.getLogger(__name__) -DEFAULT_FEATURE_SUMMARY_PROMPT_TEMPLATE = """You are a senior staff-level product engineer and system architect working on production-grade backend systems. +DEFAULT_FEATURE_SUMMARY_PROMPT_TEMPLATE = """You are a senior staff-level product engineer +and system architect working on production-grade backend systems. -Your task is to convert a high-level feature idea into a precise, implementation-ready technical specification. +Your task is to convert a high-level feature idea into a precise, +implementation-ready technical specification. -You must think in terms of real backend development: APIs, database schema, data consistency, edge cases, and scalability. +You must think in terms of real backend development: APIs, database schema, +data consistency, edge cases, and scalability. --- From 51c0217c12671d4798cf1c143fcfbf21dcdb2995 Mon Sep 17 00:00:00 2001 From: serhii lemeshko Date: Thu, 16 Apr 2026 11:00:53 +0300 Subject: [PATCH 38/52] feat: update README to reflect changes in feature specification endpoints and project structure --- README.md | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index fd6b720..309d479 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ Production-ready FastAPI backend for authentication, LLM-powered specification g - JWT authentication based on fastapi-users - User registration and user management endpoints -- LLM generation endpoint with multiple response formats (text, sections, json) +- Feature specification generation endpoints powered by Ollama - Readiness and health probes for runtime checks - Alembic database migrations - Security middleware baseline: @@ -49,8 +49,8 @@ Production-ready FastAPI backend for authentication, LLM-powered specification g - app/api/: health, readiness, OpenAPI customization - app/middlewares/: security middleware composition and implementations - app/modules/auth/: auth domain (models, schemas, dependencies, router) -- app/modules/llm/: LLM API, schemas, providers -- app/scripts/: utility scripts (admin bootstrap) +- app/modules/feature_spec/: feature spec API, schemas, prompts, providers +- app/scripts/: utility scripts (admin and prompt/model bootstrap) - alembic/: migration config and versions - docker-compose.yml: containerized app run @@ -97,6 +97,7 @@ source venv/bin/activate pip install --upgrade pip pip install -r requirements.txt +pip install -r requirements-dev.txt python -m alembic upgrade head python -m app.scripts.bootstrap_admin @@ -133,12 +134,14 @@ curl http://localhost:11434/api/generate -d '{ ## API Docs -When server is running: +When server is running locally: - Swagger UI: http://localhost:8000/docs - ReDoc: http://localhost:8000/redoc (pinned ReDoc 2.x script) - OpenAPI JSON: http://localhost:8000/openapi.json +For Docker Compose deployment, use port 8001 instead of 8000. + ## API Endpoints Base API prefix: /api/v1 @@ -169,16 +172,16 @@ Login note: - `/api/v1/auth/jwt/logout` clears refresh cookie on client side. - Swagger `Authorize` value must contain only raw JWT token (without `Bearer ` prefix). -### LLM +### Feature Spec -- POST /api/v1/llm/generate +- POST /api/v1/feature-spec/generate +- GET /api/v1/feature-spec/history?limit=10 -Request body example: +Request body example for generation: ```json { - "prompt": "Generate API specification for user profile module", - "response_format": "sections" + "feature_idea": "payment for premium posts" } ``` @@ -187,7 +190,7 @@ Request body example: Run linter: ```bash -python -m flake8 +python -m flake8 . ``` Run auth unit tests: From 6d1e2f5612b563cbc864907fcf95b7bbcd68229e Mon Sep 17 00:00:00 2001 From: serhii lemeshko Date: Thu, 16 Apr 2026 11:19:41 +0300 Subject: [PATCH 39/52] feat: implement admin panel with authentication and user management views --- app/admin/__init__.py | 3 ++ app/admin/auth.py | 64 +++++++++++++++++++++++++++++ app/admin/setup.py | 24 +++++++++++ app/admin/views.py | 95 +++++++++++++++++++++++++++++++++++++++++++ app/api/openapi.py | 13 ++++++ app/main.py | 16 +++----- requirements.txt | 3 ++ 7 files changed, 207 insertions(+), 11 deletions(-) create mode 100644 app/admin/__init__.py create mode 100644 app/admin/auth.py create mode 100644 app/admin/setup.py create mode 100644 app/admin/views.py diff --git a/app/admin/__init__.py b/app/admin/__init__.py new file mode 100644 index 0000000..c24bd4b --- /dev/null +++ b/app/admin/__init__.py @@ -0,0 +1,3 @@ +from app.admin.setup import setup_admin + +__all__ = ["setup_admin"] diff --git a/app/admin/auth.py b/app/admin/auth.py new file mode 100644 index 0000000..d6cd6e6 --- /dev/null +++ b/app/admin/auth.py @@ -0,0 +1,64 @@ +from fastapi.security import OAuth2PasswordRequestForm +from sqlalchemy import select +from sqladmin.authentication import AuthenticationBackend +from starlette.requests import Request + +from app.core.database import AsyncSessionLocal +from app.core.settings import settings +from app.modules.auth.infrastructure.fastapi_users_adapter import ( + UserManager, + UsernameAwareUserDatabase, +) +from app.modules.auth.models import User + + +class AdminAuthBackend(AuthenticationBackend): + def __init__(self) -> None: + super().__init__(secret_key=settings.SECRET_KEY) + + async def login(self, request: Request) -> bool: + form = await request.form() + username = str(form.get("username", "")).strip() + password = str(form.get("password", "")).strip() + if not username or not password: + return False + + credentials = OAuth2PasswordRequestForm( + username=username, + password=password, + scope="", + client_id=None, + client_secret=None, + ) + + async with AsyncSessionLocal() as session: + user_db = UsernameAwareUserDatabase(session, User) + user_manager = UserManager(user_db) + user = await user_manager.authenticate(credentials) + + if user is None or not user.is_active or not user.is_superuser: + return False + + request.session.update({"admin_user_id": user.id}) + return True + + async def logout(self, request: Request) -> bool: + request.session.clear() + return True + + async def authenticate(self, request: Request) -> bool: + user_id = request.session.get("admin_user_id") + if user_id is None: + return False + + async with AsyncSessionLocal() as session: + statement = select(User).where(User.id == int(user_id)) + user = await session.scalar(statement) + + if user is None or not user.is_active or not user.is_superuser: + request.session.clear() + return False + return True + + +admin_auth_backend = AdminAuthBackend() diff --git a/app/admin/setup.py b/app/admin/setup.py new file mode 100644 index 0000000..dfd0b61 --- /dev/null +++ b/app/admin/setup.py @@ -0,0 +1,24 @@ +from fastapi import FastAPI +from sqladmin import Admin + +from app.admin.auth import admin_auth_backend +from app.admin.views import ( + FeatureSpecRunAdmin, + PromptTemplateAdmin, + RefreshTokenAdmin, + UserAdmin, +) +from app.core.database import engine + + +def setup_admin(app: FastAPI) -> None: + admin = Admin( + app=app, + engine=engine, + authentication_backend=admin_auth_backend, + title="Specification Generator Admin", + ) + admin.add_view(UserAdmin) + admin.add_view(RefreshTokenAdmin) + admin.add_view(PromptTemplateAdmin) + admin.add_view(FeatureSpecRunAdmin) diff --git a/app/admin/views.py b/app/admin/views.py new file mode 100644 index 0000000..db09b47 --- /dev/null +++ b/app/admin/views.py @@ -0,0 +1,95 @@ +from sqladmin import ModelView + +from app.modules.auth.models import RefreshToken, User +from app.modules.feature_spec.models import FeatureSpecRun, PromptTemplate + + +class UserAdmin(ModelView, model=User): + name = "User" + name_plural = "Users" + icon = "fa-solid fa-user" + + column_list = [ + User.id, + User.username, + User.email, + User.is_active, + User.is_superuser, + User.is_verified, + User.created_at, + ] + column_searchable_list = [User.username, User.email] + column_filters = [User.is_active, User.is_superuser, User.is_verified, User.created_at] + column_sortable_list = [User.id, User.username, User.email, User.created_at] + + column_exclude_list = [User.hashed_password] + form_excluded_columns = [User.hashed_password, User.created_at] + can_create = False + can_delete = False + + +class RefreshTokenAdmin(ModelView, model=RefreshToken): + name = "Refresh Token" + name_plural = "Refresh Tokens" + icon = "fa-solid fa-key" + + column_list = [ + RefreshToken.id, + RefreshToken.user_id, + RefreshToken.expires_at, + RefreshToken.created_at, + RefreshToken.revoked_at, + ] + column_searchable_list = [RefreshToken.user_id] + column_filters = [RefreshToken.expires_at, RefreshToken.created_at, RefreshToken.revoked_at] + column_sortable_list = [RefreshToken.id, RefreshToken.user_id, RefreshToken.expires_at] + + column_exclude_list = [RefreshToken.token_hash] + form_excluded_columns = [RefreshToken.token_hash, RefreshToken.created_at] + can_create = False + can_edit = False + + +class PromptTemplateAdmin(ModelView, model=PromptTemplate): + name = "Prompt Template" + name_plural = "Prompt Templates" + icon = "fa-solid fa-file-lines" + + column_list = [ + PromptTemplate.id, + PromptTemplate.is_active, + PromptTemplate.updated_at, + PromptTemplate.feature_to_feature_summary, + ] + column_searchable_list = [PromptTemplate.feature_to_feature_summary] + column_filters = [PromptTemplate.is_active, PromptTemplate.updated_at] + column_sortable_list = [PromptTemplate.id, PromptTemplate.updated_at] + + can_create = False + can_delete = False + + +class FeatureSpecRunAdmin(ModelView, model=FeatureSpecRun): + name = "Feature Spec Run" + name_plural = "Feature Spec Runs" + icon = "fa-solid fa-wand-magic-sparkles" + + column_list = [ + FeatureSpecRun.id, + FeatureSpecRun.user_id, + FeatureSpecRun.status, + FeatureSpecRun.feature_idea, + FeatureSpecRun.created_at, + FeatureSpecRun.updated_at, + ] + column_searchable_list = [FeatureSpecRun.feature_idea, FeatureSpecRun.status] + column_filters = [FeatureSpecRun.status, FeatureSpecRun.created_at, FeatureSpecRun.updated_at] + column_sortable_list = [ + FeatureSpecRun.id, + FeatureSpecRun.user_id, + FeatureSpecRun.status, + FeatureSpecRun.created_at, + ] + + can_create = False + can_delete = False diff --git a/app/api/openapi.py b/app/api/openapi.py index d9350a8..1de66dc 100644 --- a/app/api/openapi.py +++ b/app/api/openapi.py @@ -1,4 +1,5 @@ from fastapi import FastAPI +from fastapi.openapi.docs import get_redoc_html from fastapi.openapi.utils import get_openapi @@ -43,3 +44,15 @@ def custom_openapi() -> dict: return app.openapi_schema app.openapi = custom_openapi + + +def configure_redoc_route(app: FastAPI) -> None: + @app.get("/redoc", include_in_schema=False) + async def redoc_html(): + return get_redoc_html( + openapi_url=app.openapi_url, + title=f"{app.title} - ReDoc", + redoc_js_url=( + "https://cdn.jsdelivr.net/npm/redoc@2.1.5/bundles/redoc.standalone.js" + ), + ) diff --git a/app/main.py b/app/main.py index dd4a95f..0543db8 100644 --- a/app/main.py +++ b/app/main.py @@ -1,8 +1,8 @@ from fastapi import FastAPI -from fastapi.openapi.docs import get_redoc_html +from app.admin import setup_admin from app.api.health import router as health_router -from app.api.openapi import configure_openapi_bearer_auth +from app.api.openapi import configure_openapi_bearer_auth, configure_redoc_route from app.core.settings import settings from app.core.startup import lifespan from app.middlewares import configure_security_middlewares @@ -16,6 +16,8 @@ redoc_url=None, ) +setup_admin(app) + configure_security_middlewares(app) app.include_router(auth_router, prefix=settings.API_V1_PREFIX) @@ -23,12 +25,4 @@ app.include_router(health_router) configure_openapi_bearer_auth(app) - - -@app.get("/redoc", include_in_schema=False) -async def redoc_html(): - return get_redoc_html( - openapi_url=app.openapi_url, - title=f"{app.title} - ReDoc", - redoc_js_url="https://cdn.jsdelivr.net/npm/redoc@2.1.5/bundles/redoc.standalone.js", - ) +configure_redoc_route(app) diff --git a/requirements.txt b/requirements.txt index c55ea22..52c9043 100644 --- a/requirements.txt +++ b/requirements.txt @@ -28,3 +28,6 @@ python-dotenv==1.0.1 # HTTP requests httpx==0.27.0 + +# Admin panel +sqladmin==0.17.0 From 45343763563e4a8e39d21356563ed679999b2211 Mon Sep 17 00:00:00 2001 From: serhii lemeshko Date: Thu, 16 Apr 2026 11:29:24 +0300 Subject: [PATCH 40/52] feat: remove excluded columns from User and RefreshToken admin views --- app/admin/views.py | 2 -- requirements.txt | 1 + 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/app/admin/views.py b/app/admin/views.py index db09b47..3848b64 100644 --- a/app/admin/views.py +++ b/app/admin/views.py @@ -22,7 +22,6 @@ class UserAdmin(ModelView, model=User): column_filters = [User.is_active, User.is_superuser, User.is_verified, User.created_at] column_sortable_list = [User.id, User.username, User.email, User.created_at] - column_exclude_list = [User.hashed_password] form_excluded_columns = [User.hashed_password, User.created_at] can_create = False can_delete = False @@ -44,7 +43,6 @@ class RefreshTokenAdmin(ModelView, model=RefreshToken): column_filters = [RefreshToken.expires_at, RefreshToken.created_at, RefreshToken.revoked_at] column_sortable_list = [RefreshToken.id, RefreshToken.user_id, RefreshToken.expires_at] - column_exclude_list = [RefreshToken.token_hash] form_excluded_columns = [RefreshToken.token_hash, RefreshToken.created_at] can_create = False can_edit = False diff --git a/requirements.txt b/requirements.txt index 52c9043..89bbdb0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -31,3 +31,4 @@ httpx==0.27.0 # Admin panel sqladmin==0.17.0 +itsdangerous==2.2.0 From 7fa07e966379c06b17c7bdcbe0e59de03cef893b Mon Sep 17 00:00:00 2001 From: serhii lemeshko Date: Thu, 16 Apr 2026 13:05:08 +0300 Subject: [PATCH 41/52] feat: integrate Celery for background task processing and add feature specification generation --- .env.example | 8 ++ app/core/settings.py | 5 ++ app/infrastructure/__init__.py | 1 + app/infrastructure/celery_app.py | 20 +++++ app/infrastructure/ollama_client.py | 39 ++++++++++ app/infrastructure/tasks/__init__.py | 1 + .../tasks/feature_spec_tasks.py | 75 +++++++++++++++++++ app/modules/feature_spec/router.py | 36 ++++----- app/modules/feature_spec/schemas.py | 12 +++ app/modules/feature_spec/service.py | 56 ++++++++++++++ docker-compose.yml | 48 +++++++++++- entrypoint.sh | 2 +- requirements.txt | 3 + 13 files changed, 284 insertions(+), 22 deletions(-) create mode 100644 app/infrastructure/__init__.py create mode 100644 app/infrastructure/celery_app.py create mode 100644 app/infrastructure/ollama_client.py create mode 100644 app/infrastructure/tasks/__init__.py create mode 100644 app/infrastructure/tasks/feature_spec_tasks.py create mode 100644 app/modules/feature_spec/service.py diff --git a/.env.example b/.env.example index 57ec2b3..1051262 100644 --- a/.env.example +++ b/.env.example @@ -5,6 +5,10 @@ VERSION= HOST= PORT= +FASTAPI_HOST_PORT= +FASTAPI_CONTAINER_PORT= +REDIS_HOST_PORT= +REDIS_CONTAINER_PORT= DATABASE_URL=postgresql://:@:5432/?sslmode=disable SECRET_KEY= @@ -44,6 +48,10 @@ OLLAMA_RETRY_BACKOFF_SECONDS= OLLAMA_SYSTEM_PROMPT= LLM_PROMPT_MAX_LENGTH= FEATURE_SPEC_HISTORY_DEFAULT_LIMIT= +CELERY_BROKER_URL= +CELERY_RESULT_BACKEND= +CELERY_TASK_MAX_RETRIES= +CELERY_TASK_RETRY_BASE_SECONDS= TEST_DB_HOST= TEST_DB_PORT= diff --git a/app/core/settings.py b/app/core/settings.py index 56e29d6..e1c7339 100644 --- a/app/core/settings.py +++ b/app/core/settings.py @@ -81,6 +81,11 @@ class Settings(BaseSettings): LLM_PROMPT_MAX_LENGTH: int = 8000 FEATURE_SPEC_HISTORY_DEFAULT_LIMIT: int = 10 + CELERY_BROKER_URL: str = "redis://redis:6379/0" + CELERY_RESULT_BACKEND: str = "redis://redis:6379/1" + CELERY_TASK_MAX_RETRIES: int = 3 + CELERY_TASK_RETRY_BASE_SECONDS: int = 2 + model_config = SettingsConfigDict( env_file=".env", case_sensitive=True, diff --git a/app/infrastructure/__init__.py b/app/infrastructure/__init__.py new file mode 100644 index 0000000..2182630 --- /dev/null +++ b/app/infrastructure/__init__.py @@ -0,0 +1 @@ +"""Infrastructure layer modules.""" diff --git a/app/infrastructure/celery_app.py b/app/infrastructure/celery_app.py new file mode 100644 index 0000000..ab63472 --- /dev/null +++ b/app/infrastructure/celery_app.py @@ -0,0 +1,20 @@ +from celery import Celery + +from app.core.settings import settings + + +celery_app = Celery( + "specification_generator", + broker=settings.CELERY_BROKER_URL, + backend=settings.CELERY_RESULT_BACKEND, + include=["app.infrastructure.tasks.feature_spec_tasks"], +) + +celery_app.conf.update( + task_track_started=True, + task_serializer="json", + result_serializer="json", + accept_content=["json"], + timezone="UTC", + enable_utc=True, +) diff --git a/app/infrastructure/ollama_client.py b/app/infrastructure/ollama_client.py new file mode 100644 index 0000000..ae50925 --- /dev/null +++ b/app/infrastructure/ollama_client.py @@ -0,0 +1,39 @@ +import httpx + +from app.core.settings import settings + + +class OllamaSyncClient: + def __init__(self) -> None: + self._base_url = settings.OLLAMA_BASE_URL.rstrip("/") + self._model = settings.OLLAMA_MODEL + self._timeout = httpx.Timeout( + float(settings.OLLAMA_TIMEOUT), + connect=float(settings.OLLAMA_CONNECT_TIMEOUT), + ) + self._system_prompt = settings.OLLAMA_SYSTEM_PROMPT + + def _build_payload(self, user_prompt: str) -> dict: + messages = [] + if self._system_prompt.strip(): + messages.append({"role": "system", "content": self._system_prompt}) + messages.append({"role": "user", "content": user_prompt}) + return { + "model": self._model, + "stream": False, + "format": "json", + "messages": messages, + } + + def generate(self, user_prompt: str) -> str: + url = f"{self._base_url}/api/chat" + payload = self._build_payload(user_prompt) + with httpx.Client(timeout=self._timeout) as client: + response = client.post(url, json=payload) + response.raise_for_status() + data = response.json() + content = data.get("message", {}).get("content", "") + return content if isinstance(content, str) else str(content) + + +ollama_sync_client = OllamaSyncClient() diff --git a/app/infrastructure/tasks/__init__.py b/app/infrastructure/tasks/__init__.py new file mode 100644 index 0000000..27a2a9f --- /dev/null +++ b/app/infrastructure/tasks/__init__.py @@ -0,0 +1 @@ +"""Celery task modules.""" diff --git a/app/infrastructure/tasks/feature_spec_tasks.py b/app/infrastructure/tasks/feature_spec_tasks.py new file mode 100644 index 0000000..bc9fcf4 --- /dev/null +++ b/app/infrastructure/tasks/feature_spec_tasks.py @@ -0,0 +1,75 @@ +import httpx +from celery.exceptions import MaxRetriesExceededError + +from app.core.database import SessionLocal +from app.core.settings import settings +from app.infrastructure.celery_app import celery_app +from app.infrastructure.ollama_client import ollama_sync_client +from app.modules.feature_spec.models import FeatureSpecRun +from app.modules.feature_spec.prompts.feature_summary import ( + build_feature_summary_prompt_from_db, + parse_feature_summary_response, +) +from app.modules.feature_spec.schemas import FeatureSummaryResult + + +@celery_app.task(bind=True, name="feature_spec.generate") +def generate_feature_spec_task(self, run_id: int, feature_idea: str, user_id: int) -> dict: + db = SessionLocal() + try: + run = db.get(FeatureSpecRun, run_id) + if run is None: + return {"run_id": run_id, "status": "error", "message": "Run not found"} + + if run.status == "success" and run.response_json is not None: + return { + "run_id": run.id, + "status": run.status, + "feature_idea": run.feature_idea, + "feature_summary": run.response_json, + } + + try: + prompt = build_feature_summary_prompt_from_db(feature_idea, db) + feature_summary_raw = ollama_sync_client.generate(prompt) + feature_summary_json = parse_feature_summary_response(feature_summary_raw) + feature_summary_typed = FeatureSummaryResult.model_validate(feature_summary_json) + + run.status = "success" + run.response_json = feature_summary_typed.model_dump(mode="json") + run.error_message = None + db.add(run) + db.commit() + + return { + "run_id": run.id, + "status": run.status, + "feature_idea": run.feature_idea, + "feature_summary": run.response_json, + } + except ( + httpx.TimeoutException, + httpx.RequestError, + httpx.HTTPStatusError, + ) as exc: + retry_number = int(self.request.retries) + 1 + if retry_number <= settings.CELERY_TASK_MAX_RETRIES: + delay_seconds = settings.CELERY_TASK_RETRY_BASE_SECONDS**retry_number + try: + raise self.retry(exc=exc, countdown=delay_seconds) + except MaxRetriesExceededError: + pass + + run.status = "error" + run.error_message = "LLM provider request failed" + db.add(run) + db.commit() + raise RuntimeError("LLM task failed after retries") from exc + except Exception as exc: + run.status = "error" + run.error_message = "Failed to process feature specification" + db.add(run) + db.commit() + raise RuntimeError("Feature specification task failed") from exc + finally: + db.close() diff --git a/app/modules/feature_spec/router.py b/app/modules/feature_spec/router.py index de4d668..49beb22 100644 --- a/app/modules/feature_spec/router.py +++ b/app/modules/feature_spec/router.py @@ -1,41 +1,37 @@ -from fastapi import APIRouter, Depends, HTTPException, status +from fastapi import APIRouter, Depends from sqlalchemy.orm import Session from app.core.database import get_db from app.core.settings import settings from app.modules.auth.dependencies import get_current_user from app.modules.auth.models import User -from app.modules.feature_spec.orchestrator import ( - generate_feature_spec, - get_feature_spec_history, -) +from app.modules.feature_spec.orchestrator import get_feature_spec_history from app.modules.feature_spec.schemas import ( FeatureSpecGenerateRequest, - FeatureSpecGenerateResponse, FeatureSpecHistoryResponse, + FeatureSpecTaskStatusResponse, + FeatureSpecTaskSubmitResponse, +) +from app.modules.feature_spec.service import ( + get_feature_spec_task_status, + submit_feature_spec_generation, ) router = APIRouter(prefix="/feature-spec", tags=["feature-spec"]) -@router.post("/generate", response_model=FeatureSpecGenerateResponse) +@router.post("/generate", response_model=FeatureSpecTaskSubmitResponse) async def generate_feature_spec_endpoint( payload: FeatureSpecGenerateRequest, current_user: User = Depends(get_current_user), db: Session = Depends(get_db), -) -> FeatureSpecGenerateResponse: - try: - return await generate_feature_spec(payload.feature_idea, db, current_user.id) - except ValueError as exc: - raise HTTPException( - status_code=status.HTTP_502_BAD_GATEWAY, - detail=str(exc), - ) from exc - except RuntimeError as exc: - raise HTTPException( - status_code=status.HTTP_502_BAD_GATEWAY, - detail=str(exc), - ) from exc +) -> FeatureSpecTaskSubmitResponse: + return submit_feature_spec_generation(payload.feature_idea, db, current_user.id) + + +@router.get("/tasks/{task_id}", response_model=FeatureSpecTaskStatusResponse) +async def get_feature_spec_task_status_endpoint(task_id: str) -> FeatureSpecTaskStatusResponse: + return get_feature_spec_task_status(task_id) @router.get("/history", response_model=FeatureSpecHistoryResponse) diff --git a/app/modules/feature_spec/schemas.py b/app/modules/feature_spec/schemas.py index 79af927..bcde2a7 100644 --- a/app/modules/feature_spec/schemas.py +++ b/app/modules/feature_spec/schemas.py @@ -70,6 +70,18 @@ class FeatureSpecGenerateResponse(BaseModel): feature_summary: FeatureSummaryResult +class FeatureSpecTaskSubmitResponse(BaseModel): + task_id: str + status: Literal["processing"] + + +class FeatureSpecTaskStatusResponse(BaseModel): + task_id: str + status: Literal["PENDING", "STARTED", "SUCCESS", "FAILURE"] + result: dict[str, Any] | None = None + error: str | None = None + + class FeatureSpecHistoryItem(BaseModel): id: int feature_idea: str diff --git a/app/modules/feature_spec/service.py b/app/modules/feature_spec/service.py new file mode 100644 index 0000000..ff13d09 --- /dev/null +++ b/app/modules/feature_spec/service.py @@ -0,0 +1,56 @@ +from celery.result import AsyncResult +from sqlalchemy.orm import Session + +from app.infrastructure.celery_app import celery_app +from app.infrastructure.tasks.feature_spec_tasks import generate_feature_spec_task +from app.modules.feature_spec.models import FeatureSpecRun +from app.modules.feature_spec.schemas import ( + FeatureSpecTaskStatusResponse, + FeatureSpecTaskSubmitResponse, +) + + +TERMINAL_STATES = {"SUCCESS", "FAILURE"} + + +def submit_feature_spec_generation( + feature_idea: str, + db: Session, + user_id: int, +) -> FeatureSpecTaskSubmitResponse: + run = FeatureSpecRun( + user_id=user_id, + feature_idea=feature_idea, + status="pending", + ) + db.add(run) + db.commit() + db.refresh(run) + + task = generate_feature_spec_task.delay(run.id, feature_idea, user_id) + return FeatureSpecTaskSubmitResponse(task_id=task.id, status="processing") + + +def get_feature_spec_task_status(task_id: str) -> FeatureSpecTaskStatusResponse: + async_result = AsyncResult(task_id, app=celery_app) + state = async_result.state + + if state == "SUCCESS": + payload = async_result.result if isinstance(async_result.result, dict) else None + return FeatureSpecTaskStatusResponse( + task_id=task_id, + status="SUCCESS", + result=payload, + ) + + if state == "FAILURE": + return FeatureSpecTaskStatusResponse( + task_id=task_id, + status="FAILURE", + error="Task execution failed", + ) + + if state == "STARTED": + return FeatureSpecTaskStatusResponse(task_id=task_id, status="STARTED") + + return FeatureSpecTaskStatusResponse(task_id=task_id, status="PENDING") diff --git a/docker-compose.yml b/docker-compose.yml index 906423e..ef1c199 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -8,10 +8,14 @@ services: env_file: - .env ports: - - "8001:8001" + - "${FASTAPI_HOST_PORT:-8005}:${FASTAPI_CONTAINER_PORT:-8001}" depends_on: + redis: + condition: service_healthy ollama: condition: service_healthy + celery-worker: + condition: service_healthy healthcheck: test: [ @@ -27,6 +31,48 @@ services: stop_grace_period: 20s restart: unless-stopped + celery-worker: + build: + context: . + dockerfile: Dockerfile + container_name: specification-generator-celery-worker + init: true + env_file: + - .env + command: ["celery", "-A", "app.infrastructure.celery_app:celery_app", "worker", "--loglevel=INFO"] + depends_on: + redis: + condition: service_healthy + ollama: + condition: service_healthy + healthcheck: + test: + [ + "CMD", + "python", + "-c", + "from app.infrastructure.celery_app import celery_app; import sys; i=celery_app.control.inspect(timeout=2); sys.exit(0 if i and i.ping() else 1)", + ] + interval: 30s + timeout: 5s + retries: 3 + start_period: 20s + restart: unless-stopped + + redis: + image: redis:7-alpine + container_name: specification-generator-redis + init: true + ports: + - "${REDIS_HOST_PORT:-6380}:${REDIS_CONTAINER_PORT:-6379}" + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 10s + timeout: 3s + retries: 5 + start_period: 10s + restart: unless-stopped + ollama: image: ollama/ollama:latest container_name: specification-generator-ollama diff --git a/entrypoint.sh b/entrypoint.sh index b6729c1..d7f387a 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -14,7 +14,7 @@ echo "[start] Ensuring Ollama model is available..." python -m app.scripts.ensure_ollama_model if [ "$#" -eq 0 ]; then - set -- python -m uvicorn app.main:app --host 0.0.0.0 --port 8001 + set -- python -m uvicorn app.main:app --host 0.0.0.0 --port "${PORT:-8001}" fi echo "[start] Starting: $*" diff --git a/requirements.txt b/requirements.txt index 89bbdb0..2f846be 100644 --- a/requirements.txt +++ b/requirements.txt @@ -29,6 +29,9 @@ python-dotenv==1.0.1 # HTTP requests httpx==0.27.0 +# Background processing +celery[redis]==5.4.0 + # Admin panel sqladmin==0.17.0 itsdangerous==2.2.0 From eb3083278003b21ad61bb510812f8462045c60bd Mon Sep 17 00:00:00 2001 From: serhii lemeshko Date: Thu, 16 Apr 2026 13:12:23 +0300 Subject: [PATCH 42/52] feat: remove feature_spec module initialization file --- app/modules/feature_spec/__init__.py | 3 --- 1 file changed, 3 deletions(-) delete mode 100644 app/modules/feature_spec/__init__.py diff --git a/app/modules/feature_spec/__init__.py b/app/modules/feature_spec/__init__.py deleted file mode 100644 index c57be5d..0000000 --- a/app/modules/feature_spec/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from app.modules.feature_spec.router import router - -__all__ = ["router"] From 66b0b24a5a3db9f45289cf1fd618bbbf98b7086c Mon Sep 17 00:00:00 2001 From: serhii lemeshko Date: Thu, 16 Apr 2026 13:29:49 +0300 Subject: [PATCH 43/52] feat: increase OLLAMA_TIMEOUT to improve response handling and add timeout to post request --- app/core/settings.py | 2 +- app/infrastructure/ollama_client.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/app/core/settings.py b/app/core/settings.py index e1c7339..ab3fe72 100644 --- a/app/core/settings.py +++ b/app/core/settings.py @@ -68,7 +68,7 @@ class Settings(BaseSettings): OLLAMA_BASE_URL: str = "http://localhost:11434" OLLAMA_MODEL: str = "mistral" - OLLAMA_TIMEOUT: int = 120 + OLLAMA_TIMEOUT: int = 180 OLLAMA_CONNECT_TIMEOUT: int = 10 OLLAMA_MAX_RETRIES: int = 2 OLLAMA_RETRY_BACKOFF_SECONDS: float = 1.0 diff --git a/app/infrastructure/ollama_client.py b/app/infrastructure/ollama_client.py index ae50925..35b0046 100644 --- a/app/infrastructure/ollama_client.py +++ b/app/infrastructure/ollama_client.py @@ -29,7 +29,7 @@ def generate(self, user_prompt: str) -> str: url = f"{self._base_url}/api/chat" payload = self._build_payload(user_prompt) with httpx.Client(timeout=self._timeout) as client: - response = client.post(url, json=payload) + response = client.post(url, json=payload, timeout=self._timeout) response.raise_for_status() data = response.json() content = data.get("message", {}).get("content", "") From f2c96661a323ac337fb77924dd6e76e35528adba Mon Sep 17 00:00:00 2001 From: serhii lemeshko Date: Thu, 16 Apr 2026 14:50:12 +0300 Subject: [PATCH 44/52] feat: add rollback logic for database transactions on error in feature specification generation --- app/infrastructure/tasks/feature_spec_tasks.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/app/infrastructure/tasks/feature_spec_tasks.py b/app/infrastructure/tasks/feature_spec_tasks.py index bc9fcf4..f74a62b 100644 --- a/app/infrastructure/tasks/feature_spec_tasks.py +++ b/app/infrastructure/tasks/feature_spec_tasks.py @@ -13,8 +13,16 @@ from app.modules.feature_spec.schemas import FeatureSummaryResult +def _ensure_auth_models_loaded() -> None: + # Import at runtime to register referenced tables in SQLAlchemy metadata. + from app.modules.auth import models as auth_models + + _ = auth_models.User + + @celery_app.task(bind=True, name="feature_spec.generate") def generate_feature_spec_task(self, run_id: int, feature_idea: str, user_id: int) -> dict: + _ensure_auth_models_loaded() db = SessionLocal() try: run = db.get(FeatureSpecRun, run_id) @@ -60,12 +68,14 @@ def generate_feature_spec_task(self, run_id: int, feature_idea: str, user_id: in except MaxRetriesExceededError: pass + db.rollback() run.status = "error" run.error_message = "LLM provider request failed" db.add(run) db.commit() raise RuntimeError("LLM task failed after retries") from exc except Exception as exc: + db.rollback() run.status = "error" run.error_message = "Failed to process feature specification" db.add(run) From 6b5260bcc189a6b2964565be72801ac23053fca1 Mon Sep 17 00:00:00 2001 From: serhii lemeshko Date: Thu, 16 Apr 2026 15:15:20 +0300 Subject: [PATCH 45/52] feat: update README and schemas for async feature generation, enhance Ollama client error handling, and refine requirements --- README.md | 127 +++++++++++++++--- .../tasks/feature_spec_tasks.py | 1 - app/modules/feature_spec/providers/ollama.py | 10 +- app/modules/feature_spec/schemas.py | 36 +++-- requirements.txt | 9 -- 5 files changed, 145 insertions(+), 38 deletions(-) diff --git a/README.md b/README.md index 309d479..89da555 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,11 @@ # Specification Generator API -Production-ready FastAPI backend for authentication, LLM-powered specification generation, health checks, and secure API middleware baseline. +Production-ready FastAPI backend with async LLM generation via Celery + Redis, JWT auth, and Docker Compose orchestration. + +

+ FastAPI +

python @@ -19,7 +23,9 @@ Production-ready FastAPI backend for authentication, LLM-powered specification g - JWT authentication based on fastapi-users - User registration and user management endpoints -- Feature specification generation endpoints powered by Ollama +- Async feature specification generation with Celery tasks +- Redis as Celery broker/result backend +- Ollama integration for LLM responses - Readiness and health probes for runtime checks - Alembic database migrations - Security middleware baseline: @@ -38,10 +44,20 @@ Production-ready FastAPI backend for authentication, LLM-powered specification g - SQLAlchemy 2.0 - Alembic - fastapi-users +- Celery +- Redis - PostgreSQL (via DATABASE_URL) - Ollama (LLM provider) - Pytest +## Architecture (Async Flow) + +1. Client calls `POST /api/v1/feature-spec/generate`. +2. API stores a run row and enqueues Celery task to Redis. +3. API returns immediately with `task_id` and `processing` status. +4. Celery worker calls Ollama and persists success/error in DB. +5. Client polls `GET /api/v1/feature-spec/tasks/{task_id}` for result. + ## Project Structure - app/main.py: FastAPI app bootstrap and router registration @@ -49,10 +65,11 @@ Production-ready FastAPI backend for authentication, LLM-powered specification g - app/api/: health, readiness, OpenAPI customization - app/middlewares/: security middleware composition and implementations - app/modules/auth/: auth domain (models, schemas, dependencies, router) -- app/modules/feature_spec/: feature spec API, schemas, prompts, providers +- app/modules/feature_spec/: API layer + application services for feature spec +- app/infrastructure/: Celery app, task workers, Ollama client - app/scripts/: utility scripts (admin and prompt/model bootstrap) - alembic/: migration config and versions -- docker-compose.yml: containerized app run +- docker-compose.yml: app + celery-worker + redis + ollama ## Setup Guide @@ -60,7 +77,7 @@ Production-ready FastAPI backend for authentication, LLM-powered specification g - Python 3.10+ - PostgreSQL database -- Optional: Docker + Docker Compose (recommended for VPS) +- Docker + Docker Compose (recommended) ### 1) Configure environment @@ -70,6 +87,8 @@ Required minimum: - DATABASE_URL - SECRET_KEY +- CELERY_BROKER_URL +- CELERY_RESULT_BACKEND Recommended auth bootstrap values: @@ -81,10 +100,20 @@ LLM values: - OLLAMA_BASE_URL - OLLAMA_MODEL +- OLLAMA_TIMEOUT + +Compose ports (host -> container): + +- FASTAPI_HOST_PORT=8005 +- FASTAPI_CONTAINER_PORT=8001 +- REDIS_HOST_PORT=6380 +- REDIS_CONTAINER_PORT=6379 For Docker Compose in this project use: - OLLAMA_BASE_URL=http://ollama:11434 +- CELERY_BROKER_URL=redis://redis:6379/0 +- CELERY_RESULT_BACKEND=redis://redis:6379/1 ### 2A) Run locally @@ -102,7 +131,11 @@ pip install -r requirements-dev.txt python -m alembic upgrade head python -m app.scripts.bootstrap_admin python -m app.scripts.bootstrap_prompt_template -python -m uvicorn app.main:app --host 0.0.0.0 --port 8000 +# terminal 1: api +python -m uvicorn app.main:app --host 0.0.0.0 --port 8005 + +# terminal 2: worker +celery -A app.infrastructure.celery_app:celery_app worker --loglevel=INFO ``` ### 2B) Run with Docker @@ -119,28 +152,32 @@ Notes: - prompt template bootstrap script (`python -m app.scripts.bootstrap_prompt_template`) - Ollama model bootstrap (`python -m app.scripts.ensure_ollama_model`) - uvicorn app startup +- Celery worker runs in a dedicated container (`celery-worker`). +- Redis runs only in Docker Compose and is used internally by service name `redis`. - On first deploy, startup may take longer while the configured `OLLAMA_MODEL` is downloaded. - FastAPI container reaches Ollama via internal Docker network URL: http://ollama:11434 -Verify Ollama API: +Verify services: ```bash -curl http://localhost:11434/api/generate -d '{ - "model": "mistral", - "prompt": "hello", - "stream": false -}' +docker compose ps +docker compose logs -f app +docker compose logs -f celery-worker ``` ## API Docs -When server is running locally: +When server is running locally (custom port): -- Swagger UI: http://localhost:8000/docs -- ReDoc: http://localhost:8000/redoc (pinned ReDoc 2.x script) -- OpenAPI JSON: http://localhost:8000/openapi.json +- Swagger UI: http://localhost:8005/docs +- ReDoc: http://localhost:8005/redoc (pinned ReDoc 2.x script) +- OpenAPI JSON: http://localhost:8005/openapi.json -For Docker Compose deployment, use port 8001 instead of 8000. +For Docker Compose deployment (default host mapping): + +- Swagger UI: http://localhost:8005/docs +- ReDoc: http://localhost:8005/redoc +- OpenAPI JSON: http://localhost:8005/openapi.json ## API Endpoints @@ -175,9 +212,10 @@ Login note: ### Feature Spec - POST /api/v1/feature-spec/generate +- GET /api/v1/feature-spec/tasks/{task_id} - GET /api/v1/feature-spec/history?limit=10 -Request body example for generation: +Generate request: ```json { @@ -185,6 +223,45 @@ Request body example for generation: } ``` +Generate response: + +```json +{ + "task_id": "3b44daff-1e83-4328-925f-62c22a9163d2", + "status": "processing" +} +``` + +Task status response examples: + +```json +{ + "task_id": "3b44daff-1e83-4328-925f-62c22a9163d2", + "status": "PENDING" +} +``` + +```json +{ + "task_id": "3b44daff-1e83-4328-925f-62c22a9163d2", + "status": "SUCCESS", + "result": { + "run_id": 10, + "status": "success", + "feature_idea": "payment for premium posts", + "feature_summary": { + "user_stories": [], + "acceptance_criteria": [], + "db_models_and_api_endpoints": { + "db_models": [], + "api_endpoints": [] + }, + "risk_assessment": [] + } + } +} +``` + ## Quality Checks Run linter: @@ -225,8 +302,20 @@ If app cannot connect to DB: - Verify DATABASE_URL - Verify DB network access and sslmode if needed -If LLM requests fail: +If Celery tasks stay in `PENDING`: + +- Check worker is healthy: `docker compose ps` +- Check worker logs: `docker compose logs -f celery-worker` +- Verify Redis URLs in `.env` point to `redis` service inside Docker network + +If LLM requests fail or timeout: - Verify OLLAMA_BASE_URL - Ensure Ollama is running and model is available - For Docker deployment, ensure OLLAMA_BASE_URL is http://ollama:11434 +- Increase `OLLAMA_TIMEOUT` for long generations + +If compose prints `variable is not set` for random token-like names: + +- Your `.env` likely has `$` inside secret values +- Escape `$` as `$$` in `.env` values used by Docker Compose diff --git a/app/infrastructure/tasks/feature_spec_tasks.py b/app/infrastructure/tasks/feature_spec_tasks.py index f74a62b..bd39ff1 100644 --- a/app/infrastructure/tasks/feature_spec_tasks.py +++ b/app/infrastructure/tasks/feature_spec_tasks.py @@ -14,7 +14,6 @@ def _ensure_auth_models_loaded() -> None: - # Import at runtime to register referenced tables in SQLAlchemy metadata. from app.modules.auth import models as auth_models _ = auth_models.User diff --git a/app/modules/feature_spec/providers/ollama.py b/app/modules/feature_spec/providers/ollama.py index 08ff0d3..8e0fcca 100644 --- a/app/modules/feature_spec/providers/ollama.py +++ b/app/modules/feature_spec/providers/ollama.py @@ -54,6 +54,11 @@ async def generate(self, user_prompt: str) -> str: message = data.get("message", {}) content = message.get("content", "") return content if isinstance(content, str) else str(content) + except (json.JSONDecodeError, ValueError) as exc: + if attempt < self._max_retries: + await self._backoff(attempt) + continue + raise RuntimeError("Ollama returned invalid JSON response") from exc except httpx.TimeoutException as exc: if attempt < self._max_retries: await self._backoff(attempt) @@ -90,6 +95,7 @@ async def generate_stream(self, user_prompt: str) -> AsyncGenerator[str, None]: try: async with client.stream("POST", url, json=payload) as response: response.raise_for_status() + stream_done = False async for line in response.aiter_lines(): if not line: continue @@ -103,8 +109,10 @@ async def generate_stream(self, user_prompt: str) -> AsyncGenerator[str, None]: yielded_any = True yield token if chunk.get("done"): + stream_done = True return - return + if not stream_done: + raise RuntimeError("Ollama stream ended before completion") except httpx.TimeoutException as exc: if not yielded_any and attempt < self._max_retries: await self._backoff(attempt) diff --git a/app/modules/feature_spec/schemas.py b/app/modules/feature_spec/schemas.py index bcde2a7..07f3d43 100644 --- a/app/modules/feature_spec/schemas.py +++ b/app/modules/feature_spec/schemas.py @@ -36,6 +36,25 @@ class FeatureSummaryResult(BaseModel): db_models_and_api_endpoints: DbModelsAndApiEndpoints risk_assessment: list[str] + @staticmethod + def _is_missing(value: Any) -> bool: + if value is None: + return True + if isinstance(value, str): + return not value.strip() + if isinstance(value, (list, dict)): + return not value + return False + + @staticmethod + def _coerce_legacy_string_list(value: Any) -> list[str] | list[Any] | None: + if isinstance(value, str): + stripped = value.strip() + return [stripped] if stripped else None + if isinstance(value, list): + return value + return None + @model_validator(mode="before") @classmethod def normalize_legacy_user_stories(cls, data): @@ -47,19 +66,20 @@ def normalize_legacy_user_stories(cls, data): if "user_stories" not in normalized and "feature_summary_items" in normalized: normalized["user_stories"] = normalized["feature_summary_items"] - if "acceptance_criteria" not in normalized: - if "acceptance" in normalized: - normalized["acceptance_criteria"] = normalized["acceptance"] + if cls._is_missing(normalized.get("acceptance_criteria")): + legacy_acceptance = cls._coerce_legacy_string_list(normalized.get("acceptance")) + if legacy_acceptance is not None: + normalized["acceptance_criteria"] = legacy_acceptance if "db_models_and_api_endpoints" not in normalized: db_models = normalized.get("db_models", []) api_endpoints = normalized.get("api_endpoints", []) - normalized["db_models_and_api_endpoints"] = { - "db_models": db_models, - "api_endpoints": api_endpoints, - } + normalized["db_models_and_api_endpoints"] = DbModelsAndApiEndpoints( + db_models=db_models, + api_endpoints=api_endpoints, + ).model_dump() - if "risk_assessment" not in normalized and "risks" in normalized: + if cls._is_missing(normalized.get("risk_assessment")) and "risks" in normalized: normalized["risk_assessment"] = normalized["risks"] return normalized diff --git a/requirements.txt b/requirements.txt index 2f846be..da29dc2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,20 +1,15 @@ -# FastAPI core fastapi==0.111.0 uvicorn[standard]==0.30.0 -# Database sqlalchemy==2.0.30 psycopg2-binary==2.9.9 -# Migrations alembic==1.13.1 -# Data validation pydantic==2.7.1 pydantic-settings==2.2.1 -# Auth / security python-multipart==0.0.20 fastapi-users==14.0.1 fastapi-users-db-sqlalchemy[sqlalchemy]==7.0.0 @@ -23,15 +18,11 @@ asyncpg==0.30.0 argon2-cffi==23.1.0 -# Environment python-dotenv==1.0.1 -# HTTP requests httpx==0.27.0 -# Background processing celery[redis]==5.4.0 -# Admin panel sqladmin==0.17.0 itsdangerous==2.2.0 From 7d24c19774adfa83c146b5a7ad12a293d52078b7 Mon Sep 17 00:00:00 2001 From: serhii lemeshko Date: Thu, 16 Apr 2026 15:25:51 +0300 Subject: [PATCH 46/52] feat: add broker connection retry on startup in Celery configuration --- app/infrastructure/celery_app.py | 1 + 1 file changed, 1 insertion(+) diff --git a/app/infrastructure/celery_app.py b/app/infrastructure/celery_app.py index ab63472..bc5b403 100644 --- a/app/infrastructure/celery_app.py +++ b/app/infrastructure/celery_app.py @@ -11,6 +11,7 @@ ) celery_app.conf.update( + broker_connection_retry_on_startup=True, task_track_started=True, task_serializer="json", result_serializer="json", From b1d5b10103f6e1d030935f8c2e55519233019738 Mon Sep 17 00:00:00 2001 From: serhii lemeshko Date: Thu, 16 Apr 2026 15:41:44 +0300 Subject: [PATCH 47/52] feat: integrate normalization of feature summary payload and enhance response parsing --- app/modules/feature_spec/orchestrator.py | 5 +- app/modules/feature_spec/parser.py | 75 +++++++++++++++++++ .../feature_spec/prompts/feature_summary.py | 10 ++- app/modules/feature_spec/schemas.py | 50 +------------ 4 files changed, 88 insertions(+), 52 deletions(-) diff --git a/app/modules/feature_spec/orchestrator.py b/app/modules/feature_spec/orchestrator.py index 79f7f8b..b3f2a30 100644 --- a/app/modules/feature_spec/orchestrator.py +++ b/app/modules/feature_spec/orchestrator.py @@ -3,6 +3,7 @@ from app.core.settings import settings from app.modules.feature_spec.models import FeatureSpecRun +from app.modules.feature_spec.parser import normalize_feature_summary_payload from app.modules.feature_spec.prompts.feature_summary import ( build_feature_summary_prompt_from_db, parse_feature_summary_response, @@ -92,7 +93,9 @@ def get_feature_spec_history( feature_idea=row.feature_idea, status=row.status, response_json=( - FeatureSummaryResult.model_validate(row.response_json) + FeatureSummaryResult.model_validate( + normalize_feature_summary_payload(row.response_json) + ) if row.response_json is not None else None ), diff --git a/app/modules/feature_spec/parser.py b/app/modules/feature_spec/parser.py index de7a156..3e8812d 100644 --- a/app/modules/feature_spec/parser.py +++ b/app/modules/feature_spec/parser.py @@ -2,6 +2,81 @@ import re +def _is_missing(value) -> bool: + if value is None: + return True + if isinstance(value, str): + return not value.strip() + if isinstance(value, (list, dict)): + return not value + return False + + +def _normalize_acceptance_criteria(value) -> list[str] | None: + if value is None: + return None + + items = value if isinstance(value, list) else [value] + normalized_items: list[str] = [] + + for item in items: + if isinstance(item, str): + text = item.strip() + elif isinstance(item, dict): + text = next( + ( + candidate.strip() + for key in ( + "title", + "description", + "details", + "criterion", + "text", + "message", + ) + if isinstance((candidate := item.get(key)), str) + and candidate.strip() + ), + str(item).strip(), + ) + else: + text = str(item).strip() + + if text: + normalized_items.append(text) + + return normalized_items or None + + +def normalize_feature_summary_payload(data: dict | list) -> dict | list: + if not isinstance(data, dict): + return data + + normalized = dict(data) + + if "user_stories" not in normalized and "feature_summary_items" in normalized: + normalized["user_stories"] = normalized["feature_summary_items"] + + acceptance_source = normalized.get("acceptance_criteria") + if _is_missing(acceptance_source): + acceptance_source = normalized.get("acceptance") + + normalized_acceptance = _normalize_acceptance_criteria(acceptance_source) + if normalized_acceptance is not None: + normalized["acceptance_criteria"] = normalized_acceptance + + if "db_models_and_api_endpoints" not in normalized: + normalized["db_models_and_api_endpoints"] = { + "db_models": normalized.get("db_models", []), + "api_endpoints": normalized.get("api_endpoints", []), + } + + if _is_missing(normalized.get("risk_assessment")) and "risks" in normalized: + normalized["risk_assessment"] = normalized["risks"] + + return normalized + + def extract_json(text: str) -> dict | list: decoder = json.JSONDecoder() for index, char in enumerate(text): diff --git a/app/modules/feature_spec/prompts/feature_summary.py b/app/modules/feature_spec/prompts/feature_summary.py index 895889e..9deb419 100644 --- a/app/modules/feature_spec/prompts/feature_summary.py +++ b/app/modules/feature_spec/prompts/feature_summary.py @@ -4,7 +4,12 @@ from sqlalchemy.orm import Session from app.modules.feature_spec.models import PromptTemplate -from app.modules.feature_spec.parser import extract_json, normalize_whitespace, strip_markdown +from app.modules.feature_spec.parser import ( + extract_json, + normalize_feature_summary_payload, + normalize_whitespace, + strip_markdown, +) def build_feature_summary_prompt_from_template(template: str, feature_idea: str) -> str: @@ -37,4 +42,5 @@ def build_feature_summary_prompt_from_db(feature_idea: str, db: Session) -> str: def parse_feature_summary_response(raw_response: str) -> dict | list: normalized_content = normalize_whitespace(strip_markdown(raw_response)) - return extract_json(normalized_content) + parsed = extract_json(normalized_content) + return normalize_feature_summary_payload(parsed) diff --git a/app/modules/feature_spec/schemas.py b/app/modules/feature_spec/schemas.py index 07f3d43..43eb23f 100644 --- a/app/modules/feature_spec/schemas.py +++ b/app/modules/feature_spec/schemas.py @@ -1,7 +1,7 @@ from datetime import datetime from typing import Any, Literal -from pydantic import BaseModel, Field, field_validator, model_validator +from pydantic import BaseModel, Field, field_validator from app.core.settings import settings @@ -36,54 +36,6 @@ class FeatureSummaryResult(BaseModel): db_models_and_api_endpoints: DbModelsAndApiEndpoints risk_assessment: list[str] - @staticmethod - def _is_missing(value: Any) -> bool: - if value is None: - return True - if isinstance(value, str): - return not value.strip() - if isinstance(value, (list, dict)): - return not value - return False - - @staticmethod - def _coerce_legacy_string_list(value: Any) -> list[str] | list[Any] | None: - if isinstance(value, str): - stripped = value.strip() - return [stripped] if stripped else None - if isinstance(value, list): - return value - return None - - @model_validator(mode="before") - @classmethod - def normalize_legacy_user_stories(cls, data): - if not isinstance(data, dict): - return data - - normalized = dict(data) - - if "user_stories" not in normalized and "feature_summary_items" in normalized: - normalized["user_stories"] = normalized["feature_summary_items"] - - if cls._is_missing(normalized.get("acceptance_criteria")): - legacy_acceptance = cls._coerce_legacy_string_list(normalized.get("acceptance")) - if legacy_acceptance is not None: - normalized["acceptance_criteria"] = legacy_acceptance - - if "db_models_and_api_endpoints" not in normalized: - db_models = normalized.get("db_models", []) - api_endpoints = normalized.get("api_endpoints", []) - normalized["db_models_and_api_endpoints"] = DbModelsAndApiEndpoints( - db_models=db_models, - api_endpoints=api_endpoints, - ).model_dump() - - if cls._is_missing(normalized.get("risk_assessment")) and "risks" in normalized: - normalized["risk_assessment"] = normalized["risks"] - - return normalized - class FeatureSpecGenerateResponse(BaseModel): feature_idea: str From 25329932164e44e1ba9eeeac3cc6ce9663024211 Mon Sep 17 00:00:00 2001 From: serhii lemeshko Date: Thu, 16 Apr 2026 15:51:42 +0300 Subject: [PATCH 48/52] feat: implement normalization for risk assessment and integrate it into feature summary payload --- app/modules/feature_spec/parser.py | 38 ++++++++++++++++++++++++++++-- 1 file changed, 36 insertions(+), 2 deletions(-) diff --git a/app/modules/feature_spec/parser.py b/app/modules/feature_spec/parser.py index 3e8812d..7abf424 100644 --- a/app/modules/feature_spec/parser.py +++ b/app/modules/feature_spec/parser.py @@ -48,6 +48,35 @@ def _normalize_acceptance_criteria(value) -> list[str] | None: return normalized_items or None +def _normalize_risk_assessment(value) -> list[str] | None: + if value is None: + return None + + items = value if isinstance(value, list) else [value] + normalized_items: list[str] = [] + + for item in items: + if isinstance(item, str): + text = item.strip() + elif isinstance(item, dict): + text = next( + ( + candidate.strip() + for key in ("title", "risk", "description", "details", "message") + if isinstance((candidate := item.get(key)), str) + and candidate.strip() + ), + str(item).strip(), + ) + else: + text = str(item).strip() + + if text: + normalized_items.append(text) + + return normalized_items or None + + def normalize_feature_summary_payload(data: dict | list) -> dict | list: if not isinstance(data, dict): return data @@ -71,8 +100,13 @@ def normalize_feature_summary_payload(data: dict | list) -> dict | list: "api_endpoints": normalized.get("api_endpoints", []), } - if _is_missing(normalized.get("risk_assessment")) and "risks" in normalized: - normalized["risk_assessment"] = normalized["risks"] + risk_source = normalized.get("risk_assessment") + if _is_missing(risk_source): + risk_source = normalized.get("risks") + + normalized_risks = _normalize_risk_assessment(risk_source) + if normalized_risks is not None: + normalized["risk_assessment"] = normalized_risks return normalized From 9081429bca40b05b6f9abd5fcad5eb6c28fef2b3 Mon Sep 17 00:00:00 2001 From: serhii lemeshko Date: Thu, 16 Apr 2026 15:58:07 +0300 Subject: [PATCH 49/52] feat: enhance normalization functions with strict validation and logging --- app/modules/feature_spec/parser.py | 183 ++++++++++++++++++++++++++--- 1 file changed, 165 insertions(+), 18 deletions(-) diff --git a/app/modules/feature_spec/parser.py b/app/modules/feature_spec/parser.py index 7abf424..5b71088 100644 --- a/app/modules/feature_spec/parser.py +++ b/app/modules/feature_spec/parser.py @@ -1,8 +1,13 @@ import json +import logging import re +from typing import Any -def _is_missing(value) -> bool: +logger = logging.getLogger(__name__) + + +def _is_missing(value: Any) -> bool: if value is None: return True if isinstance(value, str): @@ -12,7 +17,7 @@ def _is_missing(value) -> bool: return False -def _normalize_acceptance_criteria(value) -> list[str] | None: +def _normalize_acceptance_criteria(value: Any, *, strict: bool = False) -> list[str] | None: if value is None: return None @@ -44,11 +49,13 @@ def _normalize_acceptance_criteria(value) -> list[str] | None: if text: normalized_items.append(text) + elif strict: + raise ValueError("Invalid acceptance_criteria item") return normalized_items or None -def _normalize_risk_assessment(value) -> list[str] | None: +def _normalize_risk_assessment(value: Any, *, strict: bool = False) -> list[str] | None: if value is None: return None @@ -73,38 +80,161 @@ def _normalize_risk_assessment(value) -> list[str] | None: if text: normalized_items.append(text) + elif strict: + raise ValueError("Invalid risk_assessment item") + + return normalized_items or None + + +def _normalize_mixed_items(value: Any, *, strict: bool = False) -> list[str | dict] | None: + if value is None: + return None + + items = value if isinstance(value, list) else [value] + normalized_items: list[str | dict] = [] + + for item in items: + if isinstance(item, dict): + normalized_items.append(item) + continue + + text = str(item).strip() + if text: + normalized_items.append(text) + elif strict: + raise ValueError("Invalid db/api item") + + return normalized_items or None + + +def _first_non_empty(source: dict[str, Any], *keys: str) -> str | None: + for key in keys: + value = source.get(key) + if isinstance(value, str) and value.strip(): + return value.strip() + return None + + +def _normalize_user_stories(value: Any, *, strict: bool = False) -> list[dict[str, str]] | None: + if value is None: + return None + + items = value if isinstance(value, list) else [value] + normalized_items: list[dict] = [] + + for index, item in enumerate(items, start=1): + if isinstance(item, dict): + title = _first_non_empty(item, "title", "name") + as_a = _first_non_empty(item, "as_a", "as", "actor", "user") + i_want = _first_non_empty(item, "i_want", "want", "goal", "objective") + so_that = _first_non_empty(item, "so_that", "benefit", "because", "outcome") + if not i_want: + i_want = _first_non_empty(item, "description", "details", "text") + if not so_that: + so_that = i_want + + if not all([title, as_a, i_want, so_that]): + if strict: + raise ValueError(f"Invalid user story at index {index}") + logger.warning("Skipping invalid user story item at index %s", index) + continue + + normalized_items.append( + { + "title": title, + "as_a": as_a, + "i_want": i_want, + "so_that": so_that, + } + ) + continue + + if isinstance(item, str) and item.strip(): + text = item.strip() + if strict: + raise ValueError( + f"Invalid user story string item at index {index}; object expected" + ) + normalized_items.append( + { + "title": text, + "as_a": "User", + "i_want": text, + "so_that": text, + } + ) + elif strict: + raise ValueError(f"Invalid user story item type at index {index}") return normalized_items or None -def normalize_feature_summary_payload(data: dict | list) -> dict | list: +def normalize_feature_summary_payload( + data: dict[str, Any] | list, + *, + strict: bool = False, +) -> dict[str, Any] | list: if not isinstance(data, dict): return data normalized = dict(data) - if "user_stories" not in normalized and "feature_summary_items" in normalized: - normalized["user_stories"] = normalized["feature_summary_items"] + if isinstance(normalized.get("feature_summary"), dict): + nested = normalized["feature_summary"] + logger.info("Using nested feature_summary payload") + for key in ( + "user_stories", + "feature_summary_items", + "acceptance_criteria", + "acceptance", + "db_models_and_api_endpoints", + "db_models", + "api_endpoints", + "risk_assessment", + "risks", + ): + if key not in normalized and key in nested: + normalized[key] = nested[key] + + user_stories_source = normalized.get("user_stories") + if _is_missing(user_stories_source): + user_stories_source = normalized.get("feature_summary_items") + + normalized_user_stories = _normalize_user_stories(user_stories_source, strict=strict) + if normalized_user_stories is not None: + normalized["user_stories"] = normalized_user_stories acceptance_source = normalized.get("acceptance_criteria") if _is_missing(acceptance_source): + logger.info("Using legacy acceptance field") acceptance_source = normalized.get("acceptance") - normalized_acceptance = _normalize_acceptance_criteria(acceptance_source) + normalized_acceptance = _normalize_acceptance_criteria( + acceptance_source, + strict=strict, + ) if normalized_acceptance is not None: normalized["acceptance_criteria"] = normalized_acceptance - if "db_models_and_api_endpoints" not in normalized: - normalized["db_models_and_api_endpoints"] = { - "db_models": normalized.get("db_models", []), - "api_endpoints": normalized.get("api_endpoints", []), - } + db_api_source = normalized.get("db_models_and_api_endpoints") + db_source = normalized.get("db_models") + api_source = normalized.get("api_endpoints") + + if isinstance(db_api_source, dict): + db_source = db_api_source.get("db_models", db_source) + api_source = db_api_source.get("api_endpoints", api_source) + + normalized["db_models_and_api_endpoints"] = { + "db_models": _normalize_mixed_items(db_source, strict=strict) or [], + "api_endpoints": _normalize_mixed_items(api_source, strict=strict) or [], + } risk_source = normalized.get("risk_assessment") if _is_missing(risk_source): + logger.info("Using legacy risks field") risk_source = normalized.get("risks") - normalized_risks = _normalize_risk_assessment(risk_source) + normalized_risks = _normalize_risk_assessment(risk_source, strict=strict) if normalized_risks is not None: normalized["risk_assessment"] = normalized_risks @@ -112,16 +242,33 @@ def normalize_feature_summary_payload(data: dict | list) -> dict | list: def extract_json(text: str) -> dict | list: - decoder = json.JSONDecoder() - for index, char in enumerate(text): - if char not in "[{": - continue + stripped = text.strip() + if stripped: try: - parsed, _ = decoder.raw_decode(text[index:]) + parsed = json.loads(stripped) + if isinstance(parsed, (dict, list)): + return parsed + except json.JSONDecodeError: + pass + + fenced_match = re.search(r"```(?:json)?\s*(.*?)\s*```", text, re.DOTALL) + if fenced_match: + candidate = fenced_match.group(1).strip() + try: + parsed = json.loads(candidate) + if isinstance(parsed, (dict, list)): + return parsed + except json.JSONDecodeError: + pass + + for candidate in re.findall(r"\{[\s\S]*?\}|\[[\s\S]*?\]", text): + try: + parsed = json.loads(candidate) except json.JSONDecodeError: continue if isinstance(parsed, (dict, list)): return parsed + raise ValueError("No JSON found in LLM response") From 0e1db1b5eb27b414de5facc08ec3097341d88bff Mon Sep 17 00:00:00 2001 From: Serhii Lemeshko <135651866+opsmanager1@users.noreply.github.com> Date: Thu, 16 Apr 2026 16:53:56 +0300 Subject: [PATCH 50/52] Update README.md --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index 89da555..8d06abb 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,9 @@ Production-ready FastAPI backend with async LLM generation via Celery + Redis, J alembic tests docs + celery + redis + ollama

From d2dedda487b5593cfb765e7644de0b14b3ef7e5c Mon Sep 17 00:00:00 2001 From: Serhii Lemeshko <135651866+opsmanager1@users.noreply.github.com> Date: Thu, 16 Apr 2026 16:54:32 +0300 Subject: [PATCH 51/52] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 8d06abb..5dfc508 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@
-# Specification Generator API +# Feature Specification Generator API Production-ready FastAPI backend with async LLM generation via Celery + Redis, JWT auth, and Docker Compose orchestration. From 91fedc8de250ae0bf7b7a81f25a0306edb9a7a13 Mon Sep 17 00:00:00 2001 From: Serhii Lemeshko <135651866+opsmanager1@users.noreply.github.com> Date: Thu, 16 Apr 2026 21:43:18 +0300 Subject: [PATCH 52/52] Update README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 5dfc508..c6fe7fe 100644 --- a/README.md +++ b/README.md @@ -49,8 +49,8 @@ Production-ready FastAPI backend with async LLM generation via Celery + Redis, J - fastapi-users - Celery - Redis -- PostgreSQL (via DATABASE_URL) -- Ollama (LLM provider) +- PostgreSQL +- Ollama - Pytest ## Architecture (Async Flow)