Feature/logging

.vscode/settings.json

@@ -1 +1,11 @@
-{}
+{
+  "dotnet.defaultSolution": "APITemplate.slnx",
+  "dotnet.solution.autoOpen": "APITemplate.slnx",
+  "dotnet.enableWorkspaceBasedDevelopment": false,
+  "files.exclude": {
+    "**/monolith": true
+  },
+  "search.exclude": {
+    "**/monolith": true
+  }
+}

APITemplate.Microservices.code-workspace

@@ -0,0 +1,42 @@
+{
+  "folders": [
+    {
+      "name": "src",
+      "path": "src"
+    },
+    {
+      "name": "tests",
+      "path": "tests"
+    },
+    {
+      "name": "docs",
+      "path": "docs"
+    },
+    {
+      "name": "infrastructure",
+      "path": "infrastructure"
+    }
+  ],
+  "settings": {
+    "dotnet.defaultSolution": "APITemplate.slnx",
+    "dotnet.solution.autoOpen": "APITemplate.slnx",
+    "dotnet.enableWorkspaceBasedDevelopment": false,
+    "git.openRepositoryInParentFolders": "always",
+    "files.exclude": {
+      "**/monolith": true,
+      "**/bin": true,
+      "**/obj": true
+    },
+    "search.exclude": {
+      "**/monolith": true,
+      "**/bin": true,
+      "**/obj": true
+    }
+  },
+  "extensions": {
+    "recommendations": [
+      "ms-dotnettools.csdevkit",
+      "ms-dotnettools.csharp"
+    ]
+  }
+}

APITemplate.slnx

@@ -28,7 +28,7 @@
   </Folder>
 
   <!-- ================================================================
-       SOURCE — monolith + microservices
+       SOURCE — microservices
        ================================================================ -->
   <Folder Name="/src/">
 

Directory.Packages.props

@@ -68,6 +68,7 @@
     <PackageVersion Include="OpenTelemetry.Instrumentation.Runtime" Version="1.15.0" />
     <PackageVersion Include="OpenTelemetry.Instrumentation.StackExchangeRedis" Version="1.15.0-beta.1" />
     <PackageVersion Include="Scalar.AspNetCore" Version="2.13.16" />
+    <PackageVersion Include="Serilog" Version="4.3.0" />
     <PackageVersion Include="Serilog.AspNetCore" Version="10.0.0" />
     <PackageVersion Include="Serilog.Enrichers.Environment" Version="3.0.1" />
     <PackageVersion Include="Serilog.Enrichers.Thread" Version="4.0.0" />

TODO-Architecture.md

@@ -28,6 +28,119 @@ All 7 microservices extracted from the monolith and running independently.
 ### Gateway — Demo / Example APIs
 - Expose monolith-style demo endpoints (`IdempotentController`, `PatchController`, `SseController` under `src/APITemplate.Api/`) via a dedicated Examples service or Gateway host, and register routes in YARP.
 
+### Efficiency — Must Fix (pre-production)
+
+- **RabbitMQ health check connection churn** (`SharedKernel.Messaging/HealthChecks/RabbitMqHealthCheck.cs`):
+  Every health check poll (30s × 7 services) creates a brand-new TCP+AMQP connection and immediately disposes it. Reuse the existing Wolverine-managed connection or cache a single health-check connection.
+
+- **Webhook delivery is sequential per subscriber** (`Webhooks.Infrastructure/Delivery/WebhookDeliveryService.cs:52-56`):
+  `foreach` + `await` for each subscriber = N sequential HTTP round-trips. Each delivery log is also persisted individually (N separate DB writes). Use `Parallel.ForEachAsync` with bounded concurrency and batch `SaveChangesAsync` after all deliveries.
+
+- **No PostgreSQL connection pooling configuration** (all service `Program.cs` + `docker-compose.microservices.yml`):
+  7 services + Wolverine persistence = up to 700 simultaneous connections (default Npgsql pool = 100). Add `Maximum Pool Size=25;Minimum Pool Size=2;Connection Idle Lifetime=60` to connection strings.
+
+- **Docker builds copy entire repository** (all `Dockerfile`s):
+  `COPY . .` invalidates cache on any file change in any service. Split into csproj-only COPY for restore layer, then full copy for build. Use `.dockerignore` to exclude unrelated services.
+
+### Efficiency — Should Fix
+
+- **Gateway blocks startup until ALL 7 services healthy** (`docker-compose.microservices.yml:118-133`):
+  YARP handles upstream failures gracefully. Remove `depends_on: service_healthy` conditions — let Gateway start immediately and route to healthy backends dynamically.
+
+- **FileStorage missing TenantId index** (`FileStorage.Infrastructure/Persistence/Configurations/StoredFileConfiguration.cs`):
+  No indexes beyond PK. Global query filter on `TenantId` causes full table scans. All other services have `TenantId` indexes — FileStorage is the only one missing it.
+
+- **`MigrateDbAsync` blocks startup** (`SharedKernel.Api/Extensions/HostExtensions.cs`):
+  7 services run migrations concurrently against shared PostgreSQL → lock contention. Consider advisory lock or sequential migration orchestration.
+
+- **Sequential saga completion publishes** (`ProductCatalog.Application/EventHandlers/TenantDeactivatedEventHandler.cs:53-58`):
+  Two independent `bus.PublishAsync` calls awaited sequentially. Use Wolverine's `OutgoingMessages` cascading pattern (already used elsewhere via `CacheInvalidationCascades`).
+
+- **ChangeTracker `.ToList()` on every SaveChanges** (`SharedKernel.Infrastructure/Persistence/TenantAuditableDbContext.cs:70-76`):
+  Materializes all change-tracked entries on every `SaveChangesAsync` (hot path). Needed because loop can modify collection during soft-delete, but could split into two passes: streaming for non-Delete, `.ToList()` only for Delete.
+
+- **`FluentValidationActionFilter` uses reflection per request** (`SharedKernel.Api/Filters/Validation/FluentValidationActionFilter.cs:38`):
+  `MakeGenericType` + DI lookup on every request, for every action argument. Cache resolved validators in `ConcurrentDictionary<Type, IValidator?>`.
+
+- **`CacheInvalidationCascades.None` returns shared mutable instance** (`SharedKernel.Application/Common/Events/CacheInvalidationCascades.cs:12-13`):
+  `OutgoingMessages` inherits `List<object>` — if any caller accidentally adds to the shared `None` singleton, it corrupts all consumers. Return `new OutgoingMessages()` each time or use a read-only wrapper.
+
+### DRY — Service Registration
+
+- **Split `AddSharedInfrastructure<TDbContext>`** (`SharedKernel.Api/Extensions/SharedServiceRegistration.cs`):
+  Extract non-generic `AddSharedCoreServices()` (TimeProvider, HttpContextAccessor, context providers, error handling, versioning) so Webhooks/Notifications can call it without needing tenant-aware DbContext. The generic `AddSharedInfrastructure<TDbContext>()` then calls `AddSharedCoreServices()` plus UoW/audit/soft-delete.
+
+- **Move `DbContext` base-type registration into `AddSharedInfrastructure<TDbContext>`**:
+  `services.AddScoped<DbContext>(sp => sp.GetRequiredService<TDbContext>())` is copy-pasted in 5 services (Identity, ProductCatalog, Reviews, FileStorage, BackgroundJobs). The generic type parameter is already available in `AddSharedInfrastructure<TDbContext>`.
+
+- **Move `IRolePermissionMap` registration into `AddSharedAuthorization`**:
+  `AddSingleton<IRolePermissionMap, DefaultRolePermissionMap>()` is duplicated in 4 services. Register it automatically when `enablePermissionPolicies: true`.
+
+- **Automate `HasQueryFilter` for `IAuditableTenantEntity`** in `TenantAuditableDbContext.OnModelCreating`:
+  The identical filter expression `(!HasTenant || e.TenantId == CurrentTenantId) && !e.IsDeleted` is copy-pasted 8× across 4 DbContexts. Scan for all entities implementing `IAuditableTenantEntity` and apply automatically. Subclasses opt out for special cases only.
+
+- **Wolverine bootstrap helper** (`SharedKernel.Messaging`):
+  Add `opts.ApplySharedWolverineDefaults(connectionString)` that bundles conventions + retry + PostgreSQL persistence + EF transactions. 5 services repeat the same 4-line preamble.
+
+### DRY — Code Quality
+
+- **Use existing `TenantAuditableDbContextDependencies` parameter object**:
+  The record was created but never used. All 4 derived DbContexts manually forward 7 constructor parameters to `base(...)`. Switch constructors to accept the single parameter object.
+
+- **Shared `DesignTimeDbContextFactoryBase<TContext>`** with NullObject implementations:
+  4 identical DesignTime factories pass `null!` for `tenantProvider`, `actorProvider`, `entityStateManager`, `softDeleteProcessor`. Runtime NRE risk if any EF tooling path triggers `SaveChangesAsync`.
+
+- **Collapse webhook event handlers into a generic handler** (`Webhooks.Application/Features/Delivery/EventHandlers/`):
+  4 handlers (`ProductCreated`, `ProductDeleted`, `ReviewCreated`, `CategoryDeleted`) are identical: log → serialize → deliver. Extract common `ITenantEvent` interface and one generic handler.
+
+- **Saga `NotFound` helper** (`TenantDeactivationSaga`, `ProductDeletionSaga`):
+  7 identical `NotFound` methods across 2 sagas. Extract shared helper or Wolverine convention.
+
+- **Webhooks resilience pipeline key** (`Webhooks.Api/Program.cs:53`):
+  Raw string `"outgoing-webhook-retry"` — other services use typed constants. Add to a constants class.
+
+### Architectural Consistency
+
+- **Webhooks bypasses UnitOfWork / tenant auditing** (`Webhooks.Infrastructure/`):
+  Repositories call `_dbContext.SaveChangesAsync()` directly instead of `IUnitOfWork.CommitAsync()`. Entities implement `IAuditableTenantEntity` but use plain `DbContext` → no audit stamping, no query filters, no soft-delete processing. Manual tenant filtering in queries. Risk: deleted webhook subscriptions could be served.
+
+- **BackgroundJobs registers unused shared infrastructure** (`BackgroundJobs.Api/Program.cs:56`):
+  Calls `AddSharedInfrastructure<BackgroundJobsDbContext>` but `BackgroundJobsDbContext` extends raw `DbContext`, not `TenantAuditableDbContext`. UoW, auditing, and soft-delete services are registered but never invoked.
+
+- **Notifications skips API versioning** (`Notifications.Api/Program.cs`):
+  Does not call `AddSharedInfrastructure` → misses API versioning registration that all other services get. Calls `AddSharedApiErrorHandling()` separately.
+
+- **Redundant queue interface layer in Notifications Domain**:
+  `IQueue<T>` and `IQueueReader<T>` in `Notifications.Domain.Interfaces` are empty marker interfaces re-exporting SharedKernel interfaces. `IEmailQueue`/`IEmailQueueReader` could extend SharedKernel directly (like `BackgroundJobs.Application.Common.IJobQueue` already does).
+
+- **Reviews `ProductProjection` inconsistency**:
+  Uses `IsActive` flag instead of `IsDeleted` + `AuditInfo` pattern used everywhere else. Semantically different from the rest of the system.
+
+### Testing
+
+- **SQLite test setup boilerplate** (`tests/Identity.Tests/`, `tests/ProductCatalog.Tests/`):
+  Identical SQLite connection + DbContext + `EnsureCreated()` + `IDisposable` teardown copy-pasted across test classes. Add `SqliteTestDbContextFactory<TContext>` to `Tests.Common`.
+
+- **Per-service `TestDbContext` duplication** (`tests/Identity.Tests/TestDbContext.cs`, `tests/ProductCatalog.Tests/TestDbContext.cs`, `tests/Reviews.Tests/TestDbContext.cs`):
+  Identical `OwnsOne(Audit)` + key convention per entity. Add `TestModelBuilderExtensions.ConfigureTestAuditableEntity<T>()` to `Tests.Common`.
+
+### Architectural Decision Review
+
+> **Question to evaluate:** Is microservices the right architecture for this project's scale?
+>
+> **Evidence against:**
+> - SharedKernel = 6717 LOC, all services combined = 3537 LOC (without migrations). Shared code is 1.9× larger than business code.
+> - 7 services × 4 layers = 28 projects + 5 SharedKernel + 1 Contracts + 1 Gateway = 35 projects (was 5 in monolith — 7× increase).
+> - All services are structurally identical (same middleware, same auth, same persistence patterns).
+> - `RabbitMqTopology` is a static map in SharedKernel — changing a queue name requires coordinated deployment.
+> - Saga timeouts accept partial completion silently (`MarkCompleted()` on timeout) — was a single DB transaction in the monolith.
+> - Local dev requires 16+ containers vs 8 for monolith.
+> - No documented drivers (independent scaling, deployment cadence, team boundaries, technology heterogeneity).
+>
+> **Alternative considered:** Modular monolith — same boundary enforcement (separate assemblies per bounded context, explicit integration event contracts, Wolverine as in-process bus) at a fraction of operational cost. Extract individual services later when concrete production metrics justify it.
+>
+> **Decision:** [TO BE EVALUATED]
+
 ### Optional polish
 - **Output cache:** ProductCatalog, Identity, Reviews, and FileStorage use `AddSharedOutputCaching` + `[OutputCache]` + invalidation. BackgroundJobs still runs with `useOutputCaching: false` (it has a GET on `JobsController`); enable caching there only if you want read responses cached like the other APIs.
 

global.json

@@ -1,7 +1,7 @@
 {
   "sdk": {
-    "version": "10.0.0",
-    "rollForward": "latestMajor",
+    "version": "10.0.100",
+    "rollForward": "latestFeature",
     "allowPrerelease": true
   }
-}
+}

src/Gateway/Gateway.Api/appsettings.Production.json

@@ -0,0 +1,6 @@
+{
+  "Redaction": {
+    "HmacKeyEnvironmentVariable": "APITEMPLATE_REDACTION_HMAC_KEY",
+    "KeyId": 1001
+  }
+}

src/Services/BackgroundJobs/BackgroundJobs.Api/appsettings.Production.json

@@ -0,0 +1,6 @@
+{
+  "Redaction": {
+    "HmacKeyEnvironmentVariable": "APITEMPLATE_REDACTION_HMAC_KEY",
+    "KeyId": 1001
+  }
+}

src/Services/FileStorage/FileStorage.Api/appsettings.Production.json

@@ -0,0 +1,6 @@
+{
+  "Redaction": {
+    "HmacKeyEnvironmentVariable": "APITEMPLATE_REDACTION_HMAC_KEY",
+    "KeyId": 1001
+  }
+}

src/Services/Identity/Identity.Api/appsettings.Production.json

@@ -0,0 +1,6 @@
+{
+  "Redaction": {
+    "HmacKeyEnvironmentVariable": "APITEMPLATE_REDACTION_HMAC_KEY",
+    "KeyId": 1001
+  }
+}

src/Services/Identity/Identity.Infrastructure/Security/Keycloak/KeycloakAdminServiceLogs.cs

@@ -0,0 +1,84 @@
+using Microsoft.Extensions.Logging;
+using SharedKernel.Infrastructure.Logging;
+
+namespace Identity.Infrastructure.Security.Keycloak;
+
+internal static partial class KeycloakAdminServiceLogs
+{
+    [LoggerMessage(
+        EventId = 2101,
+        Level = LogLevel.Information,
+        Message = "Created Keycloak user {Username} with id {KeycloakUserId}"
+    )]
+    public static partial void UserCreated(
+        this ILogger logger,
+        [PersonalData] string username,
+        [SensitiveData] string keycloakUserId
+    );
+
+    [LoggerMessage(
+        EventId = 2102,
+        Level = LogLevel.Warning,
+        Message = "Failed to send setup email for Keycloak user {KeycloakUserId}. User was created but has no setup email."
+    )]
+    public static partial void SetupEmailFailed(
+        this ILogger logger,
+        Exception exception,
+        [SensitiveData] string keycloakUserId
+    );
+
+    [LoggerMessage(
+        EventId = 2103,
+        Level = LogLevel.Information,
+        Message = "Sent password reset email to Keycloak user {KeycloakUserId}"
+    )]
+    public static partial void PasswordResetEmailSent(
+        this ILogger logger,
+        [SensitiveData] string keycloakUserId
+    );
+
+    [LoggerMessage(
+        EventId = 2104,
+        Level = LogLevel.Information,
+        Message = "Set Keycloak user {KeycloakUserId} enabled={Enabled}"
+    )]
+    public static partial void UserEnabledStateChanged(
+        this ILogger logger,
+        [SensitiveData] string keycloakUserId,
+        bool enabled
+    );
+
+    [LoggerMessage(
+        EventId = 2105,
+        Level = LogLevel.Warning,
+        Message = "Keycloak user {KeycloakUserId} was not found during delete — treating as already deleted."
+    )]
+    public static partial void UserDeleteNotFound(
+        this ILogger logger,
+        [SensitiveData] string keycloakUserId
+    );
+
+    [LoggerMessage(
+        EventId = 2106,
+        Level = LogLevel.Information,
+        Message = "Deleted Keycloak user {KeycloakUserId}"
+    )]
+    public static partial void UserDeleted(
+        this ILogger logger,
+        [SensitiveData] string keycloakUserId
+    );
+}
+
+internal static partial class KeycloakAdminTokenProviderLogs
+{
+    [LoggerMessage(
+        EventId = 2201,
+        Level = LogLevel.Error,
+        Message = "Failed to acquire Keycloak admin token. Status: {Status}, Body: {Body}"
+    )]
+    public static partial void TokenAcquireFailed(
+        this ILogger logger,
+        int status,
+        [SensitiveData] string body
+    );
+}

src/Services/Identity/Identity.Infrastructure/Security/Tenant/TenantClaimValidatorLogs.cs

@@ -0,0 +1,34 @@
+using Microsoft.Extensions.Logging;
+using SharedKernel.Infrastructure.Logging;
+
+namespace Identity.Infrastructure.Security.Tenant;
+
+internal static partial class TenantClaimValidatorLogs
+{
+    [LoggerMessage(
+        EventId = 2001,
+        Level = LogLevel.Warning,
+        Message = "[{Scheme}] Token validated but no identity found"
+    )]
+    public static partial void TokenValidatedNoIdentity(this ILogger logger, string scheme);
+
+    [LoggerMessage(
+        EventId = 2002,
+        Level = LogLevel.Information,
+        Message = "[{Scheme}] Authenticated user={User}, tenant={TenantId}, roles=[{Roles}]"
+    )]
+    public static partial void UserAuthenticated(
+        this ILogger logger,
+        string scheme,
+        [PersonalData] string? user,
+        [SensitiveData] string? tenantId,
+        string roles
+    );
+
+    [LoggerMessage(
+        EventId = 2003,
+        Level = LogLevel.Warning,
+        Message = "User provisioning failed during token validation — authentication will continue"
+    )]
+    public static partial void UserProvisioningFailed(this ILogger logger, Exception exception);
+}

src/Services/Notifications/Notifications.Api/appsettings.Production.json

@@ -0,0 +1,6 @@
+{
+  "Redaction": {
+    "HmacKeyEnvironmentVariable": "APITEMPLATE_REDACTION_HMAC_KEY",
+    "KeyId": 1001
+  }
+}

src/Services/ProductCatalog/ProductCatalog.Api/Health/MongoDbHealthCheck.cs

@@ -0,0 +1,26 @@
+using Microsoft.Extensions.Diagnostics.HealthChecks;
+using ProductCatalog.Infrastructure.Persistence;
+
+namespace ProductCatalog.Api.Health;
+
+/// <summary>
+/// Verifies MongoDB availability using the application's configured MongoDbContext.
+/// </summary>
+public sealed class MongoDbHealthCheck(IMongoDbHealthProbe mongoDbHealthProbe) : IHealthCheck
+{
+    public async Task<HealthCheckResult> CheckHealthAsync(
+        HealthCheckContext context,
+        CancellationToken cancellationToken = default
+    )
+    {
+        try
+        {
+            await mongoDbHealthProbe.PingAsync(cancellationToken);
+            return HealthCheckResult.Healthy();
+        }
+        catch (Exception ex)
+        {
+            return HealthCheckResult.Unhealthy("MongoDB server is unavailable.", ex);
+        }
+    }
+}

src/Services/ProductCatalog/ProductCatalog.Api/appsettings.Production.json

@@ -0,0 +1,6 @@
+{
+  "Redaction": {
+    "HmacKeyEnvironmentVariable": "APITEMPLATE_REDACTION_HMAC_KEY",
+    "KeyId": 1001
+  }
+}