SF-3819 Add support for draft rate limiting

src/SIL.XForge.Scripture/ClientApp/src/app/translate/draft-generation/draft-generation.service.spec.ts

@@ -1,4 +1,4 @@
-import { HttpStatusCode } from '@angular/common/http';
+import { HttpErrorResponse, HttpStatusCode } from '@angular/common/http';
 import { HttpTestingController } from '@angular/common/http/testing';
 import { fakeAsync, TestBed, tick } from '@angular/core/testing';
 import { Canon } from '@sillsdev/scripture';
@@ -600,6 +600,97 @@ describe('DraftGenerationService', () => {
       tick();
     }));
 
+    it('should fail on a Serval outage error', fakeAsync(() => {
+      const spyGetBuildProgress = spyOn(service, 'getBuildProgress').and.returnValue(of(undefined));
+
+      // SUT
+      service
+        .startBuildOrGetActiveBuild(buildConfig)
+        .pipe(first())
+        .subscribe(result => {
+          expect(result).toBeUndefined();
+          verify(mockNoticeService.showError(anything())).once();
+          expect(spyGetBuildProgress).toHaveBeenCalledWith(projectId);
+        });
+      tick();
+
+      // Setup the HTTP request
+      const req = httpTestingController.expectOne(`${MACHINE_API_BASE_URL}translation/pretranslations`);
+      expect(req.request.method).toEqual('POST');
+      expect(req.request.body).toEqual(buildConfig);
+      req.flush(null, { status: HttpStatusCode.ServiceUnavailable, statusText: 'Serval Down' });
+      tick();
+    }));
+
+    it('should fail on a rate limit error', fakeAsync(() => {
+      const spyGetBuildProgress = spyOn(service, 'getBuildProgress').and.returnValue(of(undefined));
+
+      // SUT
+      service
+        .startBuildOrGetActiveBuild(buildConfig)
+        .pipe(first())
+        .subscribe(result => {
+          expect(result).toBeUndefined();
+          verify(mockNoticeService.showError(anything())).once();
+          expect(spyGetBuildProgress).toHaveBeenCalledWith(projectId);
+        });
+      tick();
+
+      // Setup the HTTP request
+      const req = httpTestingController.expectOne(`${MACHINE_API_BASE_URL}translation/pretranslations`);
+      expect(req.request.method).toEqual('POST');
+      expect(req.request.body).toEqual(buildConfig);
+      req.flush(null, { status: HttpStatusCode.TooManyRequests, statusText: 'Too many requests' });
+      tick();
+    }));
+
+    it('should fail on an forbidden error', fakeAsync(() => {
+      const spyGetBuildProgress = spyOn(service, 'getBuildProgress').and.returnValue(of(undefined));
+
+      // SUT
+      service
+        .startBuildOrGetActiveBuild(buildConfig)
+        .pipe(first())
+        .subscribe(result => {
+          expect(result).toBeUndefined();
+          verify(mockNoticeService.showError(anything())).never();
+          expect(spyGetBuildProgress).toHaveBeenCalledWith(projectId);
+        });
+      tick();
+
+      // Setup the HTTP request
+      const req = httpTestingController.expectOne(`${MACHINE_API_BASE_URL}translation/pretranslations`);
+      expect(req.request.method).toEqual('POST');
+      expect(req.request.body).toEqual(buildConfig);
+      req.flush(null, { status: HttpStatusCode.Forbidden, statusText: 'Forbidden' });
+      tick();
+    }));
+
+    it('should rethrow an unauthorized error', fakeAsync(() => {
+      const spyGetBuildProgress = spyOn(service, 'getBuildProgress').and.returnValue(of(undefined));
+
+      // SUT
+      service
+        .startBuildOrGetActiveBuild(buildConfig)
+        .pipe(first())
+        .subscribe({
+          next: () => fail('Expected an error'),
+          error: (err: HttpErrorResponse) => {
+            expect(err.status).toBe(HttpStatusCode.Unauthorized);
+            verify(mockNoticeService.showError(anything())).never();
+            expect(spyGetBuildProgress).toHaveBeenCalledWith(projectId);
+          }
+        });
+      tick();
+
+      // Setup the HTTP request
+      const req = httpTestingController.expectOne(`${MACHINE_API_BASE_URL}translation/pretranslations`);
+      expect(req.request.method).toEqual('POST');
+      expect(req.request.body).toEqual(buildConfig);
+      req.flush(null, { status: HttpStatusCode.Unauthorized, statusText: 'Unauthorized' });
+      tick();
+    }));
+
     it('should return already active build job', fakeAsync(() => {
       const spyGetBuildProgress = spyOn(service, 'getBuildProgress').and.returnValue(of(buildDto));
       const spyPollBuildProgress = spyOn(service, 'pollBuildProgress').and.returnValue(of(buildDto));

src/SIL.XForge.Scripture/ClientApp/src/app/translate/draft-generation/draft-generation.service.ts

@@ -259,9 +259,32 @@ export class DraftGenerationService {
         }
 
         // Otherwise, start build and then poll
-        return this.startBuild(buildConfig).pipe(
-          // No errors means build successfully started, so start polling
-          switchMap(() => this.pollBuildProgress(buildConfig.projectId))
+        return this.httpClient.post<void>(`translation/pretranslations`, buildConfig).pipe(
+          map(() => true),
+          catchError(err => {
+            if (err.status === 401) {
+              // Expired Paratext credentials. Rethrow to be caught by DraftGenerationComponent.startBuild()
+              throw err;
+            }
+
+            if (err.status === 403 || err.status === 404) {
+              return of(false);
+            }
+
+            if (err.status === 429) {
+              this.noticeService.showError(this.i18n.translateStatic('draft_generation.quota_exceeded'));
+              return of(false);
+            }
+
+            this.noticeService.showError(this.i18n.translateStatic('draft_generation.temporarily_unavailable'));
+            return of(false);
+          }),
+          switchMap(started => {
+            if (!started) return of(undefined);
+
+            // No error means build successfully started, so start polling
+            return this.pollBuildProgress(buildConfig.projectId);
+          })
         );
       })
     );
@@ -525,13 +548,4 @@ export class DraftGenerationService {
       });
     });
   }
-
-  /**
-   * Calls the machine api to start a pre-translation build job.
-   * This should only be called if no build is currently active.
-   * @param buildConfig The build configuration.
-   */
-  private startBuild(buildConfig: BuildConfig): Observable<void> {
-    return this.httpClient.post<void>(`translation/pretranslations`, buildConfig).pipe(map(res => res.data));
-  }
 }

src/SIL.XForge.Scripture/ClientApp/src/assets/i18n/non_checking_en.json

@@ -221,6 +221,7 @@
     "preview_last_draft_button": "Preview last draft",
     "preview_last_draft_detail": "The last draft build did not complete, but you can still view the most recent successful draft.",
     "preview_last_draft_header": "Preview last draft",
+    "quota_exceeded": "Generating drafts is temporarily unavailable for this project as its quota has been exceeded",
     "report_problem": "Report problem",
     "signup_already_submitted": "{{ name }} submitted a request to sign up for drafting on {{ date }}. A team member will contact you within {{ min }} to {{ max }} business days to discuss your project and next steps.",
     "sign_up_for_drafting": "Sign up for drafting",

src/SIL.XForge.Scripture/Controllers/MachineApiController.cs

@@ -1038,6 +1038,7 @@ public async Task<ActionResult> StartBuildAsync([FromBody] string sfProjectId, C
     /// <response code="401">Your Paratext tokens have expired, and you must log in again.</response>
     /// <response code="403">You do not have permission to build this project.</response>
     /// <response code="404">The project does not exist or is not configured on the ML server.</response>
+    /// <response code="429">The project's build quota has been exceeded.</response>
     /// <response code="503">The ML server is temporarily unavailable or unresponsive.</response>
     /// <remarks>
     /// If a JSON string is passed in the format "project_id", then a default build configuration will be created for
@@ -1071,6 +1072,10 @@ await _machineApiService.StartPreTranslationBuildAsync(
         {
             return Forbid();
         }
+        catch (LimitExceededException e)
+        {
+            return StatusCode(StatusCodes.Status429TooManyRequests, e.Message);
+        }
         catch (UnauthorizedAccessException)
         {
             return Unauthorized();

src/SIL.XForge.Scripture/Services/MachineApiService.cs

@@ -56,6 +56,7 @@ public class MachineApiService(
     ISFProjectRights projectRights,
     ISFProjectService projectService,
     IRealtimeService realtimeService,
+    IRepository<SiteConfig> siteConfigs,
     IOptions<SiteOptions> siteOptions,
     ISyncService syncService,
     ITranslationBuildsClient translationBuildsClient,
@@ -2794,6 +2795,62 @@ CancellationToken cancellationToken
             throw new ForbiddenException();
         }
 
+        // See if there is a build limit enforced
+        SiteConfig? siteConfig = await siteConfigs
+            .Query()
+            .FirstOrDefaultAsync(s => s.Name == siteOptions.Value.Id, cancellationToken);
+
+        // Check the build quota for the last day
+        if (siteConfig is not null && siteConfig.BuildQuotaPerDay > 0)
+        {
+            // Get all build events during the last day
+            QueryResults<EventMetric> eventMetrics = await eventMetricService.GetEventMetricsAsync(
+                buildConfig.ProjectId,
+                scopes: [EventScope.Drafting],
+                eventTypes: [nameof(MachineProjectService.BuildProjectAsync)],
+                fromDate: DateTime.UtcNow.AddDays(-1),
+                pageSize: 0 // Use a zero page size as we only want the count not the results
+            );
+            if (eventMetrics.UnpagedCount >= siteConfig.BuildQuotaPerDay)
+            {
+                throw new LimitExceededException("Build quota exceeded.");
+            }
+        }
+
+        // Check the build quota for the last week
+        if (siteConfig is not null && siteConfig.BuildQuotaPerWeek > 0)
+        {
+            // Get all build events during the last week
+            QueryResults<EventMetric> eventMetrics = await eventMetricService.GetEventMetricsAsync(
+                buildConfig.ProjectId,
+                scopes: [EventScope.Drafting],
+                eventTypes: [nameof(MachineProjectService.BuildProjectAsync)],
+                fromDate: DateTime.UtcNow.AddDays(-7),
+                pageSize: 0 // Use a zero page size as we only want the count not the results
+            );
+            if (eventMetrics.UnpagedCount >= siteConfig.BuildQuotaPerWeek)
+            {
+                throw new LimitExceededException("Build quota exceeded.");
+            }
+        }
+
+        // Check the build quota for the last month
+        if (siteConfig is not null && siteConfig.BuildQuotaPerMonth > 0)
+        {
+            // Get all build events during the last month
+            QueryResults<EventMetric> eventMetrics = await eventMetricService.GetEventMetricsAsync(
+                buildConfig.ProjectId,
+                scopes: [EventScope.Drafting],
+                eventTypes: [nameof(MachineProjectService.BuildProjectAsync)],
+                fromDate: DateTime.UtcNow.AddMonths(-1),
+                pageSize: 0 // Use a zero page size as we only want the count not the results
+            );
+            if (eventMetrics.UnpagedCount >= siteConfig.BuildQuotaPerMonth)
+            {
+                throw new LimitExceededException("Build quota exceeded.");
+            }
+        }
+
         // Save the selected books
         await projectDoc.SubmitJson0OpAsync(op =>
         {

src/SIL.XForge.Scripture/Services/MachineBackgroundService.cs

@@ -47,9 +47,12 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken)
             try
             {
                 // If we do not have it already, load the site_config document by name, creating a new one if missing
-                siteConfig ??=
-                    await siteConfigs.Query().FirstOrDefaultAsync(s => s.Name == siteName, stoppingToken)
-                    ?? new SiteConfig { Id = ObjectId.GenerateNewId().ToString(), Name = siteName };
+                siteConfig ??= await siteConfigs.Query().FirstOrDefaultAsync(s => s.Name == siteName, stoppingToken);
+                if (siteConfig is null)
+                {
+                    siteConfig = new SiteConfig { Id = ObjectId.GenerateNewId().ToString(), Name = siteName };
+                    await siteConfigs.InsertAsync(siteConfig, stoppingToken);
+                }
 
                 // Poll for the latest build
                 TranslationBuild build = await translationBuildsClient.GetNextFinishedBuildAsync(
@@ -66,7 +69,11 @@ await siteConfigs.Query().FirstOrDefaultAsync(s => s.Name == siteName, stoppingT
                 // On failure throw an exception, so the next iteration of the loop will begin.
                 // This is OK, as we will record the last build finished on the next iteration of the loop
                 siteConfig.LastFinishedBuildId = build.Id;
-                await siteConfigs.ReplaceAsync(siteConfig, upsert: true, stoppingToken);
+                await siteConfigs.UpdateAsync(
+                    siteConfig.Id,
+                    u => u.Set(sc => sc.LastFinishedBuildId, build.Id),
+                    cancellationToken: stoppingToken
+                );
 
                 // Reset the consecutive failure count
                 consecutiveFailures = 0;

src/SIL.XForge/DataAccess/DataAccessServiceCollectionExtensions.cs

@@ -64,7 +64,7 @@ public static IServiceCollection AddDataAccess(this IServiceCollection services,
         );
         services.AddMongoRepository<SiteConfig>(
             "site_configs",
-            cm => cm.MapIdProperty(sm => sm.Id),
+            cm => cm.MapIdProperty(sc => sc.Id),
             im =>
                 im.CreateOne(new CreateIndexModel<SiteConfig>(Builders<SiteConfig>.IndexKeys.Ascending(sc => sc.Name)))
         );

src/SIL.XForge/Models/SiteConfig.cs

@@ -1,8 +1,43 @@
 namespace SIL.XForge.Models;
 
+/// <summary>
+/// The sitewide configuration stored in MongoDB.
+/// </summary>
 public class SiteConfig : IIdentifiable
 {
+    /// <summary>
+    /// The object identifier.
+    /// </summary>
     public required string Id { get; set; }
+
+    /// <summary>
+    /// The site name.
+    /// </summary>
+    /// <remarks>
+    /// This corresponds to <see cref="Configuration.SiteOptions.Id"/>.
+    /// </remarks>
     public required string Name { get; set; }
+
+    /// <summary>
+    /// The last finished build identifier.
+    /// </summary>
+    /// <remarks>
+    /// This is used by the Machine Background Service.
+    /// </remarks>
     public string? LastFinishedBuildId { get; set; }
+
+    /// <summary>
+    /// The number of builds allowed per day for every project.
+    /// </summary>
+    public int BuildQuotaPerDay { get; set; }
+
+    /// <summary>
+    /// The number of builds allowed per week for every project.
+    /// </summary>
+    public int BuildQuotaPerWeek { get; set; }
+
+    /// <summary>
+    /// The number of builds allowed per month for every project.
+    /// </summary>
+    public int BuildQuotaPerMonth { get; set; }
 }

src/SIL.XForge/Services/EventMetricService.cs

@@ -69,6 +69,12 @@ public async Task<QueryResults<EventMetric>> GetEventMetricsAsync(
             results = await orderedQuery.ToListAsync();
             unpagedCount = results.Count;
         }
+        else if (pageSize == 0)
+        {
+            // We only want the unpaginated count
+            results = [];
+            unpagedCount = await query.CountAsync();
+        }
         else
         {
             // Execute count and paged results

src/SIL.XForge/Services/LimitExceededException.cs

@@ -0,0 +1,9 @@
+using System;
+
+namespace SIL.XForge.Services;
+
+/// <summary>
+/// A rate limit has been exceeded.
+/// </summary>
+/// <param name="message">The error message.</param>
+public class LimitExceededException(string message) : Exception(message);

test/SIL.XForge.Scripture.Tests/Controllers/MachineApiControllerTests.cs

@@ -2573,6 +2573,28 @@ public async Task StartPreTranslationBuildAsync_NoProject()
         Assert.IsInstanceOf<NotFoundResult>(actual);
     }
 
+    [Test]
+    public async Task StartPreTranslationBuildAsync_RateLimitExceeded()
+    {
+        // Set up test environment
+        var env = new TestEnvironment();
+        env.MachineApiService.StartPreTranslationBuildAsync(
+                User01,
+                Arg.Is<BuildConfig>(p => p.ProjectId == Project01),
+                CancellationToken.None
+            )
+            .Throws(new LimitExceededException("Too many drafts"));
+
+        // SUT
+        ActionResult actual = await env.Controller.StartPreTranslationBuildAsync(
+            new BuildConfig { ProjectId = Project01 },
+            CancellationToken.None
+        );
+
+        Assert.IsInstanceOf<ObjectResult>(actual);
+        Assert.AreEqual(StatusCodes.Status429TooManyRequests, (actual as ObjectResult)?.StatusCode);
+    }
+
     [Test]
     public async Task StartPreTranslationBuildAsync_Success()
     {