SF-3817 Partial book drafting

scripts/db_tools/parse-version.ts

@@ -42,7 +42,8 @@ class ParseVersion {
     'Enable Lynx insights',
     'Preview new draft history interface',
     'USFM Format',
-    'Show in-app onboarding form instead of external link'
+    'Show in-app onboarding form instead of external link',
+    'Partial book drafting'
   ];
 
   constructor() {

src/RealtimeServer/scriptureforge/models/translate-config.ts

@@ -74,6 +74,8 @@ export interface DraftConfig {
   draftingSources: TranslateSource[];
   trainingSources: TranslateSource[];
   lastSelectedTrainingDataFiles: string[];
+  /** Training data files available at the last build, to distinguish newly added files from deliberately deselected ones. */
+  lastAvailableTrainingDataFiles?: string[];
   lastSelectedTrainingScriptureRanges?: ProjectScriptureRange[];
   lastSelectedTranslationScriptureRanges?: ProjectScriptureRange[];
   fastTraining?: boolean;

src/RealtimeServer/scriptureforge/services/sf-project-service.ts

@@ -216,6 +216,12 @@ export class SFProjectService extends ProjectService<SFProject> {
                   bsonType: 'string'
                 }
               },
+              lastAvailableTrainingDataFiles: {
+                bsonType: 'array',
+                items: {
+                  bsonType: 'string'
+                }
+              },
               lastSelectedTrainingScriptureRanges: {
                 bsonType: 'array',
                 items: {

src/SIL.XForge.Scripture/ClientApp/e2e/characterize-tests.mts

@@ -6,7 +6,7 @@ import { numberOfTimesToAttemptTest } from './pass-probability.ts';
 import { ScreenshotContext } from './presets.ts';
 import { tests } from './test-definitions.ts';
 
-const retriesToStopAt = 3; // Stop characterization after a test is reliable enough to only need this many retries
+const retriesToStopAt = 4; // Stop characterization after a test is reliable enough to only need this many retries
 const resultFilePath = 'test_characterization.json';
 const testNames = Object.keys(tests) as (keyof typeof tests)[];
 let mostRecentResultData = JSON.parse(await Deno.readTextFile(resultFilePath));

src/SIL.XForge.Scripture/ClientApp/e2e/test-definitions.ts

@@ -6,6 +6,7 @@ import { editTranslation } from './workflows/edit-translation.ts';
 import { generateDraft } from './workflows/generate-draft.ts';
 import { localizedScreenshots } from './workflows/localized-screenshots.ts';
 import { onboardingFlow } from './workflows/onboarding-flow.ts';
+import { partialBookDrafting } from './workflows/partial-book-drafting.ts';
 import { runSmokeTests, traverseHomePageAndLoginPage } from './workflows/smoke-tests.mts';
 
 export const tests = {
@@ -24,6 +25,9 @@ export const tests = {
   generate_draft: async (_engine: BrowserType, page: Page, screenshotContext: ScreenshotContext) => {
     await generateDraft(page, screenshotContext, secrets.users[0]);
   },
+  partial_book_drafting: async (_engine: BrowserType, page: Page, screenshotContext: ScreenshotContext) => {
+    await partialBookDrafting(page, screenshotContext, secrets.users[0]);
+  },
   onboarding_flow: async (engine: BrowserType, page: Page, screenshotContext: ScreenshotContext) => {
     await onboardingFlow(engine, page, screenshotContext, secrets.users[0]);
   },

src/SIL.XForge.Scripture/ClientApp/e2e/test_characterization.json

@@ -26,5 +26,9 @@
   "onboarding_flow": {
     "success": 7,
     "failure": 0
+  },
+  "partial_book_drafting": {
+    "success": 7,
+    "failure": 0
   }
 }

src/SIL.XForge.Scripture/ClientApp/e2e/test_data/partial_draft_training_1.tsv

@@ -0,0 +1,4 @@
+Source	Target
+In the beginning God created the heavens and the earth.	En el principio creó Dios los cielos y la tierra.
+And God said, Let there be light: and there was light.	Y dijo Dios: Sea la luz; y fue la luz.
+And God called the light Day, and the darkness he called Night.	Y llamó Dios a la luz Día, y a las tinieblas llamó Noche.

src/SIL.XForge.Scripture/ClientApp/e2e/test_data/partial_draft_training_2.tsv

@@ -0,0 +1,4 @@
+Source	Target
+Now the serpent was more subtil than any beast of the field.	Pero la serpiente era astuta, más que todos los animales del campo.
+And they heard the voice of the LORD God walking in the garden.	Y oyeron la voz de Jehová Dios que se paseaba en el huerto.
+And the LORD God called unto Adam, and said unto him, Where art thou?	Mas Jehová Dios llamó al hombre, y le dijo: ¿Dónde estás tú?

src/SIL.XForge.Scripture/ClientApp/src/app/app.component.html

@@ -149,6 +149,11 @@
           <button mat-menu-item appRouterLink="/projects" id="project-home-link">
             <mat-icon>home</mat-icon> {{ "my_projects.my_projects" | transloco }}
           </button>
+          @if (experimentalFeatures.showExperimentalFeaturesInMenu) {
+            <button mat-menu-item (click)="openExperimentalFeaturesDialog()">
+              <mat-icon>science</mat-icon> {{ t("experimental_features") }}
+            </button>
+          }
           @if (featureFlags.showDeveloperTools.enabled) {
             <button mat-menu-item [matMenuTriggerFor]="appearanceMenu">
               <mat-icon>palette</mat-icon>

src/SIL.XForge.Scripture/ClientApp/src/app/app.component.ts

@@ -28,6 +28,8 @@ import { DataLoadingComponent } from 'xforge-common/data-loading-component';
 import { DiagnosticOverlayService } from 'xforge-common/diagnostic-overlay.service';
 import { DialogService } from 'xforge-common/dialog.service';
 import { ErrorReportingService } from 'xforge-common/error-reporting.service';
+import { ExperimentalFeaturesDialogComponent } from 'xforge-common/experimental-features/experimental-features-dialog.component';
+import { ExperimentalFeaturesService } from 'xforge-common/experimental-features/experimental-features.service';
 import { ExternalUrlService } from 'xforge-common/external-url.service';
 import { FeatureFlagService } from 'xforge-common/feature-flags/feature-flag.service';
 import { FeatureFlagsDialogComponent } from 'xforge-common/feature-flags/feature-flags-dialog.component';
@@ -138,6 +140,7 @@ export class AppComponent extends DataLoadingComponent implements OnInit, OnDest
     private readonly fontService: FontService,
     private readonly brandingService: BrandingService,
     onlineStatusService: OnlineStatusService,
+    readonly experimentalFeatures: ExperimentalFeaturesService,
     private destroyRef: DestroyRef
   ) {
     super(noticeService, 'AppComponent');
@@ -472,6 +475,10 @@ export class AppComponent extends DataLoadingComponent implements OnInit, OnDest
     this.dialogService.openMatDialog(FeatureFlagsDialogComponent);
   }
 
+  openExperimentalFeaturesDialog(): void {
+    this.dialogService.openMatDialog(ExperimentalFeaturesDialogComponent);
+  }
+
   openDiagnosticOverlay(): void {
     this.diagnosticOverlayService.open();
   }

src/SIL.XForge.Scripture/ClientApp/src/app/app.routes.ts

@@ -31,6 +31,7 @@ import { ConfigureSourcesComponent } from './translate/draft-generation/configur
 import { DraftGenerationComponent } from './translate/draft-generation/draft-generation.component';
 import { DraftOnboardingFormComponent } from './translate/draft-generation/draft-signup-form/draft-onboarding-form.component';
 import { DraftUsfmFormatComponent } from './translate/draft-generation/draft-usfm-format/draft-usfm-format.component';
+import { NewDraftComponent } from './translate/draft-generation/new-draft/new-draft.component';
 import { EditorComponent } from './translate/editor/editor.component';
 import { TranslateOverviewComponent } from './translate/translate-overview/translate-overview.component';
 import { UsersComponent } from './users/users.component';
@@ -82,6 +83,11 @@ export const APP_ROUTES: Routes = [
     component: DraftGenerationComponent,
     canActivate: [NmtDraftAuthGuard]
   },
+  {
+    path: 'projects/:projectId/draft-generation/new-draft',
+    component: NewDraftComponent,
+    canActivate: [NmtDraftAuthGuard]
+  },
   { path: 'projects/:projectId/event-log', component: EventMetricsComponent, canActivate: [EventMetricsAuthGuard] },
   { path: 'projects/:projectId/settings', component: SettingsComponent, canActivate: [SettingsAuthGuard] },
   { path: 'projects/:projectId/sync', component: SyncComponent, canActivate: [SyncAuthGuard] },

src/SIL.XForge.Scripture/ClientApp/src/app/core/sf-project.service.ts

@@ -27,7 +27,7 @@ import { QueryParameters, QueryResults } from 'xforge-common/query-parameters';
 import { RealtimeService } from 'xforge-common/realtime.service';
 import { RetryingRequest, RetryingRequestService } from 'xforge-common/retrying-request.service';
 import { EventMetric } from '../event-metrics/event-metric';
-import { BookProgress } from '../shared/progress-service/progress.service';
+import { BookProgressWithChapterProgress } from '../shared/progress-service/progress.service';
 import { expandNumbers } from '../shared/utils';
 import { BiblicalTermDoc } from './models/biblical-term-doc';
 import { InviteeStatus } from './models/invitee-status';
@@ -98,8 +98,10 @@ export class SFProjectService extends ProjectService<SFProject, SFProjectDoc> {
       // If the book is present without chapters, then the chapter is present
       if (range.length === bookId.length) return true;
 
-      // Expand the chapter range, and see if the specified chapter number is present
-      if (expandNumbers(range.slice(bookId.length)).includes(chapterNum)) return true;
+      // Expand the chapter range, and see if the specified chapter number is present. Invalid notation expands to
+      // null and is treated as the chapter not being present.
+      const chapters: number[] | null = expandNumbers(range.slice(bookId.length));
+      if (chapters != null && chapters.includes(chapterNum)) return true;
     }
 
     // The chapter was not present in the book
@@ -451,7 +453,7 @@ export class SFProjectService extends ProjectService<SFProject, SFProjectDoc> {
   }
 
   /** Gets project progress by calling the backend aggregation endpoint. */
-  async getProjectProgress(projectId: string): Promise<BookProgress[]> {
-    return await this.onlineInvoke<BookProgress[]>('getProjectProgress', { projectId });
+  async getProjectProgress(projectId: string): Promise<BookProgressWithChapterProgress[]> {
+    return await this.onlineInvoke<BookProgressWithChapterProgress[]>('getProjectProgress', { projectId });
   }
 }

src/SIL.XForge.Scripture/ClientApp/src/app/serval-administration/serval-build-report.ts

@@ -214,8 +214,9 @@ function parseBooksAndChapters(scriptureRange: string): BookAndChapters[] {
       if (chapterPart === '') {
         return { bookId };
       }
-      const chapters: number[] = expandNumbers(chapterPart);
-      return chapters.length > 0 ? { bookId, chapters } : { bookId };
+      // Invalid chapter notation falls back to the whole book (no chapter detail), same as an empty chapter part.
+      const chapters: number[] | null = expandNumbers(chapterPart);
+      return chapters != null && chapters.length > 0 ? { bookId, chapters } : { bookId };
     })
     .filter(notNull);
 }

src/SIL.XForge.Scripture/ClientApp/src/app/serval-administration/serval-builds.component.ts

@@ -53,6 +53,7 @@ import { UserService } from 'xforge-common/user.service';
 import { isPopulatedString, isString, notNull } from '../../type-utils';
 import { InfoComponent } from '../shared/info/info.component';
 import { NoticeComponent } from '../shared/notice/notice.component';
+import { ChapterSet } from '../shared/scripture-range';
 import { BookConfidence, ChapterConfidence } from '../translate/draft-generation/build-confidences/build-confidences';
 import { DisplayConfidenceComponent } from '../translate/draft-generation/build-confidences/display-confidence.component';
 import { DraftGenerationService } from '../translate/draft-generation/draft-generation.service';
@@ -1036,24 +1037,9 @@ export class ServalBuildsComponent extends DataLoadingComponent implements OnIni
       .join('; ');
   }
 
-  /** Formats a sorted array of numbers into compact range notation, e.g. [1,2,3,5,7,8,9] → "1-3, 5, 7-9". */
+  /** Formats numbers into compact range notation, e.g. [7, 1, 3, 2] → "1-3, 7" (sorted, de-duplicated). */
   static compactRangeNotation(nums: number[]): string {
-    if (nums.length === 0) return '';
-    const sortedUnique: number[] = [...new Set(nums)].sort((a, b) => a - b);
-    const ranges: string[] = [];
-    let rangeStart: number = sortedUnique[0];
-    let rangeEnd: number = sortedUnique[0];
-    for (let i = 1; i < sortedUnique.length; i++) {
-      if (sortedUnique[i] === rangeEnd + 1) {
-        rangeEnd = sortedUnique[i];
-      } else {
-        ranges.push(rangeStart === rangeEnd ? `${rangeStart}` : `${rangeStart}-${rangeEnd}`);
-        rangeStart = sortedUnique[i];
-        rangeEnd = sortedUnique[i];
-      }
-    }
-    ranges.push(rangeStart === rangeEnd ? `${rangeStart}` : `${rangeStart}-${rangeEnd}`);
-    return ranges.join(', ');
+    return new ChapterSet(nums).toStringForDisplay();
   }
 
   /** (The return type is string if the input is a type string (at compile time), or undefined

src/SIL.XForge.Scripture/ClientApp/src/app/shared/book-multi-select/book-multi-select.component.html

@@ -1,43 +1,45 @@
 <ng-container *transloco="let t; read: 'book_select'">
-  @if (availableBooks.length > 0 && !readonly && !basicMode) {
+  @if (!readonly && (projectName != null || (bulkBookSelection && availableBooks.length > 0))) {
     <div class="scope-selection-wrapper">
       @if (projectName != null) {
         <span class="project-name">{{ projectName }}</span>
       }
-      <div class="scope-selection">
-        <mat-checkbox
-          class="ot-checkbox"
-          value="OT"
-          [disabled]="!isOldTestamentAvailable()"
-          [checked]="selectedAllOT"
-          [indeterminate]="partialOT"
-          (change)="select('OT', $event.checked)"
-        >
-          {{ t("old_testament") }}
-        </mat-checkbox>
-        <mat-checkbox
-          class="nt-checkbox"
-          value="NT"
-          [disabled]="!isNewTestamentAvailable()"
-          [checked]="selectedAllNT"
-          [indeterminate]="partialNT"
-          (change)="select('NT', $event.checked)"
-        >
-          {{ t("new_testament") }}
-        </mat-checkbox>
-
-        @if (isDeuterocanonAvailable()) {
+      @if (bulkBookSelection && availableBooks.length > 0) {
+        <div class="scope-selection">
           <mat-checkbox
-            class="dc-checkbox"
-            value="DC"
-            [checked]="selectedAllDC"
-            [indeterminate]="partialDC"
-            (change)="select('DC', $event.checked)"
+            class="ot-checkbox"
+            value="OT"
+            [disabled]="!isOldTestamentAvailable()"
+            [checked]="selectedAllOT"
+            [indeterminate]="partialOT"
+            (change)="select('OT', $event.checked)"
           >
-            {{ t("deuterocanon") }}
+            {{ t("old_testament") }}
           </mat-checkbox>
-        }
-      </div>
+          <mat-checkbox
+            class="nt-checkbox"
+            value="NT"
+            [disabled]="!isNewTestamentAvailable()"
+            [checked]="selectedAllNT"
+            [indeterminate]="partialNT"
+            (change)="select('NT', $event.checked)"
+          >
+            {{ t("new_testament") }}
+          </mat-checkbox>
+
+          @if (isDeuterocanonAvailable()) {
+            <mat-checkbox
+              class="dc-checkbox"
+              value="DC"
+              [checked]="selectedAllDC"
+              [indeterminate]="partialDC"
+              (change)="select('DC', $event.checked)"
+            >
+              {{ t("deuterocanon") }}
+            </mat-checkbox>
+          }
+        </div>
+      }
     </div>
   }
   <div class="flex-row">
@@ -49,7 +51,7 @@
         [disabled]="readonly"
         (change)="onChipListChange(book)"
       >
-        @if (book.progress != null && basicMode === false) {
+        @if (book.progress != null && showProgress) {
           <mat-chip-option
             [value]="book"
             [selected]="book.selected"
@@ -66,10 +68,4 @@
       </mat-chip-listbox>
     }
   </div>
-  @if (!loaded) {
-    <div class="loading-message">
-      <mat-spinner [diameter]="15"></mat-spinner>
-      {{ t("loading_message") }}
-    </div>
-  }
 </ng-container>

src/SIL.XForge.Scripture/ClientApp/src/app/shared/book-multi-select/book-multi-select.component.scss

@@ -28,10 +28,3 @@
     background: var(--sf-book-border-fill);
   }
 }
-
-.loading-message {
-  display: flex;
-  gap: 0.5em;
-  align-items: center;
-  margin-block: 1em;
-}