From e52b0c5798b918b6f2b9b3c9f88e017a4a50673e Mon Sep 17 00:00:00 2001 From: Enkidu93 Date: Wed, 1 Jul 2026 13:21:18 -0400 Subject: [PATCH 1/2] Add chapter-level data counts to execution data --- .../Models/BuildExecutionData.cs | 7 +- .../Services/NmtPreprocessBuildJob.cs | 21 +++-- .../Services/PreprocessBuildJob.cs | 17 ++-- .../Services/PreprocessStats.cs | 11 +++ .../ServalTranslationPlatformService.cs | 6 +- .../ServalWordAlignmentPlatformService.cs | 6 +- .../Services/TranslationPreprocessBuildJob.cs | 84 ++++++++++++++++--- .../WordAlignmentPreprocessBuildJob.cs | 84 ++++++++++++++++--- src/Serval/src/Serval.Client/Client.g.cs | 24 ++++++ .../ExecutionDataContract.cs | 4 + .../Dtos/ExecutionDataDto.cs | 4 + .../Models/ExecutionData.cs | 4 + .../Serval.Translation/Services/DtoMapper.cs | 4 + .../Services/PlatformService.cs | 4 + .../ExecutionDataContract.cs | 4 + .../Dtos/WordAlignmentExecutionDataDto.cs | 4 + .../Models/ExecutionData.cs | 4 + .../Services/DtoMapper.cs | 4 + .../Services/PlatformService.cs | 4 + .../Services/PlatformServiceTests.cs | 35 +++++++- 20 files changed, 286 insertions(+), 49 deletions(-) create mode 100644 src/Machine/src/Serval.Machine.Shared/Services/PreprocessStats.cs diff --git a/src/Machine/src/Serval.Machine.Shared/Models/BuildExecutionData.cs b/src/Machine/src/Serval.Machine.Shared/Models/BuildExecutionData.cs index 7f1e8ad3e..2fc38e662 100644 --- a/src/Machine/src/Serval.Machine.Shared/Models/BuildExecutionData.cs +++ b/src/Machine/src/Serval.Machine.Shared/Models/BuildExecutionData.cs @@ -3,8 +3,11 @@ namespace Serval.Machine.Shared.Models; public record BuildExecutionData { public int? TrainCount { get; init; } - public int? PretranslateCount { get; init; } - public int? WordAlignCount { get; init; } + public int? InferenceCount { get; init; } + public bool? IsTrainFilteredByChapter { get; init; } + public bool? IsInferenceFilteredByChapter { get; init; } + public Dictionary>? TrainVerseCount { get; init; } + public Dictionary>? InferenceVerseCount { get; init; } public IReadOnlyList? Warnings { get; init; } public string? EngineSourceLanguageTag { get; init; } public string? EngineTargetLanguageTag { get; init; } diff --git a/src/Machine/src/Serval.Machine.Shared/Services/NmtPreprocessBuildJob.cs b/src/Machine/src/Serval.Machine.Shared/Services/NmtPreprocessBuildJob.cs index bdc176ff1..dfd389858 100644 --- a/src/Machine/src/Serval.Machine.Shared/Services/NmtPreprocessBuildJob.cs +++ b/src/Machine/src/Serval.Machine.Shared/Services/NmtPreprocessBuildJob.cs @@ -52,8 +52,7 @@ await PlatformService.UpdateTargetQuoteConventionAsync( protected override async Task UpdateBuildExecutionData( string engineId, string buildId, - int trainCount, - int pretranslateCount, + PreprocessStats stats, string sourceLanguageTag, string targetLanguageTag, IReadOnlyList parallelCorpora, @@ -63,7 +62,7 @@ CancellationToken cancellationToken bool sourceLanguageHasNativeSupport = ResolveLanguageCode(sourceLanguageTag, out string resolvedSourceLanguage); bool targetLanguageHasNativeSupport = ResolveLanguageCode(targetLanguageTag, out string resolvedTargetLanguage); - if (trainCount == 0 && (!sourceLanguageHasNativeSupport || !targetLanguageHasNativeSupport)) + if (stats.TrainCount == 0 && (!sourceLanguageHasNativeSupport || !targetLanguageHasNativeSupport)) { throw new InvalidOperationException( $"At least one language code in build {buildId} is unknown to the base model, and the data specified for training was empty. Build canceled." @@ -71,8 +70,8 @@ CancellationToken cancellationToken } IReadOnlyList warnings = GetWarnings( - trainCount, - pretranslateCount, + stats.TrainCount, + stats.InferenceCount, sourceLanguageTag, targetLanguageTag, parallelCorpora @@ -92,8 +91,8 @@ CancellationToken cancellationToken { "Event", "BuildPreprocess" }, { "EngineId", engineId }, { "BuildId", buildId }, - { "NumTrainRows", trainCount }, - { "NumPretranslateRows", pretranslateCount }, + { "NumTrainRows", stats.TrainCount }, + { "NumPretranslateRows", stats.InferenceCount }, { "EngineSourceLanguageTag", sourceLanguageTag }, { "EngineTargetLanguageTag", targetLanguageTag }, { "SourceLanguageResolved", resolvedSourceLanguage }, @@ -103,8 +102,12 @@ CancellationToken cancellationToken Logger.LogInformation("{summary}", buildPreprocessSummary.ToJsonString()); var executionData = new BuildExecutionData() { - TrainCount = trainCount, - PretranslateCount = pretranslateCount, + TrainCount = stats.TrainCount, + InferenceCount = stats.InferenceCount, + TrainVerseCount = stats.TrainVerseCount, + InferenceVerseCount = stats.InferenceVerseCount, + IsTrainFilteredByChapter = stats.IsTrainFilteredByChapter, + IsInferenceFilteredByChapter = stats.IsInferenceFilteredByChapter, Warnings = warnings, EngineSourceLanguageTag = sourceLanguageTag, EngineTargetLanguageTag = targetLanguageTag, diff --git a/src/Machine/src/Serval.Machine.Shared/Services/PreprocessBuildJob.cs b/src/Machine/src/Serval.Machine.Shared/Services/PreprocessBuildJob.cs index dfad1cdfb..d9942b5f7 100644 --- a/src/Machine/src/Serval.Machine.Shared/Services/PreprocessBuildJob.cs +++ b/src/Machine/src/Serval.Machine.Shared/Services/PreprocessBuildJob.cs @@ -45,18 +45,12 @@ CancellationToken cancellationToken if (engine is null) throw new OperationCanceledException($"Engine {engineId} does not exist. Build canceled."); - (int trainCount, int inferenceCount) = await WriteDataFilesAsync( - buildId, - data, - buildOptions, - cancellationToken - ); + PreprocessStats stats = await WriteDataFilesAsync(buildId, data, buildOptions, cancellationToken); await UpdateBuildExecutionData( engineId, buildId, - trainCount, - inferenceCount, + stats, engine.SourceLanguage, engine.TargetLanguage, data, @@ -65,7 +59,7 @@ await UpdateBuildExecutionData( await UpdateTargetQuoteConventionAsync(engineId, buildId, data, cancellationToken); - if (inferenceCount == 0 && engine is TranslationEngine { IsModelPersisted: false }) + if (stats.InferenceCount == 0 && engine is TranslationEngine { IsModelPersisted: false }) { throw new InvalidOperationException( $"There was no data specified for inferencing in build {buildId}. Build canceled." @@ -90,8 +84,7 @@ await UpdateBuildExecutionData( protected abstract Task UpdateBuildExecutionData( string engineId, string buildId, - int trainCount, - int inferenceCount, + PreprocessStats stats, string sourceLanguageTag, string targetLanguageTag, IReadOnlyList parallelCorpora, @@ -105,7 +98,7 @@ protected virtual Task UpdateTargetQuoteConventionAsync( CancellationToken cancellationToken ) => Task.CompletedTask; - protected abstract Task<(int TrainCount, int InferenceCount)> WriteDataFilesAsync( + protected abstract Task WriteDataFilesAsync( string buildId, IReadOnlyList parallelCorpora, string? buildOptions, diff --git a/src/Machine/src/Serval.Machine.Shared/Services/PreprocessStats.cs b/src/Machine/src/Serval.Machine.Shared/Services/PreprocessStats.cs new file mode 100644 index 000000000..d115e03ea --- /dev/null +++ b/src/Machine/src/Serval.Machine.Shared/Services/PreprocessStats.cs @@ -0,0 +1,11 @@ +namespace Serval.Machine.Shared.Services; + +public record PreprocessStats +{ + public required int TrainCount { get; init; } + public required int InferenceCount { get; init; } + public required bool IsTrainFilteredByChapter { get; init; } + public required bool IsInferenceFilteredByChapter { get; init; } + public required Dictionary> TrainVerseCount { get; init; } + public required Dictionary> InferenceVerseCount { get; init; } +} diff --git a/src/Machine/src/Serval.Machine.Shared/Services/ServalTranslationPlatformService.cs b/src/Machine/src/Serval.Machine.Shared/Services/ServalTranslationPlatformService.cs index 1eb9ebb44..fb848fc75 100644 --- a/src/Machine/src/Serval.Machine.Shared/Services/ServalTranslationPlatformService.cs +++ b/src/Machine/src/Serval.Machine.Shared/Services/ServalTranslationPlatformService.cs @@ -101,7 +101,11 @@ public Task UpdateBuildExecutionDataAsync( new ExecutionDataContract { TrainCount = executionData.TrainCount, - PretranslateCount = executionData.PretranslateCount, + PretranslateCount = executionData.InferenceCount, + TrainVerseCount = executionData.TrainVerseCount, + PretranslateVerseCount = executionData.InferenceVerseCount, + IsTrainFilteredByChapter = executionData.IsTrainFilteredByChapter, + IsPretranslateFilteredByChapter = executionData.IsInferenceFilteredByChapter, Warnings = executionData.Warnings, EngineSourceLanguageTag = executionData.EngineSourceLanguageTag, EngineTargetLanguageTag = executionData.EngineTargetLanguageTag, diff --git a/src/Machine/src/Serval.Machine.Shared/Services/ServalWordAlignmentPlatformService.cs b/src/Machine/src/Serval.Machine.Shared/Services/ServalWordAlignmentPlatformService.cs index 29217d33f..0cb970990 100644 --- a/src/Machine/src/Serval.Machine.Shared/Services/ServalWordAlignmentPlatformService.cs +++ b/src/Machine/src/Serval.Machine.Shared/Services/ServalWordAlignmentPlatformService.cs @@ -100,7 +100,11 @@ public Task UpdateBuildExecutionDataAsync( new ExecutionDataContract { TrainCount = executionData.TrainCount, - WordAlignCount = executionData.WordAlignCount, + WordAlignCount = executionData.InferenceCount, + IsTrainFilteredByChapter = executionData.IsTrainFilteredByChapter, + IsWordAlignmentFilteredByChapter = executionData.IsInferenceFilteredByChapter, + TrainVerseCount = executionData.TrainVerseCount, + WordAlignVerseCount = executionData.InferenceVerseCount, Warnings = executionData.Warnings, EngineSourceLanguageTag = executionData.EngineSourceLanguageTag, EngineTargetLanguageTag = executionData.EngineTargetLanguageTag, diff --git a/src/Machine/src/Serval.Machine.Shared/Services/TranslationPreprocessBuildJob.cs b/src/Machine/src/Serval.Machine.Shared/Services/TranslationPreprocessBuildJob.cs index 606c11a20..7b06970fd 100644 --- a/src/Machine/src/Serval.Machine.Shared/Services/TranslationPreprocessBuildJob.cs +++ b/src/Machine/src/Serval.Machine.Shared/Services/TranslationPreprocessBuildJob.cs @@ -1,3 +1,5 @@ +using SIL.Extensions; + namespace Serval.Machine.Shared.Services; public class TranslationPreprocessBuildJob( @@ -21,7 +23,7 @@ IOptionsMonitor options options ) { - protected override async Task<(int TrainCount, int InferenceCount)> WriteDataFilesAsync( + protected override async Task WriteDataFilesAsync( string buildId, IReadOnlyList parallelCorpora, string? buildOptions, @@ -51,9 +53,20 @@ await SharedFileService.OpenWriteAsync($"builds/{buildId}/train.key-terms.trg.tx cancellationToken ); await using Utf8JsonWriter pretranslateWriter = new(pretranslateStream, InferenceWriterOptions); - + bool isTrainFilteredByChapter = parallelCorpora.Any(pc => + pc.SourceCorpora.Any(c => + c.TrainOnChapters is not null && c.TrainOnChapters.Values.Any(chapters => chapters.Count > 0) + ) + ); + bool isPretranslationFilteredByChapter = parallelCorpora.Any(pc => + pc.SourceCorpora.Any(c => + c.InferenceChapters is not null && c.InferenceChapters.Values.Any(chapters => chapters.Count > 0) + ) + ); int trainCount = 0; int pretranslateCount = 0; + Dictionary> trainVerseCountByChapter = []; + Dictionary> pretranslateVerseCountByChapter = []; pretranslateWriter.WriteStartArray(); await ParallelCorpusService.PreprocessAsync( parallelCorpora, @@ -73,7 +86,27 @@ await ParallelCorpusService.PreprocessAsync( } } if (row.SourceSegment.Length > 0 && row.TargetSegment.Length > 0) + { trainCount++; + foreach (object? reference in row.SourceRefs) + { + if (reference is not null and ScriptureRef sr) + { + trainVerseCountByChapter.UpdateValue( + sr.Book, + () => [], + chapters => + { + if (chapters.TryGetValue(sr.Chapter, out int count)) + chapters[sr.Chapter] = count + 1; + else + chapters[sr.Chapter] = 1; + return chapters; + } + ); + } + } + } }, async (row, isInTrainingData, corpusId) => { @@ -93,6 +126,24 @@ await ParallelCorpusService.PreprocessAsync( pretranslateWriter.WriteString("translation", row.SourceSegment); pretranslateWriter.WriteEndObject(); pretranslateCount++; + foreach (object? reference in row.SourceRefs) + { + if (reference is not null and ScriptureRef sr) + { + pretranslateVerseCountByChapter.UpdateValue( + sr.Book, + () => [], + chapters => + { + if (chapters.TryGetValue(sr.Chapter, out int count)) + chapters[sr.Chapter] = count + 1; + else + chapters[sr.Chapter] = 1; + return chapters; + } + ); + } + } } if (pretranslateWriter.BytesPending > 1024 * 1024) await pretranslateWriter.FlushAsync(); @@ -103,14 +154,21 @@ await ParallelCorpusService.PreprocessAsync( pretranslateWriter.WriteEndArray(); - return (trainCount, pretranslateCount); + return new PreprocessStats + { + TrainCount = trainCount, + InferenceCount = pretranslateCount, + IsTrainFilteredByChapter = isTrainFilteredByChapter, + IsInferenceFilteredByChapter = isPretranslationFilteredByChapter, + TrainVerseCount = trainVerseCountByChapter, + InferenceVerseCount = pretranslateVerseCountByChapter, + }; } protected override async Task UpdateBuildExecutionData( string engineId, string buildId, - int trainCount, - int pretranslateCount, + PreprocessStats stats, string sourceLanguageTag, string targetLanguageTag, IReadOnlyList parallelCorpora, @@ -118,8 +176,8 @@ CancellationToken cancellationToken ) { IReadOnlyList warnings = GetWarnings( - trainCount, - pretranslateCount, + stats.TrainCount, + stats.InferenceCount, sourceLanguageTag, targetLanguageTag, parallelCorpora @@ -131,8 +189,8 @@ CancellationToken cancellationToken { "Event", "BuildPreprocess" }, { "EngineId", engineId }, { "BuildId", buildId }, - { "NumTrainRows", trainCount }, - { "NumPretranslateRows", pretranslateCount }, + { "NumTrainRows", stats.TrainCount }, + { "NumPretranslateRows", stats.InferenceCount }, { "EngineSourceLanguageTag", sourceLanguageTag }, { "EngineTargetLanguageTag", targetLanguageTag }, { "Warnings", new JsonArray(warnings.Select(w => JsonValue.Create(w)).ToArray()) }, @@ -140,8 +198,12 @@ CancellationToken cancellationToken Logger.LogInformation("{summary}", buildPreprocessSummary.ToJsonString()); var executionData = new BuildExecutionData() { - TrainCount = trainCount, - PretranslateCount = pretranslateCount, + TrainCount = stats.TrainCount, + InferenceCount = stats.InferenceCount, + IsTrainFilteredByChapter = stats.IsTrainFilteredByChapter, + IsInferenceFilteredByChapter = stats.IsInferenceFilteredByChapter, + TrainVerseCount = stats.TrainVerseCount, + InferenceVerseCount = stats.InferenceVerseCount, Warnings = warnings, EngineSourceLanguageTag = sourceLanguageTag, EngineTargetLanguageTag = targetLanguageTag, diff --git a/src/Machine/src/Serval.Machine.Shared/Services/WordAlignmentPreprocessBuildJob.cs b/src/Machine/src/Serval.Machine.Shared/Services/WordAlignmentPreprocessBuildJob.cs index ae3dc0447..eb4452cf4 100644 --- a/src/Machine/src/Serval.Machine.Shared/Services/WordAlignmentPreprocessBuildJob.cs +++ b/src/Machine/src/Serval.Machine.Shared/Services/WordAlignmentPreprocessBuildJob.cs @@ -1,4 +1,6 @@ -namespace Serval.Machine.Shared.Services; +using SIL.Extensions; + +namespace Serval.Machine.Shared.Services; public class WordAlignmentPreprocessBuildJob( [FromKeyedServices(EngineGroup.WordAlignment)] IPlatformService platformService, @@ -21,7 +23,7 @@ IOptionsMonitor options options ) { - protected override async Task<(int TrainCount, int InferenceCount)> WriteDataFilesAsync( + protected override async Task WriteDataFilesAsync( string buildId, IReadOnlyList parallelCorpora, string? buildOptions, @@ -51,9 +53,20 @@ await SharedFileService.OpenWriteAsync($"builds/{buildId}/train.key-terms.trg.tx cancellationToken ); await using Utf8JsonWriter wordAlignmentWriter = new(wordAlignmentStream, InferenceWriterOptions); - + bool isTrainFilteredByChapter = parallelCorpora.Any(pc => + pc.SourceCorpora.Any(c => + c.TrainOnChapters is not null && c.TrainOnChapters.Values.Any(chapters => chapters.Count > 0) + ) + ); + bool isWordAlignmentFilteredByChapter = parallelCorpora.Any(pc => + pc.SourceCorpora.Any(c => + c.InferenceChapters is not null && c.InferenceChapters.Values.Any(chapters => chapters.Count > 0) + ) + ); int trainCount = 0; int inferenceCount = 0; + Dictionary> trainVerseCountByChapter = []; + Dictionary> wordAlignVerseCountByChapter = []; wordAlignmentWriter.WriteStartArray(); await ParallelCorpusService.PreprocessAsync( parallelCorpora, @@ -73,6 +86,24 @@ await ParallelCorpusService.PreprocessAsync( } trainCount++; + foreach (object? reference in row.SourceRefs) + { + if (reference is not null and ScriptureRef sr) + { + trainVerseCountByChapter.UpdateValue( + sr.Book, + () => [], + chapters => + { + if (chapters.TryGetValue(sr.Chapter, out int count)) + chapters[sr.Chapter] = count + 1; + else + chapters[sr.Chapter] = 1; + return chapters; + } + ); + } + } } }, async (row, isInTrainingData, corpusId) => @@ -94,6 +125,24 @@ await ParallelCorpusService.PreprocessAsync( wordAlignmentWriter.WriteString("target", row.TargetSegment); wordAlignmentWriter.WriteEndObject(); inferenceCount++; + foreach (object? reference in row.SourceRefs) + { + if (reference is not null and ScriptureRef sr) + { + wordAlignVerseCountByChapter.UpdateValue( + sr.Book, + () => [], + chapters => + { + if (chapters.TryGetValue(sr.Chapter, out int count)) + chapters[sr.Chapter] = count + 1; + else + chapters[sr.Chapter] = 1; + return chapters; + } + ); + } + } } if (wordAlignmentWriter.BytesPending > 1024 * 1024) await wordAlignmentWriter.FlushAsync(); @@ -103,14 +152,21 @@ await ParallelCorpusService.PreprocessAsync( wordAlignmentWriter.WriteEndArray(); - return (trainCount, inferenceCount); + return new PreprocessStats + { + TrainCount = trainCount, + InferenceCount = inferenceCount, + IsTrainFilteredByChapter = isTrainFilteredByChapter, + IsInferenceFilteredByChapter = isWordAlignmentFilteredByChapter, + TrainVerseCount = trainVerseCountByChapter, + InferenceVerseCount = wordAlignVerseCountByChapter, + }; } protected override async Task UpdateBuildExecutionData( string engineId, string buildId, - int trainCount, - int wordAlignCount, + PreprocessStats stats, string sourceLanguageTag, string targetLanguageTag, IReadOnlyList parallelCorpora, @@ -118,8 +174,8 @@ CancellationToken cancellationToken ) { IReadOnlyList warnings = GetWarnings( - trainCount, - wordAlignCount, + stats.TrainCount, + stats.InferenceCount, sourceLanguageTag, targetLanguageTag, parallelCorpora @@ -131,8 +187,8 @@ CancellationToken cancellationToken { "Event", "BuildPreprocess" }, { "EngineId", engineId }, { "BuildId", buildId }, - { "NumTrainRows", trainCount }, - { "NumWordAlignRows", wordAlignCount }, + { "NumTrainRows", stats.TrainCount }, + { "NumWordAlignRows", stats.InferenceCount }, { "EngineSourceLanguageTag", sourceLanguageTag }, { "EngineTargetLanguageTag", targetLanguageTag }, { "Warnings", new JsonArray(warnings.Select(w => JsonValue.Create(w)).ToArray()) }, @@ -140,8 +196,12 @@ CancellationToken cancellationToken Logger.LogInformation("{summary}", buildPreprocessSummary.ToJsonString()); var executionData = new BuildExecutionData() { - TrainCount = trainCount, - WordAlignCount = wordAlignCount, + TrainCount = stats.TrainCount, + InferenceCount = stats.InferenceCount, + TrainVerseCount = stats.TrainVerseCount, + InferenceVerseCount = stats.InferenceVerseCount, + IsInferenceFilteredByChapter = stats.IsInferenceFilteredByChapter, + IsTrainFilteredByChapter = stats.IsTrainFilteredByChapter, Warnings = warnings, EngineSourceLanguageTag = sourceLanguageTag, EngineTargetLanguageTag = targetLanguageTag, diff --git a/src/Serval/src/Serval.Client/Client.g.cs b/src/Serval/src/Serval.Client/Client.g.cs index 1b173fbd7..2fe10c2af 100644 --- a/src/Serval/src/Serval.Client/Client.g.cs +++ b/src/Serval/src/Serval.Client/Client.g.cs @@ -11505,6 +11505,18 @@ public partial class ExecutionData [Newtonsoft.Json.JsonProperty("pretranslateCount", Required = Newtonsoft.Json.Required.Always)] public int PretranslateCount { get; set; } = default!; + [Newtonsoft.Json.JsonProperty("isTrainFilteredByChapter", Required = Newtonsoft.Json.Required.Always)] + public bool IsTrainFilteredByChapter { get; set; } = default!; + + [Newtonsoft.Json.JsonProperty("isPretranslateFilteredByChapter", Required = Newtonsoft.Json.Required.Always)] + public bool IsPretranslateFilteredByChapter { get; set; } = default!; + + [Newtonsoft.Json.JsonProperty("trainVerseCount", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + public System.Collections.Generic.IDictionary>? TrainVerseCount { get; set; } = default!; + + [Newtonsoft.Json.JsonProperty("pretranslateVerseCount", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + public System.Collections.Generic.IDictionary>? PretranslateVerseCount { get; set; } = default!; + [Newtonsoft.Json.JsonProperty("warnings", Required = Newtonsoft.Json.Required.Always)] [System.ComponentModel.DataAnnotations.Required] public System.Collections.Generic.IList Warnings { get; set; } = new System.Collections.ObjectModel.Collection(); @@ -12312,6 +12324,18 @@ public partial class WordAlignmentExecutionData [Newtonsoft.Json.JsonProperty("wordAlignCount", Required = Newtonsoft.Json.Required.Always)] public int WordAlignCount { get; set; } = default!; + [Newtonsoft.Json.JsonProperty("isTrainFilteredByChapter", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + public bool? IsTrainFilteredByChapter { get; set; } = default!; + + [Newtonsoft.Json.JsonProperty("isWordAlignFilteredByChapter", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + public bool? IsWordAlignFilteredByChapter { get; set; } = default!; + + [Newtonsoft.Json.JsonProperty("trainVerseCount", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + public System.Collections.Generic.IDictionary>? TrainVerseCount { get; set; } = default!; + + [Newtonsoft.Json.JsonProperty("wordAlignVerseCount", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + public System.Collections.Generic.IDictionary>? WordAlignVerseCount { get; set; } = default!; + [Newtonsoft.Json.JsonProperty("warnings", Required = Newtonsoft.Json.Required.Always)] [System.ComponentModel.DataAnnotations.Required] public System.Collections.Generic.IList Warnings { get; set; } = new System.Collections.ObjectModel.Collection(); diff --git a/src/Serval/src/Serval.Translation.Contracts/ExecutionDataContract.cs b/src/Serval/src/Serval.Translation.Contracts/ExecutionDataContract.cs index 26c31b531..b256eec9c 100644 --- a/src/Serval/src/Serval.Translation.Contracts/ExecutionDataContract.cs +++ b/src/Serval/src/Serval.Translation.Contracts/ExecutionDataContract.cs @@ -4,6 +4,10 @@ public record ExecutionDataContract { public int? TrainCount { get; init; } public int? PretranslateCount { get; init; } + public bool? IsTrainFilteredByChapter { get; init; } + public bool? IsPretranslateFilteredByChapter { get; init; } + public Dictionary>? TrainVerseCount { get; init; } + public Dictionary>? PretranslateVerseCount { get; init; } public IReadOnlyList? Warnings { get; init; } public string? EngineSourceLanguageTag { get; init; } public string? EngineTargetLanguageTag { get; init; } diff --git a/src/Serval/src/Serval.Translation/Dtos/ExecutionDataDto.cs b/src/Serval/src/Serval.Translation/Dtos/ExecutionDataDto.cs index ef48a34a8..da01c29de 100644 --- a/src/Serval/src/Serval.Translation/Dtos/ExecutionDataDto.cs +++ b/src/Serval/src/Serval.Translation/Dtos/ExecutionDataDto.cs @@ -4,6 +4,10 @@ public record ExecutionDataDto { public int TrainCount { get; init; } public int PretranslateCount { get; init; } + public bool IsTrainFilteredByChapter { get; init; } + public bool IsPretranslateFilteredByChapter { get; init; } + public Dictionary>? TrainVerseCount { get; init; } + public Dictionary>? PretranslateVerseCount { get; init; } public IReadOnlyList Warnings { get; init; } = []; public string? EngineSourceLanguageTag { get; init; } public string? EngineTargetLanguageTag { get; init; } diff --git a/src/Serval/src/Serval.Translation/Models/ExecutionData.cs b/src/Serval/src/Serval.Translation/Models/ExecutionData.cs index 79c953e73..646b53e71 100644 --- a/src/Serval/src/Serval.Translation/Models/ExecutionData.cs +++ b/src/Serval/src/Serval.Translation/Models/ExecutionData.cs @@ -4,6 +4,10 @@ public record ExecutionData { public int? TrainCount { get; init; } public int? PretranslateCount { get; init; } + public bool? IsTrainFilteredByChapter { get; init; } + public bool? IsPretranslateFilteredByChapter { get; init; } + public Dictionary>? TrainVerseCount { get; init; } + public Dictionary>? PretranslateVerseCount { get; init; } public IReadOnlyList? Warnings { get; init; } public string? EngineSourceLanguageTag { get; init; } public string? EngineTargetLanguageTag { get; init; } diff --git a/src/Serval/src/Serval.Translation/Services/DtoMapper.cs b/src/Serval/src/Serval.Translation/Services/DtoMapper.cs index 8dd3e08fc..ad09d3557 100644 --- a/src/Serval/src/Serval.Translation/Services/DtoMapper.cs +++ b/src/Serval/src/Serval.Translation/Services/DtoMapper.cs @@ -231,6 +231,10 @@ private static ExecutionDataDto Map(ExecutionData source) => { TrainCount = source.TrainCount ?? 0, PretranslateCount = source.PretranslateCount ?? 0, + PretranslateVerseCount = source.PretranslateVerseCount, + TrainVerseCount = source.TrainVerseCount, + IsPretranslateFilteredByChapter = source.IsPretranslateFilteredByChapter ?? false, + IsTrainFilteredByChapter = source.IsTrainFilteredByChapter ?? false, Warnings = source.Warnings ?? [], EngineSourceLanguageTag = source.EngineSourceLanguageTag, EngineTargetLanguageTag = source.EngineTargetLanguageTag, diff --git a/src/Serval/src/Serval.Translation/Services/PlatformService.cs b/src/Serval/src/Serval.Translation/Services/PlatformService.cs index 1616bd6ba..9b88daab1 100644 --- a/src/Serval/src/Serval.Translation/Services/PlatformService.cs +++ b/src/Serval/src/Serval.Translation/Services/PlatformService.cs @@ -311,6 +311,10 @@ await _builds.UpdateAsync( { TrainCount = executionData.TrainCount, PretranslateCount = executionData.PretranslateCount, + TrainVerseCount = executionData.TrainVerseCount, + PretranslateVerseCount = executionData.PretranslateVerseCount, + IsTrainFilteredByChapter = executionData.IsTrainFilteredByChapter, + IsPretranslateFilteredByChapter = executionData.IsPretranslateFilteredByChapter, Warnings = executionData.Warnings?.ToList() ?? [], EngineSourceLanguageTag = executionData.EngineSourceLanguageTag, EngineTargetLanguageTag = executionData.EngineTargetLanguageTag, diff --git a/src/Serval/src/Serval.WordAlignment.Contracts/ExecutionDataContract.cs b/src/Serval/src/Serval.WordAlignment.Contracts/ExecutionDataContract.cs index 78e756726..0227a5eef 100644 --- a/src/Serval/src/Serval.WordAlignment.Contracts/ExecutionDataContract.cs +++ b/src/Serval/src/Serval.WordAlignment.Contracts/ExecutionDataContract.cs @@ -4,6 +4,10 @@ public record ExecutionDataContract { public int? TrainCount { get; init; } public int? WordAlignCount { get; init; } + public bool? IsTrainFilteredByChapter { get; init; } + public bool? IsWordAlignmentFilteredByChapter { get; init; } + public Dictionary>? TrainVerseCount { get; init; } + public Dictionary>? WordAlignVerseCount { get; init; } public IReadOnlyList? Warnings { get; init; } public string? EngineSourceLanguageTag { get; init; } public string? EngineTargetLanguageTag { get; init; } diff --git a/src/Serval/src/Serval.WordAlignment/Dtos/WordAlignmentExecutionDataDto.cs b/src/Serval/src/Serval.WordAlignment/Dtos/WordAlignmentExecutionDataDto.cs index 8f37edfc6..6aa2764f4 100644 --- a/src/Serval/src/Serval.WordAlignment/Dtos/WordAlignmentExecutionDataDto.cs +++ b/src/Serval/src/Serval.WordAlignment/Dtos/WordAlignmentExecutionDataDto.cs @@ -4,6 +4,10 @@ public record WordAlignmentExecutionDataDto { public int TrainCount { get; init; } public int WordAlignCount { get; init; } + public bool? IsTrainFilteredByChapter { get; init; } + public bool? IsWordAlignFilteredByChapter { get; init; } + public Dictionary>? TrainVerseCount { get; init; } + public Dictionary>? WordAlignVerseCount { get; init; } public IReadOnlyList Warnings { get; init; } = []; public string? EngineSourceLanguageTag { get; init; } public string? EngineTargetLanguageTag { get; init; } diff --git a/src/Serval/src/Serval.WordAlignment/Models/ExecutionData.cs b/src/Serval/src/Serval.WordAlignment/Models/ExecutionData.cs index c606e08ff..5908a1093 100644 --- a/src/Serval/src/Serval.WordAlignment/Models/ExecutionData.cs +++ b/src/Serval/src/Serval.WordAlignment/Models/ExecutionData.cs @@ -4,6 +4,10 @@ public record ExecutionData { public int? TrainCount { get; init; } public int? WordAlignCount { get; init; } + public bool? IsTrainFilteredByChapter { get; init; } + public bool? IsWordAlignFilteredByChapter { get; init; } + public Dictionary>? TrainVerseCount { get; init; } + public Dictionary>? WordAlignVerseCount { get; init; } public IReadOnlyList? Warnings { get; init; } public string? EngineSourceLanguageTag { get; init; } public string? EngineTargetLanguageTag { get; init; } diff --git a/src/Serval/src/Serval.WordAlignment/Services/DtoMapper.cs b/src/Serval/src/Serval.WordAlignment/Services/DtoMapper.cs index 0731b2626..ec36d427c 100644 --- a/src/Serval/src/Serval.WordAlignment/Services/DtoMapper.cs +++ b/src/Serval/src/Serval.WordAlignment/Services/DtoMapper.cs @@ -190,6 +190,10 @@ private static WordAlignmentExecutionDataDto Map(ExecutionData source) { TrainCount = source.TrainCount ?? 0, WordAlignCount = source.WordAlignCount ?? 0, + TrainVerseCount = source.TrainVerseCount ?? [], + WordAlignVerseCount = source.WordAlignVerseCount ?? [], + IsTrainFilteredByChapter = source.IsTrainFilteredByChapter, + IsWordAlignFilteredByChapter = source.IsWordAlignFilteredByChapter, Warnings = source.Warnings ?? [], EngineSourceLanguageTag = source.EngineSourceLanguageTag, EngineTargetLanguageTag = source.EngineTargetLanguageTag, diff --git a/src/Serval/src/Serval.WordAlignment/Services/PlatformService.cs b/src/Serval/src/Serval.WordAlignment/Services/PlatformService.cs index 100aaaee5..2b2de6314 100644 --- a/src/Serval/src/Serval.WordAlignment/Services/PlatformService.cs +++ b/src/Serval/src/Serval.WordAlignment/Services/PlatformService.cs @@ -363,6 +363,10 @@ await _builds.UpdateAsync( { TrainCount = executionData.TrainCount, WordAlignCount = executionData.WordAlignCount, + TrainVerseCount = executionData.TrainVerseCount, + WordAlignVerseCount = executionData.WordAlignVerseCount, + IsTrainFilteredByChapter = executionData.IsTrainFilteredByChapter, + IsWordAlignFilteredByChapter = executionData.IsWordAlignmentFilteredByChapter, Warnings = executionData.Warnings?.ToList() ?? [], EngineSourceLanguageTag = executionData.EngineSourceLanguageTag, EngineTargetLanguageTag = executionData.EngineTargetLanguageTag, diff --git a/src/Serval/test/Serval.Translation.Tests/Services/PlatformServiceTests.cs b/src/Serval/test/Serval.Translation.Tests/Services/PlatformServiceTests.cs index 9e0d09011..9a2ff36d7 100644 --- a/src/Serval/test/Serval.Translation.Tests/Services/PlatformServiceTests.cs +++ b/src/Serval/test/Serval.Translation.Tests/Services/PlatformServiceTests.cs @@ -157,7 +157,27 @@ public async Task UpdateBuildExecutionData() await env.PlatformService.UpdateBuildExecutionDataAsync( engine.Id, "123", - new() { TrainCount = 4, PretranslateCount = 5 } + new() + { + TrainCount = 4, + PretranslateCount = 5, + TrainVerseCount = new() + { + { + "MAT", + new() { { "1", 4 } } + }, + }, + PretranslateVerseCount = new() + { + { + "MAT", + new() { { "1", 10 } } + }, + }, + IsTrainFilteredByChapter = false, + IsPretranslateFilteredByChapter = true, + } ); build = await env.Builds.GetAsync(c => c.Id == build.Id); @@ -167,6 +187,19 @@ await env.PlatformService.UpdateBuildExecutionDataAsync( Assert.That(executionData, Is.Not.Null); Assert.That(executionData.TrainCount, Is.GreaterThan(0)); Assert.That(executionData.PretranslateCount, Is.GreaterThan(0)); + + Assert.That(executionData.TrainVerseCount, Is.Not.Null); + Assert.That(executionData.TrainVerseCount, Has.Count.EqualTo(1)); + Assert.That(executionData.TrainVerseCount.First().Value, Has.Count.EqualTo(1)); + Assert.That(executionData.TrainVerseCount.First().Value.First().Value, Is.EqualTo(4)); + + Assert.That(executionData.PretranslateVerseCount, Is.Not.Null); + Assert.That(executionData.PretranslateVerseCount, Has.Count.EqualTo(1)); + Assert.That(executionData.PretranslateVerseCount.First().Value, Has.Count.EqualTo(1)); + Assert.That(executionData.PretranslateVerseCount.First().Value.First().Value, Is.EqualTo(10)); + + Assert.That(executionData.IsTrainFilteredByChapter, Is.False); + Assert.That(executionData.IsPretranslateFilteredByChapter, Is.True); } [Test] From 22463b833c002d5bb87d527545e47e4dcd2e0716 Mon Sep 17 00:00:00 2001 From: Enkidu93 Date: Wed, 8 Jul 2026 11:09:31 -0400 Subject: [PATCH 2/2] Extract preprocessing functions for testing --- .../Services/PreprocessStats.cs | 12 +- .../Services/TranslationPreprocessBuildJob.cs | 202 ++++++---- .../WordAlignmentPreprocessBuildJob.cs | 201 +++++---- .../Services/PreprocessBuildJobTests.cs | 381 ++++++++++++++++++ 4 files changed, 613 insertions(+), 183 deletions(-) diff --git a/src/Machine/src/Serval.Machine.Shared/Services/PreprocessStats.cs b/src/Machine/src/Serval.Machine.Shared/Services/PreprocessStats.cs index d115e03ea..781d292be 100644 --- a/src/Machine/src/Serval.Machine.Shared/Services/PreprocessStats.cs +++ b/src/Machine/src/Serval.Machine.Shared/Services/PreprocessStats.cs @@ -2,10 +2,10 @@ namespace Serval.Machine.Shared.Services; public record PreprocessStats { - public required int TrainCount { get; init; } - public required int InferenceCount { get; init; } - public required bool IsTrainFilteredByChapter { get; init; } - public required bool IsInferenceFilteredByChapter { get; init; } - public required Dictionary> TrainVerseCount { get; init; } - public required Dictionary> InferenceVerseCount { get; init; } + public int TrainCount { get; set; } + public int InferenceCount { get; set; } + public bool IsTrainFilteredByChapter { get; set; } + public bool IsInferenceFilteredByChapter { get; set; } + public Dictionary> TrainVerseCount { get; set; } = []; + public Dictionary> InferenceVerseCount { get; set; } = []; } diff --git a/src/Machine/src/Serval.Machine.Shared/Services/TranslationPreprocessBuildJob.cs b/src/Machine/src/Serval.Machine.Shared/Services/TranslationPreprocessBuildJob.cs index 7b06970fd..4cc8aaa31 100644 --- a/src/Machine/src/Serval.Machine.Shared/Services/TranslationPreprocessBuildJob.cs +++ b/src/Machine/src/Serval.Machine.Shared/Services/TranslationPreprocessBuildJob.cs @@ -63,106 +63,33 @@ await SharedFileService.OpenWriteAsync($"builds/{buildId}/train.key-terms.trg.tx c.InferenceChapters is not null && c.InferenceChapters.Values.Any(chapters => chapters.Count > 0) ) ); - int trainCount = 0; - int pretranslateCount = 0; - Dictionary> trainVerseCountByChapter = []; - Dictionary> pretranslateVerseCountByChapter = []; + PreprocessStats preprocessStats = new() + { + IsTrainFilteredByChapter = isTrainFilteredByChapter, + IsInferenceFilteredByChapter = isPretranslationFilteredByChapter, + }; + pretranslateWriter.WriteStartArray(); await ParallelCorpusService.PreprocessAsync( parallelCorpora, async (row, trainingDataType) => - { - if (row.SourceSegment.Length > 0 || row.TargetSegment.Length > 0) - { - if (trainingDataType == TrainingDataType.KeyTerm) - { - await sourceKeyTermsTrainWriter.WriteAsync($"{row.SourceSegment}\n"); - await targetKeyTermsTrainWriter.WriteAsync($"{row.TargetSegment}\n"); - } - else - { - await sourceTrainWriter.WriteAsync($"{row.SourceSegment}\n"); - await targetTrainWriter.WriteAsync($"{row.TargetSegment}\n"); - } - } - if (row.SourceSegment.Length > 0 && row.TargetSegment.Length > 0) - { - trainCount++; - foreach (object? reference in row.SourceRefs) - { - if (reference is not null and ScriptureRef sr) - { - trainVerseCountByChapter.UpdateValue( - sr.Book, - () => [], - chapters => - { - if (chapters.TryGetValue(sr.Chapter, out int count)) - chapters[sr.Chapter] = count + 1; - else - chapters[sr.Chapter] = 1; - return chapters; - } - ); - } - } - } - }, + await preprocessStats.ProcessTranslationTrainingRowAsync( + row, + trainingDataType, + sourceTrainWriter, + targetTrainWriter, + sourceKeyTermsTrainWriter, + targetKeyTermsTrainWriter + ), async (row, isInTrainingData, corpusId) => - { - if (row.SourceSegment.Length > 0 && !isInTrainingData) - { - pretranslateWriter.WriteStartObject(); - pretranslateWriter.WriteString("corpusId", corpusId); - pretranslateWriter.WriteString("textId", row.TextId); - pretranslateWriter.WriteStartArray("sourceRefs"); - foreach (object rowRef in row.SourceRefs) - pretranslateWriter.WriteStringValue(rowRef.ToString()); - pretranslateWriter.WriteEndArray(); - pretranslateWriter.WriteStartArray("targetRefs"); - foreach (object rowRef in row.TargetRefs) - pretranslateWriter.WriteStringValue(rowRef.ToString()); - pretranslateWriter.WriteEndArray(); - pretranslateWriter.WriteString("translation", row.SourceSegment); - pretranslateWriter.WriteEndObject(); - pretranslateCount++; - foreach (object? reference in row.SourceRefs) - { - if (reference is not null and ScriptureRef sr) - { - pretranslateVerseCountByChapter.UpdateValue( - sr.Book, - () => [], - chapters => - { - if (chapters.TryGetValue(sr.Chapter, out int count)) - chapters[sr.Chapter] = count + 1; - else - chapters[sr.Chapter] = 1; - return chapters; - } - ); - } - } - } - if (pretranslateWriter.BytesPending > 1024 * 1024) - await pretranslateWriter.FlushAsync(); - }, + await preprocessStats.ProcessPretranslateRowAsync(row, isInTrainingData, corpusId, pretranslateWriter), (bool?)buildOptionsObject?["use_key_terms"] ?? true, ignoreUsfmMarkers: ["rem", "r"] ); pretranslateWriter.WriteEndArray(); - return new PreprocessStats - { - TrainCount = trainCount, - InferenceCount = pretranslateCount, - IsTrainFilteredByChapter = isTrainFilteredByChapter, - IsInferenceFilteredByChapter = isPretranslationFilteredByChapter, - TrainVerseCount = trainVerseCountByChapter, - InferenceVerseCount = pretranslateVerseCountByChapter, - }; + return preprocessStats; } protected override async Task UpdateBuildExecutionData( @@ -211,3 +138,100 @@ CancellationToken cancellationToken await PlatformService.UpdateBuildExecutionDataAsync(engineId, buildId, executionData, cancellationToken); } } + +public static partial class PreprocessStatsExtensions +{ + public static async Task ProcessTranslationTrainingRowAsync( + this PreprocessStats stats, + ParallelRowContract row, + TrainingDataType trainingDataType, + StreamWriter sourceTrainWriter, + StreamWriter targetTrainWriter, + StreamWriter sourceKeyTermsTrainWriter, + StreamWriter targetKeyTermsTrainWriter + ) + { + if (row.SourceSegment.Length > 0 || row.TargetSegment.Length > 0) + { + if (trainingDataType == TrainingDataType.KeyTerm) + { + await sourceKeyTermsTrainWriter.WriteAsync($"{row.SourceSegment}\n"); + await targetKeyTermsTrainWriter.WriteAsync($"{row.TargetSegment}\n"); + } + else + { + await sourceTrainWriter.WriteAsync($"{row.SourceSegment}\n"); + await targetTrainWriter.WriteAsync($"{row.TargetSegment}\n"); + } + } + if (row.SourceSegment.Length > 0 && row.TargetSegment.Length > 0) + { + stats.TrainCount++; + foreach (object? reference in row.SourceRefs) + { + if (reference is not null and ScriptureRef sr && sr.IsVerse) + { + stats.TrainVerseCount.UpdateValue( + sr.Book, + () => [], + chapters => + { + if (chapters.TryGetValue(sr.Chapter, out int count)) + chapters[sr.Chapter] = count + 1; + else + chapters[sr.Chapter] = 1; + return chapters; + } + ); + } + } + } + } + + public static async Task ProcessPretranslateRowAsync( + this PreprocessStats stats, + ParallelRowContract row, + bool isInTrainingData, + string corpusId, + Utf8JsonWriter pretranslateWriter + ) + { + if (row.SourceSegment.Length > 0 && !isInTrainingData) + { + pretranslateWriter.WriteStartObject(); + pretranslateWriter.WriteString("corpusId", corpusId); + pretranslateWriter.WriteString("textId", row.TextId); + pretranslateWriter.WriteStartArray("sourceRefs"); + foreach (object rowRef in row.SourceRefs) + pretranslateWriter.WriteStringValue(rowRef.ToString()); + pretranslateWriter.WriteEndArray(); + pretranslateWriter.WriteStartArray("targetRefs"); + foreach (object rowRef in row.TargetRefs) + pretranslateWriter.WriteStringValue(rowRef.ToString()); + pretranslateWriter.WriteEndArray(); + pretranslateWriter.WriteString("translation", row.SourceSegment); + pretranslateWriter.WriteEndObject(); + stats.InferenceCount++; + foreach (object? reference in row.SourceRefs) + { + if (reference is not null and ScriptureRef sr && sr.IsVerse) + { + stats.InferenceVerseCount.UpdateValue( + sr.Book, + () => [], + chapters => + { + if (chapters.TryGetValue(sr.Chapter, out int count)) + chapters[sr.Chapter] = count + 1; + else + chapters[sr.Chapter] = 1; + return chapters; + } + ); + } + } + } + if (pretranslateWriter.BytesPending > 1024 * 1024) + await pretranslateWriter.FlushAsync(); + } +} diff --git a/src/Machine/src/Serval.Machine.Shared/Services/WordAlignmentPreprocessBuildJob.cs b/src/Machine/src/Serval.Machine.Shared/Services/WordAlignmentPreprocessBuildJob.cs index eb4452cf4..d29804c00 100644 --- a/src/Machine/src/Serval.Machine.Shared/Services/WordAlignmentPreprocessBuildJob.cs +++ b/src/Machine/src/Serval.Machine.Shared/Services/WordAlignmentPreprocessBuildJob.cs @@ -63,104 +63,33 @@ await SharedFileService.OpenWriteAsync($"builds/{buildId}/train.key-terms.trg.tx c.InferenceChapters is not null && c.InferenceChapters.Values.Any(chapters => chapters.Count > 0) ) ); - int trainCount = 0; - int inferenceCount = 0; - Dictionary> trainVerseCountByChapter = []; - Dictionary> wordAlignVerseCountByChapter = []; + + PreprocessStats preprocessStats = new() + { + IsTrainFilteredByChapter = isTrainFilteredByChapter, + IsInferenceFilteredByChapter = isWordAlignmentFilteredByChapter, + }; + wordAlignmentWriter.WriteStartArray(); await ParallelCorpusService.PreprocessAsync( parallelCorpora, async (row, trainingDataType) => - { - if (row.SourceSegment.Length > 0 && row.TargetSegment.Length > 0) - { - if (trainingDataType == TrainingDataType.KeyTerm) - { - await sourceKeyTermsTrainWriter.WriteAsync($"{row.SourceSegment}\n"); - await targetKeyTermsTrainWriter.WriteAsync($"{row.TargetSegment}\n"); - } - else - { - await sourceTrainWriter.WriteAsync($"{row.SourceSegment}\n"); - await targetTrainWriter.WriteAsync($"{row.TargetSegment}\n"); - } - - trainCount++; - foreach (object? reference in row.SourceRefs) - { - if (reference is not null and ScriptureRef sr) - { - trainVerseCountByChapter.UpdateValue( - sr.Book, - () => [], - chapters => - { - if (chapters.TryGetValue(sr.Chapter, out int count)) - chapters[sr.Chapter] = count + 1; - else - chapters[sr.Chapter] = 1; - return chapters; - } - ); - } - } - } - }, + await preprocessStats.ProcessWordAlignmentTrainingRowAsync( + row, + trainingDataType, + sourceTrainWriter, + targetTrainWriter, + sourceKeyTermsTrainWriter, + targetKeyTermsTrainWriter + ), async (row, isInTrainingData, corpusId) => - { - if (row.SourceSegment.Length > 0 && row.TargetSegment.Length > 0 && !isInTrainingData) - { - wordAlignmentWriter.WriteStartObject(); - wordAlignmentWriter.WriteString("corpusId", corpusId); - wordAlignmentWriter.WriteString("textId", row.TextId); - wordAlignmentWriter.WriteStartArray("sourceRefs"); - foreach (object rowRef in row.SourceRefs) - wordAlignmentWriter.WriteStringValue(rowRef.ToString()); - wordAlignmentWriter.WriteEndArray(); - wordAlignmentWriter.WriteStartArray("targetRefs"); - foreach (object rowRef in row.TargetRefs) - wordAlignmentWriter.WriteStringValue(rowRef.ToString()); - wordAlignmentWriter.WriteEndArray(); - wordAlignmentWriter.WriteString("source", row.SourceSegment); - wordAlignmentWriter.WriteString("target", row.TargetSegment); - wordAlignmentWriter.WriteEndObject(); - inferenceCount++; - foreach (object? reference in row.SourceRefs) - { - if (reference is not null and ScriptureRef sr) - { - wordAlignVerseCountByChapter.UpdateValue( - sr.Book, - () => [], - chapters => - { - if (chapters.TryGetValue(sr.Chapter, out int count)) - chapters[sr.Chapter] = count + 1; - else - chapters[sr.Chapter] = 1; - return chapters; - } - ); - } - } - } - if (wordAlignmentWriter.BytesPending > 1024 * 1024) - await wordAlignmentWriter.FlushAsync(); - }, + await preprocessStats.ProcessWordAlignRowAsync(row, isInTrainingData, corpusId, wordAlignmentWriter), (bool?)buildOptionsObject?["use_key_terms"] ?? true ); wordAlignmentWriter.WriteEndArray(); - return new PreprocessStats - { - TrainCount = trainCount, - InferenceCount = inferenceCount, - IsTrainFilteredByChapter = isTrainFilteredByChapter, - IsInferenceFilteredByChapter = isWordAlignmentFilteredByChapter, - TrainVerseCount = trainVerseCountByChapter, - InferenceVerseCount = wordAlignVerseCountByChapter, - }; + return preprocessStats; } protected override async Task UpdateBuildExecutionData( @@ -220,3 +149,99 @@ CancellationToken cancellationToken return Task.CompletedTask; } } + +public static partial class PreprocessStatsExtensions +{ + public static async Task ProcessWordAlignmentTrainingRowAsync( + this PreprocessStats stats, + ParallelRowContract row, + TrainingDataType trainingDataType, + StreamWriter sourceTrainWriter, + StreamWriter targetTrainWriter, + StreamWriter sourceKeyTermsTrainWriter, + StreamWriter targetKeyTermsTrainWriter + ) + { + if (row.SourceSegment.Length > 0 && row.TargetSegment.Length > 0) + { + if (trainingDataType == TrainingDataType.KeyTerm) + { + await sourceKeyTermsTrainWriter.WriteAsync($"{row.SourceSegment}\n"); + await targetKeyTermsTrainWriter.WriteAsync($"{row.TargetSegment}\n"); + } + else + { + await sourceTrainWriter.WriteAsync($"{row.SourceSegment}\n"); + await targetTrainWriter.WriteAsync($"{row.TargetSegment}\n"); + } + + stats.TrainCount++; + foreach (object? reference in row.SourceRefs) + { + if (reference is not null and ScriptureRef sr && sr.IsVerse) + { + stats.TrainVerseCount.UpdateValue( + sr.Book, + () => [], + chapters => + { + if (chapters.TryGetValue(sr.Chapter, out int count)) + chapters[sr.Chapter] = count + 1; + else + chapters[sr.Chapter] = 1; + return chapters; + } + ); + } + } + } + } + + public static async Task ProcessWordAlignRowAsync( + this PreprocessStats stats, + ParallelRowContract row, + bool isInTrainingData, + string corpusId, + Utf8JsonWriter wordAlignmentWriter + ) + { + if (row.SourceSegment.Length > 0 && row.TargetSegment.Length > 0 && !isInTrainingData) + { + wordAlignmentWriter.WriteStartObject(); + wordAlignmentWriter.WriteString("corpusId", corpusId); + wordAlignmentWriter.WriteString("textId", row.TextId); + wordAlignmentWriter.WriteStartArray("sourceRefs"); + foreach (object rowRef in row.SourceRefs) + wordAlignmentWriter.WriteStringValue(rowRef.ToString()); + wordAlignmentWriter.WriteEndArray(); + wordAlignmentWriter.WriteStartArray("targetRefs"); + foreach (object rowRef in row.TargetRefs) + wordAlignmentWriter.WriteStringValue(rowRef.ToString()); + wordAlignmentWriter.WriteEndArray(); + wordAlignmentWriter.WriteString("source", row.SourceSegment); + wordAlignmentWriter.WriteString("target", row.TargetSegment); + wordAlignmentWriter.WriteEndObject(); + stats.InferenceCount++; + foreach (object? reference in row.SourceRefs) + { + if (reference is not null and ScriptureRef sr && sr.IsVerse) + { + stats.InferenceVerseCount.UpdateValue( + sr.Book, + () => [], + chapters => + { + if (chapters.TryGetValue(sr.Chapter, out int count)) + chapters[sr.Chapter] = count + 1; + else + chapters[sr.Chapter] = 1; + return chapters; + } + ); + } + } + } + if (wordAlignmentWriter.BytesPending > 1024 * 1024) + await wordAlignmentWriter.FlushAsync(); + } +} diff --git a/src/Machine/test/Serval.Machine.Shared.Tests/Services/PreprocessBuildJobTests.cs b/src/Machine/test/Serval.Machine.Shared.Tests/Services/PreprocessBuildJobTests.cs index c0972defb..1350bf2b7 100644 --- a/src/Machine/test/Serval.Machine.Shared.Tests/Services/PreprocessBuildJobTests.cs +++ b/src/Machine/test/Serval.Machine.Shared.Tests/Services/PreprocessBuildJobTests.cs @@ -95,6 +95,387 @@ public async Task RunAsync_UnknownLanguageTagsNoDataSmtTransfer() await env.RunBuildJobAsync(corpus1, engineId: "engine3", engineType: EngineType.SmtTransfer); } + [Test] + public async Task ProcessTranslationRowsAsync() + { + PreprocessStats stats = new(); + + var stream = Substitute.For(); + stream.CanWrite.Returns(true); + + var sourceTrainWriter = Substitute.For(stream); + List sourceSegments = []; + sourceTrainWriter.When(x => x.WriteAsync(Arg.Any())).Do(c => sourceSegments.Add(c.ArgAt(0))); + + var targetTrainWriter = Substitute.For(stream); + List targetSegments = []; + targetTrainWriter.When(x => x.WriteAsync(Arg.Any())).Do(c => targetSegments.Add(c.ArgAt(0))); + + var sourceKeyTermsTrainWriter = Substitute.For(stream); + List sourceKeyTermSegments = []; + sourceKeyTermsTrainWriter + .When(x => x.WriteAsync(Arg.Any())) + .Do(c => sourceKeyTermSegments.Add(c.ArgAt(0))); + + var targetKeyTermsTrainWriter = Substitute.For(stream); + List targetKeyTermSegments = []; + targetKeyTermsTrainWriter + .When(x => x.WriteAsync(Arg.Any())) + .Do(c => targetKeyTermSegments.Add(c.ArgAt(0))); + + (ParallelRowContract Row, TrainingDataType Type)[] trainingRows = + [ + ( + new( + "MAT", + [ScriptureRef.Parse("MAT 1:1")], + [ScriptureRef.Parse("MAT 1:1")], + "Source Matthew 1:1", + "Target Matthew 1:1", + 1 + ), + TrainingDataType.Text + ), + ( + new( + "MAT", + [ScriptureRef.Parse("MAT 1:2")], + [ScriptureRef.Parse("MAT 1:2")], + "Source Matthew 1:2", + "", + 1 + ), + TrainingDataType.Text + ), + ( + new( + "MAT", + [ScriptureRef.Parse("MAT 2:1")], + [ScriptureRef.Parse("MAT 2:1")], + "Source Matthew 2:1", + "Target Matthew 2:1", + 1 + ), + TrainingDataType.Text + ), + ( + new( + "MRK", + [ScriptureRef.Parse("MRK 1:1")], + [ScriptureRef.Parse("MRK 1:1")], + "Source Mark 1:1", + "Target Mark 1:1", + 1 + ), + TrainingDataType.Text + ), + ( + new( + "MRK", + ["MajorBiblicalTerms:Isaac"], + ["MajorBiblicalTerms:Isaac"], + "Source Isaac", + "Target Isaac", + 1 + ), + TrainingDataType.KeyTerm + ), + ]; + foreach ((ParallelRowContract row, TrainingDataType type) in trainingRows) + { + await stats.ProcessTranslationTrainingRowAsync( + row, + type, + sourceTrainWriter, + targetTrainWriter, + sourceKeyTermsTrainWriter, + targetKeyTermsTrainWriter + ); + } + + (ParallelRowContract Row, bool IsInTrainingData)[] pretranslateRows = + [ + ( + new( + "MAT", + [ScriptureRef.Parse("MAT 1:1/0:s")], + [ScriptureRef.Parse("MAT 1:1/0:s")], + "Source Matthew section header", + "", + 1 + ), + false + ), + ( + new( + "MAT", + [ScriptureRef.Parse("MAT 1:1")], + [ScriptureRef.Parse("MAT 1:1")], + "Source Matthew 1:1", + "Target Matthew 1:1", + 1 + ), + true + ), + ( + new("JHN", [ScriptureRef.Parse("JHN 1:1")], [ScriptureRef.Parse("JHN 1:1")], "Source John 1:1", "", 1), + false + ), + ]; + + using MemoryStream memoryStream = new(); + using (Utf8JsonWriter pretranslateWriter = new(memoryStream)) + { + pretranslateWriter.WriteStartArray(); + foreach ((ParallelRowContract row, bool isInTrainingData) in pretranslateRows) + { + await stats.ProcessPretranslateRowAsync(row, isInTrainingData, "corpus1", pretranslateWriter); + } + pretranslateWriter.WriteEndArray(); + } + + memoryStream.Seek(0, SeekOrigin.Begin); + StreamReader pretranslationReader = new(memoryStream); + string pretranslationContent = await pretranslationReader.ReadToEndAsync(); + + Assert.That( + sourceSegments.SequenceEqual([ + "Source Matthew 1:1\n", + "Source Matthew 1:2\n", + "Source Matthew 2:1\n", + "Source Mark 1:1\n", + ]) + ); + Assert.That( + targetSegments.SequenceEqual(["Target Matthew 1:1\n", "\n", "Target Matthew 2:1\n", "Target Mark 1:1\n"]) + ); + Assert.That(sourceKeyTermSegments.SequenceEqual(["Source Isaac\n"])); + Assert.That(targetKeyTermSegments.SequenceEqual(["Target Isaac\n"])); + Assert.That(stats.TrainCount, Is.EqualTo(4)); + Assert.That(stats.TrainVerseCount, Has.Count.EqualTo(2)); + + Assert.That(stats.TrainVerseCount, Does.ContainKey("MAT")); + Assert.That(stats.TrainVerseCount["MAT"], Has.Count.EqualTo(2)); + Assert.That(stats.TrainVerseCount["MAT"], Does.ContainKey("1")); + Assert.That(stats.TrainVerseCount["MAT"]["1"], Is.EqualTo(1)); + Assert.That(stats.TrainVerseCount["MAT"], Does.ContainKey("2")); + Assert.That(stats.TrainVerseCount["MAT"]["2"], Is.EqualTo(1)); + + Assert.That(stats.TrainVerseCount, Does.ContainKey("MRK")); + Assert.That(stats.TrainVerseCount["MRK"], Has.Count.EqualTo(1)); + Assert.That(stats.TrainVerseCount["MRK"], Does.ContainKey("1")); + Assert.That(stats.TrainVerseCount["MRK"]["1"], Is.EqualTo(1)); + + Assert.That( + pretranslationContent, + Is.EqualTo( + "[" + + "{\"corpusId\":\"corpus1\",\"textId\":\"MAT\",\"sourceRefs\":[\"MAT 1:1/s\"],\"targetRefs\":[\"MAT 1:1/s\"],\"translation\":\"Source Matthew section header\"}," + + "{\"corpusId\":\"corpus1\",\"textId\":\"JHN\",\"sourceRefs\":[\"JHN 1:1\"],\"targetRefs\":[\"JHN 1:1\"],\"translation\":\"Source John 1:1\"}" + + "]" + ) + ); + Assert.That(stats.InferenceCount, Is.EqualTo(2)); + Assert.That(stats.InferenceVerseCount, Has.Count.EqualTo(1)); + Assert.That(stats.InferenceVerseCount, Does.ContainKey("JHN")); + Assert.That(stats.InferenceVerseCount["JHN"], Has.Count.EqualTo(1)); + Assert.That(stats.InferenceVerseCount["JHN"], Does.ContainKey("1")); + Assert.That(stats.InferenceVerseCount["JHN"]["1"], Is.EqualTo(1)); + } + + [Test] + public async Task ProcessWordAlignmentRowsAsync() + { + PreprocessStats stats = new(); + + var stream = Substitute.For(); + stream.CanWrite.Returns(true); + + var sourceTrainWriter = Substitute.For(stream); + List sourceSegments = []; + sourceTrainWriter.When(x => x.WriteAsync(Arg.Any())).Do(c => sourceSegments.Add(c.ArgAt(0))); + + var targetTrainWriter = Substitute.For(stream); + List targetSegments = []; + targetTrainWriter.When(x => x.WriteAsync(Arg.Any())).Do(c => targetSegments.Add(c.ArgAt(0))); + + var sourceKeyTermsTrainWriter = Substitute.For(stream); + List sourceKeyTermSegments = []; + sourceKeyTermsTrainWriter + .When(x => x.WriteAsync(Arg.Any())) + .Do(c => sourceKeyTermSegments.Add(c.ArgAt(0))); + + var targetKeyTermsTrainWriter = Substitute.For(stream); + List targetKeyTermSegments = []; + targetKeyTermsTrainWriter + .When(x => x.WriteAsync(Arg.Any())) + .Do(c => targetKeyTermSegments.Add(c.ArgAt(0))); + + (ParallelRowContract Row, TrainingDataType Type)[] trainingRows = + [ + ( + new( + "MAT", + [ScriptureRef.Parse("MAT 1:1")], + [ScriptureRef.Parse("MAT 1:1")], + "Source Matthew 1:1", + "Target Matthew 1:1", + 1 + ), + TrainingDataType.Text + ), + ( + new( + "MAT", + [ScriptureRef.Parse("MAT 1:2")], + [ScriptureRef.Parse("MAT 1:2")], + "Source Matthew 1:2", + "", + 1 + ), + TrainingDataType.Text + ), + ( + new( + "MAT", + [ScriptureRef.Parse("MAT 2:1")], + [ScriptureRef.Parse("MAT 2:1")], + "Source Matthew 2:1", + "Target Matthew 2:1", + 1 + ), + TrainingDataType.Text + ), + ( + new( + "MRK", + [ScriptureRef.Parse("MRK 1:1")], + [ScriptureRef.Parse("MRK 1:1")], + "Source Mark 1:1", + "Target Mark 1:1", + 1 + ), + TrainingDataType.Text + ), + ( + new( + "MRK", + ["MajorBiblicalTerms:Isaac"], + ["MajorBiblicalTerms:Isaac"], + "Source Isaac", + "Target Isaac", + 1 + ), + TrainingDataType.KeyTerm + ), + ]; + foreach ((ParallelRowContract row, TrainingDataType type) in trainingRows) + { + await stats.ProcessWordAlignmentTrainingRowAsync( + row, + type, + sourceTrainWriter, + targetTrainWriter, + sourceKeyTermsTrainWriter, + targetKeyTermsTrainWriter + ); + } + + (ParallelRowContract Row, bool IsInTrainingData)[] wordAlignRows = + [ + ( + new( + "MAT", + [ScriptureRef.Parse("MAT 1:1/0:s")], + [ScriptureRef.Parse("MAT 1:1/0:s")], + "Source Matthew section header", + "", + 1 + ), + false + ), + ( + new( + "MAT", + [ScriptureRef.Parse("MAT 1:1")], + [ScriptureRef.Parse("MAT 1:1")], + "Source Matthew 1:1", + "Target Matthew 1:1", + 1 + ), + true + ), + ( + new("JHN", [ScriptureRef.Parse("JHN 1:1")], [ScriptureRef.Parse("JHN 1:1")], "Source John 1:1", "", 1), + false + ), + ( + new( + "JHN", + [ScriptureRef.Parse("JHN 1:2")], + [ScriptureRef.Parse("JHN 1:2")], + "Source John 1:2", + "Target John 1:2", + 1 + ), + false + ), + ]; + + using MemoryStream memoryStream = new(); + using (Utf8JsonWriter wordAlignmentWriter = new(memoryStream)) + { + wordAlignmentWriter.WriteStartArray(); + foreach ((ParallelRowContract row, bool isInTrainingData) in wordAlignRows) + { + await stats.ProcessWordAlignRowAsync(row, isInTrainingData, "corpus1", wordAlignmentWriter); + } + wordAlignmentWriter.WriteEndArray(); + } + + memoryStream.Seek(0, SeekOrigin.Begin); + StreamReader wordAlignmentReader = new(memoryStream); + string wordAlignmentContent = await wordAlignmentReader.ReadToEndAsync(); + + Assert.That( + sourceSegments.SequenceEqual(["Source Matthew 1:1\n", "Source Matthew 2:1\n", "Source Mark 1:1\n"]) + ); + Assert.That( + targetSegments.SequenceEqual(["Target Matthew 1:1\n", "Target Matthew 2:1\n", "Target Mark 1:1\n"]) + ); + Assert.That(sourceKeyTermSegments.SequenceEqual(["Source Isaac\n"])); + Assert.That(targetKeyTermSegments.SequenceEqual(["Target Isaac\n"])); + Assert.That(stats.TrainCount, Is.EqualTo(4)); + Assert.That(stats.TrainVerseCount, Has.Count.EqualTo(2)); + + Assert.That(stats.TrainVerseCount, Does.ContainKey("MAT")); + Assert.That(stats.TrainVerseCount["MAT"], Has.Count.EqualTo(2)); + Assert.That(stats.TrainVerseCount["MAT"], Does.ContainKey("1")); + Assert.That(stats.TrainVerseCount["MAT"]["1"], Is.EqualTo(1)); + Assert.That(stats.TrainVerseCount["MAT"], Does.ContainKey("2")); + Assert.That(stats.TrainVerseCount["MAT"]["2"], Is.EqualTo(1)); + + Assert.That(stats.TrainVerseCount, Does.ContainKey("MRK")); + Assert.That(stats.TrainVerseCount["MRK"], Has.Count.EqualTo(1)); + Assert.That(stats.TrainVerseCount["MRK"], Does.ContainKey("1")); + Assert.That(stats.TrainVerseCount["MRK"]["1"], Is.EqualTo(1)); + + Assert.That( + wordAlignmentContent, + Is.EqualTo( + "[" + + "{\"corpusId\":\"corpus1\",\"textId\":\"JHN\",\"sourceRefs\":[\"JHN 1:2\"],\"targetRefs\":[\"JHN 1:2\"],\"source\":\"Source John 1:2\",\"target\":\"Target John 1:2\"}" + + "]" + ) + ); + Assert.That(stats.InferenceCount, Is.EqualTo(1)); + Assert.That(stats.InferenceVerseCount, Has.Count.EqualTo(1)); + Assert.That(stats.InferenceVerseCount, Does.ContainKey("JHN")); + Assert.That(stats.InferenceVerseCount["JHN"], Has.Count.EqualTo(1)); + Assert.That(stats.InferenceVerseCount["JHN"], Does.ContainKey("1")); + Assert.That(stats.InferenceVerseCount["JHN"]["1"], Is.EqualTo(1)); + } + private class TestEnvironment { public ISharedFileService SharedFileService { get; }