diff --git a/src/Machine/src/Serval.Machine.Shared/Models/BuildExecutionData.cs b/src/Machine/src/Serval.Machine.Shared/Models/BuildExecutionData.cs index 7f1e8ad3..2fc38e66 100644 --- a/src/Machine/src/Serval.Machine.Shared/Models/BuildExecutionData.cs +++ b/src/Machine/src/Serval.Machine.Shared/Models/BuildExecutionData.cs @@ -3,8 +3,11 @@ namespace Serval.Machine.Shared.Models; public record BuildExecutionData { public int? TrainCount { get; init; } - public int? PretranslateCount { get; init; } - public int? WordAlignCount { get; init; } + public int? InferenceCount { get; init; } + public bool? IsTrainFilteredByChapter { get; init; } + public bool? IsInferenceFilteredByChapter { get; init; } + public Dictionary>? TrainVerseCount { get; init; } + public Dictionary>? InferenceVerseCount { get; init; } public IReadOnlyList? Warnings { get; init; } public string? EngineSourceLanguageTag { get; init; } public string? EngineTargetLanguageTag { get; init; } diff --git a/src/Machine/src/Serval.Machine.Shared/Services/NmtPreprocessBuildJob.cs b/src/Machine/src/Serval.Machine.Shared/Services/NmtPreprocessBuildJob.cs index bdc176ff..dfd38985 100644 --- a/src/Machine/src/Serval.Machine.Shared/Services/NmtPreprocessBuildJob.cs +++ b/src/Machine/src/Serval.Machine.Shared/Services/NmtPreprocessBuildJob.cs @@ -52,8 +52,7 @@ await PlatformService.UpdateTargetQuoteConventionAsync( protected override async Task UpdateBuildExecutionData( string engineId, string buildId, - int trainCount, - int pretranslateCount, + PreprocessStats stats, string sourceLanguageTag, string targetLanguageTag, IReadOnlyList parallelCorpora, @@ -63,7 +62,7 @@ CancellationToken cancellationToken bool sourceLanguageHasNativeSupport = ResolveLanguageCode(sourceLanguageTag, out string resolvedSourceLanguage); bool targetLanguageHasNativeSupport = ResolveLanguageCode(targetLanguageTag, out string resolvedTargetLanguage); - if (trainCount == 0 && (!sourceLanguageHasNativeSupport || !targetLanguageHasNativeSupport)) + if (stats.TrainCount == 0 && (!sourceLanguageHasNativeSupport || !targetLanguageHasNativeSupport)) { throw new InvalidOperationException( $"At least one language code in build {buildId} is unknown to the base model, and the data specified for training was empty. Build canceled." @@ -71,8 +70,8 @@ CancellationToken cancellationToken } IReadOnlyList warnings = GetWarnings( - trainCount, - pretranslateCount, + stats.TrainCount, + stats.InferenceCount, sourceLanguageTag, targetLanguageTag, parallelCorpora @@ -92,8 +91,8 @@ CancellationToken cancellationToken { "Event", "BuildPreprocess" }, { "EngineId", engineId }, { "BuildId", buildId }, - { "NumTrainRows", trainCount }, - { "NumPretranslateRows", pretranslateCount }, + { "NumTrainRows", stats.TrainCount }, + { "NumPretranslateRows", stats.InferenceCount }, { "EngineSourceLanguageTag", sourceLanguageTag }, { "EngineTargetLanguageTag", targetLanguageTag }, { "SourceLanguageResolved", resolvedSourceLanguage }, @@ -103,8 +102,12 @@ CancellationToken cancellationToken Logger.LogInformation("{summary}", buildPreprocessSummary.ToJsonString()); var executionData = new BuildExecutionData() { - TrainCount = trainCount, - PretranslateCount = pretranslateCount, + TrainCount = stats.TrainCount, + InferenceCount = stats.InferenceCount, + TrainVerseCount = stats.TrainVerseCount, + InferenceVerseCount = stats.InferenceVerseCount, + IsTrainFilteredByChapter = stats.IsTrainFilteredByChapter, + IsInferenceFilteredByChapter = stats.IsInferenceFilteredByChapter, Warnings = warnings, EngineSourceLanguageTag = sourceLanguageTag, EngineTargetLanguageTag = targetLanguageTag, diff --git a/src/Machine/src/Serval.Machine.Shared/Services/PreprocessBuildJob.cs b/src/Machine/src/Serval.Machine.Shared/Services/PreprocessBuildJob.cs index dfad1cdf..d9942b5f 100644 --- a/src/Machine/src/Serval.Machine.Shared/Services/PreprocessBuildJob.cs +++ b/src/Machine/src/Serval.Machine.Shared/Services/PreprocessBuildJob.cs @@ -45,18 +45,12 @@ CancellationToken cancellationToken if (engine is null) throw new OperationCanceledException($"Engine {engineId} does not exist. Build canceled."); - (int trainCount, int inferenceCount) = await WriteDataFilesAsync( - buildId, - data, - buildOptions, - cancellationToken - ); + PreprocessStats stats = await WriteDataFilesAsync(buildId, data, buildOptions, cancellationToken); await UpdateBuildExecutionData( engineId, buildId, - trainCount, - inferenceCount, + stats, engine.SourceLanguage, engine.TargetLanguage, data, @@ -65,7 +59,7 @@ await UpdateBuildExecutionData( await UpdateTargetQuoteConventionAsync(engineId, buildId, data, cancellationToken); - if (inferenceCount == 0 && engine is TranslationEngine { IsModelPersisted: false }) + if (stats.InferenceCount == 0 && engine is TranslationEngine { IsModelPersisted: false }) { throw new InvalidOperationException( $"There was no data specified for inferencing in build {buildId}. Build canceled." @@ -90,8 +84,7 @@ await UpdateBuildExecutionData( protected abstract Task UpdateBuildExecutionData( string engineId, string buildId, - int trainCount, - int inferenceCount, + PreprocessStats stats, string sourceLanguageTag, string targetLanguageTag, IReadOnlyList parallelCorpora, @@ -105,7 +98,7 @@ protected virtual Task UpdateTargetQuoteConventionAsync( CancellationToken cancellationToken ) => Task.CompletedTask; - protected abstract Task<(int TrainCount, int InferenceCount)> WriteDataFilesAsync( + protected abstract Task WriteDataFilesAsync( string buildId, IReadOnlyList parallelCorpora, string? buildOptions, diff --git a/src/Machine/src/Serval.Machine.Shared/Services/PreprocessStats.cs b/src/Machine/src/Serval.Machine.Shared/Services/PreprocessStats.cs new file mode 100644 index 00000000..781d292b --- /dev/null +++ b/src/Machine/src/Serval.Machine.Shared/Services/PreprocessStats.cs @@ -0,0 +1,11 @@ +namespace Serval.Machine.Shared.Services; + +public record PreprocessStats +{ + public int TrainCount { get; set; } + public int InferenceCount { get; set; } + public bool IsTrainFilteredByChapter { get; set; } + public bool IsInferenceFilteredByChapter { get; set; } + public Dictionary> TrainVerseCount { get; set; } = []; + public Dictionary> InferenceVerseCount { get; set; } = []; +} diff --git a/src/Machine/src/Serval.Machine.Shared/Services/ServalTranslationPlatformService.cs b/src/Machine/src/Serval.Machine.Shared/Services/ServalTranslationPlatformService.cs index 1eb9ebb4..fb848fc7 100644 --- a/src/Machine/src/Serval.Machine.Shared/Services/ServalTranslationPlatformService.cs +++ b/src/Machine/src/Serval.Machine.Shared/Services/ServalTranslationPlatformService.cs @@ -101,7 +101,11 @@ public Task UpdateBuildExecutionDataAsync( new ExecutionDataContract { TrainCount = executionData.TrainCount, - PretranslateCount = executionData.PretranslateCount, + PretranslateCount = executionData.InferenceCount, + TrainVerseCount = executionData.TrainVerseCount, + PretranslateVerseCount = executionData.InferenceVerseCount, + IsTrainFilteredByChapter = executionData.IsTrainFilteredByChapter, + IsPretranslateFilteredByChapter = executionData.IsInferenceFilteredByChapter, Warnings = executionData.Warnings, EngineSourceLanguageTag = executionData.EngineSourceLanguageTag, EngineTargetLanguageTag = executionData.EngineTargetLanguageTag, diff --git a/src/Machine/src/Serval.Machine.Shared/Services/ServalWordAlignmentPlatformService.cs b/src/Machine/src/Serval.Machine.Shared/Services/ServalWordAlignmentPlatformService.cs index 29217d33..0cb97099 100644 --- a/src/Machine/src/Serval.Machine.Shared/Services/ServalWordAlignmentPlatformService.cs +++ b/src/Machine/src/Serval.Machine.Shared/Services/ServalWordAlignmentPlatformService.cs @@ -100,7 +100,11 @@ public Task UpdateBuildExecutionDataAsync( new ExecutionDataContract { TrainCount = executionData.TrainCount, - WordAlignCount = executionData.WordAlignCount, + WordAlignCount = executionData.InferenceCount, + IsTrainFilteredByChapter = executionData.IsTrainFilteredByChapter, + IsWordAlignmentFilteredByChapter = executionData.IsInferenceFilteredByChapter, + TrainVerseCount = executionData.TrainVerseCount, + WordAlignVerseCount = executionData.InferenceVerseCount, Warnings = executionData.Warnings, EngineSourceLanguageTag = executionData.EngineSourceLanguageTag, EngineTargetLanguageTag = executionData.EngineTargetLanguageTag, diff --git a/src/Machine/src/Serval.Machine.Shared/Services/TranslationPreprocessBuildJob.cs b/src/Machine/src/Serval.Machine.Shared/Services/TranslationPreprocessBuildJob.cs index 606c11a2..4cc8aaa3 100644 --- a/src/Machine/src/Serval.Machine.Shared/Services/TranslationPreprocessBuildJob.cs +++ b/src/Machine/src/Serval.Machine.Shared/Services/TranslationPreprocessBuildJob.cs @@ -1,3 +1,5 @@ +using SIL.Extensions; + namespace Serval.Machine.Shared.Services; public class TranslationPreprocessBuildJob( @@ -21,7 +23,7 @@ IOptionsMonitor options options ) { - protected override async Task<(int TrainCount, int InferenceCount)> WriteDataFilesAsync( + protected override async Task WriteDataFilesAsync( string buildId, IReadOnlyList parallelCorpora, string? buildOptions, @@ -51,66 +53,49 @@ await SharedFileService.OpenWriteAsync($"builds/{buildId}/train.key-terms.trg.tx cancellationToken ); await using Utf8JsonWriter pretranslateWriter = new(pretranslateStream, InferenceWriterOptions); + bool isTrainFilteredByChapter = parallelCorpora.Any(pc => + pc.SourceCorpora.Any(c => + c.TrainOnChapters is not null && c.TrainOnChapters.Values.Any(chapters => chapters.Count > 0) + ) + ); + bool isPretranslationFilteredByChapter = parallelCorpora.Any(pc => + pc.SourceCorpora.Any(c => + c.InferenceChapters is not null && c.InferenceChapters.Values.Any(chapters => chapters.Count > 0) + ) + ); + PreprocessStats preprocessStats = new() + { + IsTrainFilteredByChapter = isTrainFilteredByChapter, + IsInferenceFilteredByChapter = isPretranslationFilteredByChapter, + }; - int trainCount = 0; - int pretranslateCount = 0; pretranslateWriter.WriteStartArray(); await ParallelCorpusService.PreprocessAsync( parallelCorpora, async (row, trainingDataType) => - { - if (row.SourceSegment.Length > 0 || row.TargetSegment.Length > 0) - { - if (trainingDataType == TrainingDataType.KeyTerm) - { - await sourceKeyTermsTrainWriter.WriteAsync($"{row.SourceSegment}\n"); - await targetKeyTermsTrainWriter.WriteAsync($"{row.TargetSegment}\n"); - } - else - { - await sourceTrainWriter.WriteAsync($"{row.SourceSegment}\n"); - await targetTrainWriter.WriteAsync($"{row.TargetSegment}\n"); - } - } - if (row.SourceSegment.Length > 0 && row.TargetSegment.Length > 0) - trainCount++; - }, + await preprocessStats.ProcessTranslationTrainingRowAsync( + row, + trainingDataType, + sourceTrainWriter, + targetTrainWriter, + sourceKeyTermsTrainWriter, + targetKeyTermsTrainWriter + ), async (row, isInTrainingData, corpusId) => - { - if (row.SourceSegment.Length > 0 && !isInTrainingData) - { - pretranslateWriter.WriteStartObject(); - pretranslateWriter.WriteString("corpusId", corpusId); - pretranslateWriter.WriteString("textId", row.TextId); - pretranslateWriter.WriteStartArray("sourceRefs"); - foreach (object rowRef in row.SourceRefs) - pretranslateWriter.WriteStringValue(rowRef.ToString()); - pretranslateWriter.WriteEndArray(); - pretranslateWriter.WriteStartArray("targetRefs"); - foreach (object rowRef in row.TargetRefs) - pretranslateWriter.WriteStringValue(rowRef.ToString()); - pretranslateWriter.WriteEndArray(); - pretranslateWriter.WriteString("translation", row.SourceSegment); - pretranslateWriter.WriteEndObject(); - pretranslateCount++; - } - if (pretranslateWriter.BytesPending > 1024 * 1024) - await pretranslateWriter.FlushAsync(); - }, + await preprocessStats.ProcessPretranslateRowAsync(row, isInTrainingData, corpusId, pretranslateWriter), (bool?)buildOptionsObject?["use_key_terms"] ?? true, ignoreUsfmMarkers: ["rem", "r"] ); pretranslateWriter.WriteEndArray(); - return (trainCount, pretranslateCount); + return preprocessStats; } protected override async Task UpdateBuildExecutionData( string engineId, string buildId, - int trainCount, - int pretranslateCount, + PreprocessStats stats, string sourceLanguageTag, string targetLanguageTag, IReadOnlyList parallelCorpora, @@ -118,8 +103,8 @@ CancellationToken cancellationToken ) { IReadOnlyList warnings = GetWarnings( - trainCount, - pretranslateCount, + stats.TrainCount, + stats.InferenceCount, sourceLanguageTag, targetLanguageTag, parallelCorpora @@ -131,8 +116,8 @@ CancellationToken cancellationToken { "Event", "BuildPreprocess" }, { "EngineId", engineId }, { "BuildId", buildId }, - { "NumTrainRows", trainCount }, - { "NumPretranslateRows", pretranslateCount }, + { "NumTrainRows", stats.TrainCount }, + { "NumPretranslateRows", stats.InferenceCount }, { "EngineSourceLanguageTag", sourceLanguageTag }, { "EngineTargetLanguageTag", targetLanguageTag }, { "Warnings", new JsonArray(warnings.Select(w => JsonValue.Create(w)).ToArray()) }, @@ -140,8 +125,12 @@ CancellationToken cancellationToken Logger.LogInformation("{summary}", buildPreprocessSummary.ToJsonString()); var executionData = new BuildExecutionData() { - TrainCount = trainCount, - PretranslateCount = pretranslateCount, + TrainCount = stats.TrainCount, + InferenceCount = stats.InferenceCount, + IsTrainFilteredByChapter = stats.IsTrainFilteredByChapter, + IsInferenceFilteredByChapter = stats.IsInferenceFilteredByChapter, + TrainVerseCount = stats.TrainVerseCount, + InferenceVerseCount = stats.InferenceVerseCount, Warnings = warnings, EngineSourceLanguageTag = sourceLanguageTag, EngineTargetLanguageTag = targetLanguageTag, @@ -149,3 +138,100 @@ CancellationToken cancellationToken await PlatformService.UpdateBuildExecutionDataAsync(engineId, buildId, executionData, cancellationToken); } } + +public static partial class PreprocessStatsExtensions +{ + public static async Task ProcessTranslationTrainingRowAsync( + this PreprocessStats stats, + ParallelRowContract row, + TrainingDataType trainingDataType, + StreamWriter sourceTrainWriter, + StreamWriter targetTrainWriter, + StreamWriter sourceKeyTermsTrainWriter, + StreamWriter targetKeyTermsTrainWriter + ) + { + if (row.SourceSegment.Length > 0 || row.TargetSegment.Length > 0) + { + if (trainingDataType == TrainingDataType.KeyTerm) + { + await sourceKeyTermsTrainWriter.WriteAsync($"{row.SourceSegment}\n"); + await targetKeyTermsTrainWriter.WriteAsync($"{row.TargetSegment}\n"); + } + else + { + await sourceTrainWriter.WriteAsync($"{row.SourceSegment}\n"); + await targetTrainWriter.WriteAsync($"{row.TargetSegment}\n"); + } + } + if (row.SourceSegment.Length > 0 && row.TargetSegment.Length > 0) + { + stats.TrainCount++; + foreach (object? reference in row.SourceRefs) + { + if (reference is not null and ScriptureRef sr && sr.IsVerse) + { + stats.TrainVerseCount.UpdateValue( + sr.Book, + () => [], + chapters => + { + if (chapters.TryGetValue(sr.Chapter, out int count)) + chapters[sr.Chapter] = count + 1; + else + chapters[sr.Chapter] = 1; + return chapters; + } + ); + } + } + } + } + + public static async Task ProcessPretranslateRowAsync( + this PreprocessStats stats, + ParallelRowContract row, + bool isInTrainingData, + string corpusId, + Utf8JsonWriter pretranslateWriter + ) + { + if (row.SourceSegment.Length > 0 && !isInTrainingData) + { + pretranslateWriter.WriteStartObject(); + pretranslateWriter.WriteString("corpusId", corpusId); + pretranslateWriter.WriteString("textId", row.TextId); + pretranslateWriter.WriteStartArray("sourceRefs"); + foreach (object rowRef in row.SourceRefs) + pretranslateWriter.WriteStringValue(rowRef.ToString()); + pretranslateWriter.WriteEndArray(); + pretranslateWriter.WriteStartArray("targetRefs"); + foreach (object rowRef in row.TargetRefs) + pretranslateWriter.WriteStringValue(rowRef.ToString()); + pretranslateWriter.WriteEndArray(); + pretranslateWriter.WriteString("translation", row.SourceSegment); + pretranslateWriter.WriteEndObject(); + stats.InferenceCount++; + foreach (object? reference in row.SourceRefs) + { + if (reference is not null and ScriptureRef sr && sr.IsVerse) + { + stats.InferenceVerseCount.UpdateValue( + sr.Book, + () => [], + chapters => + { + if (chapters.TryGetValue(sr.Chapter, out int count)) + chapters[sr.Chapter] = count + 1; + else + chapters[sr.Chapter] = 1; + return chapters; + } + ); + } + } + } + if (pretranslateWriter.BytesPending > 1024 * 1024) + await pretranslateWriter.FlushAsync(); + } +} diff --git a/src/Machine/src/Serval.Machine.Shared/Services/WordAlignmentPreprocessBuildJob.cs b/src/Machine/src/Serval.Machine.Shared/Services/WordAlignmentPreprocessBuildJob.cs index ae3dc044..d29804c0 100644 --- a/src/Machine/src/Serval.Machine.Shared/Services/WordAlignmentPreprocessBuildJob.cs +++ b/src/Machine/src/Serval.Machine.Shared/Services/WordAlignmentPreprocessBuildJob.cs @@ -1,4 +1,6 @@ -namespace Serval.Machine.Shared.Services; +using SIL.Extensions; + +namespace Serval.Machine.Shared.Services; public class WordAlignmentPreprocessBuildJob( [FromKeyedServices(EngineGroup.WordAlignment)] IPlatformService platformService, @@ -21,7 +23,7 @@ IOptionsMonitor options options ) { - protected override async Task<(int TrainCount, int InferenceCount)> WriteDataFilesAsync( + protected override async Task WriteDataFilesAsync( string buildId, IReadOnlyList parallelCorpora, string? buildOptions, @@ -51,66 +53,49 @@ await SharedFileService.OpenWriteAsync($"builds/{buildId}/train.key-terms.trg.tx cancellationToken ); await using Utf8JsonWriter wordAlignmentWriter = new(wordAlignmentStream, InferenceWriterOptions); + bool isTrainFilteredByChapter = parallelCorpora.Any(pc => + pc.SourceCorpora.Any(c => + c.TrainOnChapters is not null && c.TrainOnChapters.Values.Any(chapters => chapters.Count > 0) + ) + ); + bool isWordAlignmentFilteredByChapter = parallelCorpora.Any(pc => + pc.SourceCorpora.Any(c => + c.InferenceChapters is not null && c.InferenceChapters.Values.Any(chapters => chapters.Count > 0) + ) + ); + + PreprocessStats preprocessStats = new() + { + IsTrainFilteredByChapter = isTrainFilteredByChapter, + IsInferenceFilteredByChapter = isWordAlignmentFilteredByChapter, + }; - int trainCount = 0; - int inferenceCount = 0; wordAlignmentWriter.WriteStartArray(); await ParallelCorpusService.PreprocessAsync( parallelCorpora, async (row, trainingDataType) => - { - if (row.SourceSegment.Length > 0 && row.TargetSegment.Length > 0) - { - if (trainingDataType == TrainingDataType.KeyTerm) - { - await sourceKeyTermsTrainWriter.WriteAsync($"{row.SourceSegment}\n"); - await targetKeyTermsTrainWriter.WriteAsync($"{row.TargetSegment}\n"); - } - else - { - await sourceTrainWriter.WriteAsync($"{row.SourceSegment}\n"); - await targetTrainWriter.WriteAsync($"{row.TargetSegment}\n"); - } - - trainCount++; - } - }, + await preprocessStats.ProcessWordAlignmentTrainingRowAsync( + row, + trainingDataType, + sourceTrainWriter, + targetTrainWriter, + sourceKeyTermsTrainWriter, + targetKeyTermsTrainWriter + ), async (row, isInTrainingData, corpusId) => - { - if (row.SourceSegment.Length > 0 && row.TargetSegment.Length > 0 && !isInTrainingData) - { - wordAlignmentWriter.WriteStartObject(); - wordAlignmentWriter.WriteString("corpusId", corpusId); - wordAlignmentWriter.WriteString("textId", row.TextId); - wordAlignmentWriter.WriteStartArray("sourceRefs"); - foreach (object rowRef in row.SourceRefs) - wordAlignmentWriter.WriteStringValue(rowRef.ToString()); - wordAlignmentWriter.WriteEndArray(); - wordAlignmentWriter.WriteStartArray("targetRefs"); - foreach (object rowRef in row.TargetRefs) - wordAlignmentWriter.WriteStringValue(rowRef.ToString()); - wordAlignmentWriter.WriteEndArray(); - wordAlignmentWriter.WriteString("source", row.SourceSegment); - wordAlignmentWriter.WriteString("target", row.TargetSegment); - wordAlignmentWriter.WriteEndObject(); - inferenceCount++; - } - if (wordAlignmentWriter.BytesPending > 1024 * 1024) - await wordAlignmentWriter.FlushAsync(); - }, + await preprocessStats.ProcessWordAlignRowAsync(row, isInTrainingData, corpusId, wordAlignmentWriter), (bool?)buildOptionsObject?["use_key_terms"] ?? true ); wordAlignmentWriter.WriteEndArray(); - return (trainCount, inferenceCount); + return preprocessStats; } protected override async Task UpdateBuildExecutionData( string engineId, string buildId, - int trainCount, - int wordAlignCount, + PreprocessStats stats, string sourceLanguageTag, string targetLanguageTag, IReadOnlyList parallelCorpora, @@ -118,8 +103,8 @@ CancellationToken cancellationToken ) { IReadOnlyList warnings = GetWarnings( - trainCount, - wordAlignCount, + stats.TrainCount, + stats.InferenceCount, sourceLanguageTag, targetLanguageTag, parallelCorpora @@ -131,8 +116,8 @@ CancellationToken cancellationToken { "Event", "BuildPreprocess" }, { "EngineId", engineId }, { "BuildId", buildId }, - { "NumTrainRows", trainCount }, - { "NumWordAlignRows", wordAlignCount }, + { "NumTrainRows", stats.TrainCount }, + { "NumWordAlignRows", stats.InferenceCount }, { "EngineSourceLanguageTag", sourceLanguageTag }, { "EngineTargetLanguageTag", targetLanguageTag }, { "Warnings", new JsonArray(warnings.Select(w => JsonValue.Create(w)).ToArray()) }, @@ -140,8 +125,12 @@ CancellationToken cancellationToken Logger.LogInformation("{summary}", buildPreprocessSummary.ToJsonString()); var executionData = new BuildExecutionData() { - TrainCount = trainCount, - WordAlignCount = wordAlignCount, + TrainCount = stats.TrainCount, + InferenceCount = stats.InferenceCount, + TrainVerseCount = stats.TrainVerseCount, + InferenceVerseCount = stats.InferenceVerseCount, + IsInferenceFilteredByChapter = stats.IsInferenceFilteredByChapter, + IsTrainFilteredByChapter = stats.IsTrainFilteredByChapter, Warnings = warnings, EngineSourceLanguageTag = sourceLanguageTag, EngineTargetLanguageTag = targetLanguageTag, @@ -160,3 +149,99 @@ CancellationToken cancellationToken return Task.CompletedTask; } } + +public static partial class PreprocessStatsExtensions +{ + public static async Task ProcessWordAlignmentTrainingRowAsync( + this PreprocessStats stats, + ParallelRowContract row, + TrainingDataType trainingDataType, + StreamWriter sourceTrainWriter, + StreamWriter targetTrainWriter, + StreamWriter sourceKeyTermsTrainWriter, + StreamWriter targetKeyTermsTrainWriter + ) + { + if (row.SourceSegment.Length > 0 && row.TargetSegment.Length > 0) + { + if (trainingDataType == TrainingDataType.KeyTerm) + { + await sourceKeyTermsTrainWriter.WriteAsync($"{row.SourceSegment}\n"); + await targetKeyTermsTrainWriter.WriteAsync($"{row.TargetSegment}\n"); + } + else + { + await sourceTrainWriter.WriteAsync($"{row.SourceSegment}\n"); + await targetTrainWriter.WriteAsync($"{row.TargetSegment}\n"); + } + + stats.TrainCount++; + foreach (object? reference in row.SourceRefs) + { + if (reference is not null and ScriptureRef sr && sr.IsVerse) + { + stats.TrainVerseCount.UpdateValue( + sr.Book, + () => [], + chapters => + { + if (chapters.TryGetValue(sr.Chapter, out int count)) + chapters[sr.Chapter] = count + 1; + else + chapters[sr.Chapter] = 1; + return chapters; + } + ); + } + } + } + } + + public static async Task ProcessWordAlignRowAsync( + this PreprocessStats stats, + ParallelRowContract row, + bool isInTrainingData, + string corpusId, + Utf8JsonWriter wordAlignmentWriter + ) + { + if (row.SourceSegment.Length > 0 && row.TargetSegment.Length > 0 && !isInTrainingData) + { + wordAlignmentWriter.WriteStartObject(); + wordAlignmentWriter.WriteString("corpusId", corpusId); + wordAlignmentWriter.WriteString("textId", row.TextId); + wordAlignmentWriter.WriteStartArray("sourceRefs"); + foreach (object rowRef in row.SourceRefs) + wordAlignmentWriter.WriteStringValue(rowRef.ToString()); + wordAlignmentWriter.WriteEndArray(); + wordAlignmentWriter.WriteStartArray("targetRefs"); + foreach (object rowRef in row.TargetRefs) + wordAlignmentWriter.WriteStringValue(rowRef.ToString()); + wordAlignmentWriter.WriteEndArray(); + wordAlignmentWriter.WriteString("source", row.SourceSegment); + wordAlignmentWriter.WriteString("target", row.TargetSegment); + wordAlignmentWriter.WriteEndObject(); + stats.InferenceCount++; + foreach (object? reference in row.SourceRefs) + { + if (reference is not null and ScriptureRef sr && sr.IsVerse) + { + stats.InferenceVerseCount.UpdateValue( + sr.Book, + () => [], + chapters => + { + if (chapters.TryGetValue(sr.Chapter, out int count)) + chapters[sr.Chapter] = count + 1; + else + chapters[sr.Chapter] = 1; + return chapters; + } + ); + } + } + } + if (wordAlignmentWriter.BytesPending > 1024 * 1024) + await wordAlignmentWriter.FlushAsync(); + } +} diff --git a/src/Machine/test/Serval.Machine.Shared.Tests/Services/PreprocessBuildJobTests.cs b/src/Machine/test/Serval.Machine.Shared.Tests/Services/PreprocessBuildJobTests.cs index c0972def..1350bf2b 100644 --- a/src/Machine/test/Serval.Machine.Shared.Tests/Services/PreprocessBuildJobTests.cs +++ b/src/Machine/test/Serval.Machine.Shared.Tests/Services/PreprocessBuildJobTests.cs @@ -95,6 +95,387 @@ public async Task RunAsync_UnknownLanguageTagsNoDataSmtTransfer() await env.RunBuildJobAsync(corpus1, engineId: "engine3", engineType: EngineType.SmtTransfer); } + [Test] + public async Task ProcessTranslationRowsAsync() + { + PreprocessStats stats = new(); + + var stream = Substitute.For(); + stream.CanWrite.Returns(true); + + var sourceTrainWriter = Substitute.For(stream); + List sourceSegments = []; + sourceTrainWriter.When(x => x.WriteAsync(Arg.Any())).Do(c => sourceSegments.Add(c.ArgAt(0))); + + var targetTrainWriter = Substitute.For(stream); + List targetSegments = []; + targetTrainWriter.When(x => x.WriteAsync(Arg.Any())).Do(c => targetSegments.Add(c.ArgAt(0))); + + var sourceKeyTermsTrainWriter = Substitute.For(stream); + List sourceKeyTermSegments = []; + sourceKeyTermsTrainWriter + .When(x => x.WriteAsync(Arg.Any())) + .Do(c => sourceKeyTermSegments.Add(c.ArgAt(0))); + + var targetKeyTermsTrainWriter = Substitute.For(stream); + List targetKeyTermSegments = []; + targetKeyTermsTrainWriter + .When(x => x.WriteAsync(Arg.Any())) + .Do(c => targetKeyTermSegments.Add(c.ArgAt(0))); + + (ParallelRowContract Row, TrainingDataType Type)[] trainingRows = + [ + ( + new( + "MAT", + [ScriptureRef.Parse("MAT 1:1")], + [ScriptureRef.Parse("MAT 1:1")], + "Source Matthew 1:1", + "Target Matthew 1:1", + 1 + ), + TrainingDataType.Text + ), + ( + new( + "MAT", + [ScriptureRef.Parse("MAT 1:2")], + [ScriptureRef.Parse("MAT 1:2")], + "Source Matthew 1:2", + "", + 1 + ), + TrainingDataType.Text + ), + ( + new( + "MAT", + [ScriptureRef.Parse("MAT 2:1")], + [ScriptureRef.Parse("MAT 2:1")], + "Source Matthew 2:1", + "Target Matthew 2:1", + 1 + ), + TrainingDataType.Text + ), + ( + new( + "MRK", + [ScriptureRef.Parse("MRK 1:1")], + [ScriptureRef.Parse("MRK 1:1")], + "Source Mark 1:1", + "Target Mark 1:1", + 1 + ), + TrainingDataType.Text + ), + ( + new( + "MRK", + ["MajorBiblicalTerms:Isaac"], + ["MajorBiblicalTerms:Isaac"], + "Source Isaac", + "Target Isaac", + 1 + ), + TrainingDataType.KeyTerm + ), + ]; + foreach ((ParallelRowContract row, TrainingDataType type) in trainingRows) + { + await stats.ProcessTranslationTrainingRowAsync( + row, + type, + sourceTrainWriter, + targetTrainWriter, + sourceKeyTermsTrainWriter, + targetKeyTermsTrainWriter + ); + } + + (ParallelRowContract Row, bool IsInTrainingData)[] pretranslateRows = + [ + ( + new( + "MAT", + [ScriptureRef.Parse("MAT 1:1/0:s")], + [ScriptureRef.Parse("MAT 1:1/0:s")], + "Source Matthew section header", + "", + 1 + ), + false + ), + ( + new( + "MAT", + [ScriptureRef.Parse("MAT 1:1")], + [ScriptureRef.Parse("MAT 1:1")], + "Source Matthew 1:1", + "Target Matthew 1:1", + 1 + ), + true + ), + ( + new("JHN", [ScriptureRef.Parse("JHN 1:1")], [ScriptureRef.Parse("JHN 1:1")], "Source John 1:1", "", 1), + false + ), + ]; + + using MemoryStream memoryStream = new(); + using (Utf8JsonWriter pretranslateWriter = new(memoryStream)) + { + pretranslateWriter.WriteStartArray(); + foreach ((ParallelRowContract row, bool isInTrainingData) in pretranslateRows) + { + await stats.ProcessPretranslateRowAsync(row, isInTrainingData, "corpus1", pretranslateWriter); + } + pretranslateWriter.WriteEndArray(); + } + + memoryStream.Seek(0, SeekOrigin.Begin); + StreamReader pretranslationReader = new(memoryStream); + string pretranslationContent = await pretranslationReader.ReadToEndAsync(); + + Assert.That( + sourceSegments.SequenceEqual([ + "Source Matthew 1:1\n", + "Source Matthew 1:2\n", + "Source Matthew 2:1\n", + "Source Mark 1:1\n", + ]) + ); + Assert.That( + targetSegments.SequenceEqual(["Target Matthew 1:1\n", "\n", "Target Matthew 2:1\n", "Target Mark 1:1\n"]) + ); + Assert.That(sourceKeyTermSegments.SequenceEqual(["Source Isaac\n"])); + Assert.That(targetKeyTermSegments.SequenceEqual(["Target Isaac\n"])); + Assert.That(stats.TrainCount, Is.EqualTo(4)); + Assert.That(stats.TrainVerseCount, Has.Count.EqualTo(2)); + + Assert.That(stats.TrainVerseCount, Does.ContainKey("MAT")); + Assert.That(stats.TrainVerseCount["MAT"], Has.Count.EqualTo(2)); + Assert.That(stats.TrainVerseCount["MAT"], Does.ContainKey("1")); + Assert.That(stats.TrainVerseCount["MAT"]["1"], Is.EqualTo(1)); + Assert.That(stats.TrainVerseCount["MAT"], Does.ContainKey("2")); + Assert.That(stats.TrainVerseCount["MAT"]["2"], Is.EqualTo(1)); + + Assert.That(stats.TrainVerseCount, Does.ContainKey("MRK")); + Assert.That(stats.TrainVerseCount["MRK"], Has.Count.EqualTo(1)); + Assert.That(stats.TrainVerseCount["MRK"], Does.ContainKey("1")); + Assert.That(stats.TrainVerseCount["MRK"]["1"], Is.EqualTo(1)); + + Assert.That( + pretranslationContent, + Is.EqualTo( + "[" + + "{\"corpusId\":\"corpus1\",\"textId\":\"MAT\",\"sourceRefs\":[\"MAT 1:1/s\"],\"targetRefs\":[\"MAT 1:1/s\"],\"translation\":\"Source Matthew section header\"}," + + "{\"corpusId\":\"corpus1\",\"textId\":\"JHN\",\"sourceRefs\":[\"JHN 1:1\"],\"targetRefs\":[\"JHN 1:1\"],\"translation\":\"Source John 1:1\"}" + + "]" + ) + ); + Assert.That(stats.InferenceCount, Is.EqualTo(2)); + Assert.That(stats.InferenceVerseCount, Has.Count.EqualTo(1)); + Assert.That(stats.InferenceVerseCount, Does.ContainKey("JHN")); + Assert.That(stats.InferenceVerseCount["JHN"], Has.Count.EqualTo(1)); + Assert.That(stats.InferenceVerseCount["JHN"], Does.ContainKey("1")); + Assert.That(stats.InferenceVerseCount["JHN"]["1"], Is.EqualTo(1)); + } + + [Test] + public async Task ProcessWordAlignmentRowsAsync() + { + PreprocessStats stats = new(); + + var stream = Substitute.For(); + stream.CanWrite.Returns(true); + + var sourceTrainWriter = Substitute.For(stream); + List sourceSegments = []; + sourceTrainWriter.When(x => x.WriteAsync(Arg.Any())).Do(c => sourceSegments.Add(c.ArgAt(0))); + + var targetTrainWriter = Substitute.For(stream); + List targetSegments = []; + targetTrainWriter.When(x => x.WriteAsync(Arg.Any())).Do(c => targetSegments.Add(c.ArgAt(0))); + + var sourceKeyTermsTrainWriter = Substitute.For(stream); + List sourceKeyTermSegments = []; + sourceKeyTermsTrainWriter + .When(x => x.WriteAsync(Arg.Any())) + .Do(c => sourceKeyTermSegments.Add(c.ArgAt(0))); + + var targetKeyTermsTrainWriter = Substitute.For(stream); + List targetKeyTermSegments = []; + targetKeyTermsTrainWriter + .When(x => x.WriteAsync(Arg.Any())) + .Do(c => targetKeyTermSegments.Add(c.ArgAt(0))); + + (ParallelRowContract Row, TrainingDataType Type)[] trainingRows = + [ + ( + new( + "MAT", + [ScriptureRef.Parse("MAT 1:1")], + [ScriptureRef.Parse("MAT 1:1")], + "Source Matthew 1:1", + "Target Matthew 1:1", + 1 + ), + TrainingDataType.Text + ), + ( + new( + "MAT", + [ScriptureRef.Parse("MAT 1:2")], + [ScriptureRef.Parse("MAT 1:2")], + "Source Matthew 1:2", + "", + 1 + ), + TrainingDataType.Text + ), + ( + new( + "MAT", + [ScriptureRef.Parse("MAT 2:1")], + [ScriptureRef.Parse("MAT 2:1")], + "Source Matthew 2:1", + "Target Matthew 2:1", + 1 + ), + TrainingDataType.Text + ), + ( + new( + "MRK", + [ScriptureRef.Parse("MRK 1:1")], + [ScriptureRef.Parse("MRK 1:1")], + "Source Mark 1:1", + "Target Mark 1:1", + 1 + ), + TrainingDataType.Text + ), + ( + new( + "MRK", + ["MajorBiblicalTerms:Isaac"], + ["MajorBiblicalTerms:Isaac"], + "Source Isaac", + "Target Isaac", + 1 + ), + TrainingDataType.KeyTerm + ), + ]; + foreach ((ParallelRowContract row, TrainingDataType type) in trainingRows) + { + await stats.ProcessWordAlignmentTrainingRowAsync( + row, + type, + sourceTrainWriter, + targetTrainWriter, + sourceKeyTermsTrainWriter, + targetKeyTermsTrainWriter + ); + } + + (ParallelRowContract Row, bool IsInTrainingData)[] wordAlignRows = + [ + ( + new( + "MAT", + [ScriptureRef.Parse("MAT 1:1/0:s")], + [ScriptureRef.Parse("MAT 1:1/0:s")], + "Source Matthew section header", + "", + 1 + ), + false + ), + ( + new( + "MAT", + [ScriptureRef.Parse("MAT 1:1")], + [ScriptureRef.Parse("MAT 1:1")], + "Source Matthew 1:1", + "Target Matthew 1:1", + 1 + ), + true + ), + ( + new("JHN", [ScriptureRef.Parse("JHN 1:1")], [ScriptureRef.Parse("JHN 1:1")], "Source John 1:1", "", 1), + false + ), + ( + new( + "JHN", + [ScriptureRef.Parse("JHN 1:2")], + [ScriptureRef.Parse("JHN 1:2")], + "Source John 1:2", + "Target John 1:2", + 1 + ), + false + ), + ]; + + using MemoryStream memoryStream = new(); + using (Utf8JsonWriter wordAlignmentWriter = new(memoryStream)) + { + wordAlignmentWriter.WriteStartArray(); + foreach ((ParallelRowContract row, bool isInTrainingData) in wordAlignRows) + { + await stats.ProcessWordAlignRowAsync(row, isInTrainingData, "corpus1", wordAlignmentWriter); + } + wordAlignmentWriter.WriteEndArray(); + } + + memoryStream.Seek(0, SeekOrigin.Begin); + StreamReader wordAlignmentReader = new(memoryStream); + string wordAlignmentContent = await wordAlignmentReader.ReadToEndAsync(); + + Assert.That( + sourceSegments.SequenceEqual(["Source Matthew 1:1\n", "Source Matthew 2:1\n", "Source Mark 1:1\n"]) + ); + Assert.That( + targetSegments.SequenceEqual(["Target Matthew 1:1\n", "Target Matthew 2:1\n", "Target Mark 1:1\n"]) + ); + Assert.That(sourceKeyTermSegments.SequenceEqual(["Source Isaac\n"])); + Assert.That(targetKeyTermSegments.SequenceEqual(["Target Isaac\n"])); + Assert.That(stats.TrainCount, Is.EqualTo(4)); + Assert.That(stats.TrainVerseCount, Has.Count.EqualTo(2)); + + Assert.That(stats.TrainVerseCount, Does.ContainKey("MAT")); + Assert.That(stats.TrainVerseCount["MAT"], Has.Count.EqualTo(2)); + Assert.That(stats.TrainVerseCount["MAT"], Does.ContainKey("1")); + Assert.That(stats.TrainVerseCount["MAT"]["1"], Is.EqualTo(1)); + Assert.That(stats.TrainVerseCount["MAT"], Does.ContainKey("2")); + Assert.That(stats.TrainVerseCount["MAT"]["2"], Is.EqualTo(1)); + + Assert.That(stats.TrainVerseCount, Does.ContainKey("MRK")); + Assert.That(stats.TrainVerseCount["MRK"], Has.Count.EqualTo(1)); + Assert.That(stats.TrainVerseCount["MRK"], Does.ContainKey("1")); + Assert.That(stats.TrainVerseCount["MRK"]["1"], Is.EqualTo(1)); + + Assert.That( + wordAlignmentContent, + Is.EqualTo( + "[" + + "{\"corpusId\":\"corpus1\",\"textId\":\"JHN\",\"sourceRefs\":[\"JHN 1:2\"],\"targetRefs\":[\"JHN 1:2\"],\"source\":\"Source John 1:2\",\"target\":\"Target John 1:2\"}" + + "]" + ) + ); + Assert.That(stats.InferenceCount, Is.EqualTo(1)); + Assert.That(stats.InferenceVerseCount, Has.Count.EqualTo(1)); + Assert.That(stats.InferenceVerseCount, Does.ContainKey("JHN")); + Assert.That(stats.InferenceVerseCount["JHN"], Has.Count.EqualTo(1)); + Assert.That(stats.InferenceVerseCount["JHN"], Does.ContainKey("1")); + Assert.That(stats.InferenceVerseCount["JHN"]["1"], Is.EqualTo(1)); + } + private class TestEnvironment { public ISharedFileService SharedFileService { get; } diff --git a/src/Serval/src/Serval.Client/Client.g.cs b/src/Serval/src/Serval.Client/Client.g.cs index 1b173fbd..2fe10c2a 100644 --- a/src/Serval/src/Serval.Client/Client.g.cs +++ b/src/Serval/src/Serval.Client/Client.g.cs @@ -11505,6 +11505,18 @@ public partial class ExecutionData [Newtonsoft.Json.JsonProperty("pretranslateCount", Required = Newtonsoft.Json.Required.Always)] public int PretranslateCount { get; set; } = default!; + [Newtonsoft.Json.JsonProperty("isTrainFilteredByChapter", Required = Newtonsoft.Json.Required.Always)] + public bool IsTrainFilteredByChapter { get; set; } = default!; + + [Newtonsoft.Json.JsonProperty("isPretranslateFilteredByChapter", Required = Newtonsoft.Json.Required.Always)] + public bool IsPretranslateFilteredByChapter { get; set; } = default!; + + [Newtonsoft.Json.JsonProperty("trainVerseCount", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + public System.Collections.Generic.IDictionary>? TrainVerseCount { get; set; } = default!; + + [Newtonsoft.Json.JsonProperty("pretranslateVerseCount", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + public System.Collections.Generic.IDictionary>? PretranslateVerseCount { get; set; } = default!; + [Newtonsoft.Json.JsonProperty("warnings", Required = Newtonsoft.Json.Required.Always)] [System.ComponentModel.DataAnnotations.Required] public System.Collections.Generic.IList Warnings { get; set; } = new System.Collections.ObjectModel.Collection(); @@ -12312,6 +12324,18 @@ public partial class WordAlignmentExecutionData [Newtonsoft.Json.JsonProperty("wordAlignCount", Required = Newtonsoft.Json.Required.Always)] public int WordAlignCount { get; set; } = default!; + [Newtonsoft.Json.JsonProperty("isTrainFilteredByChapter", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + public bool? IsTrainFilteredByChapter { get; set; } = default!; + + [Newtonsoft.Json.JsonProperty("isWordAlignFilteredByChapter", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + public bool? IsWordAlignFilteredByChapter { get; set; } = default!; + + [Newtonsoft.Json.JsonProperty("trainVerseCount", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + public System.Collections.Generic.IDictionary>? TrainVerseCount { get; set; } = default!; + + [Newtonsoft.Json.JsonProperty("wordAlignVerseCount", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + public System.Collections.Generic.IDictionary>? WordAlignVerseCount { get; set; } = default!; + [Newtonsoft.Json.JsonProperty("warnings", Required = Newtonsoft.Json.Required.Always)] [System.ComponentModel.DataAnnotations.Required] public System.Collections.Generic.IList Warnings { get; set; } = new System.Collections.ObjectModel.Collection(); diff --git a/src/Serval/src/Serval.Translation.Contracts/ExecutionDataContract.cs b/src/Serval/src/Serval.Translation.Contracts/ExecutionDataContract.cs index 26c31b53..b256eec9 100644 --- a/src/Serval/src/Serval.Translation.Contracts/ExecutionDataContract.cs +++ b/src/Serval/src/Serval.Translation.Contracts/ExecutionDataContract.cs @@ -4,6 +4,10 @@ public record ExecutionDataContract { public int? TrainCount { get; init; } public int? PretranslateCount { get; init; } + public bool? IsTrainFilteredByChapter { get; init; } + public bool? IsPretranslateFilteredByChapter { get; init; } + public Dictionary>? TrainVerseCount { get; init; } + public Dictionary>? PretranslateVerseCount { get; init; } public IReadOnlyList? Warnings { get; init; } public string? EngineSourceLanguageTag { get; init; } public string? EngineTargetLanguageTag { get; init; } diff --git a/src/Serval/src/Serval.Translation/Dtos/ExecutionDataDto.cs b/src/Serval/src/Serval.Translation/Dtos/ExecutionDataDto.cs index ef48a34a..da01c29d 100644 --- a/src/Serval/src/Serval.Translation/Dtos/ExecutionDataDto.cs +++ b/src/Serval/src/Serval.Translation/Dtos/ExecutionDataDto.cs @@ -4,6 +4,10 @@ public record ExecutionDataDto { public int TrainCount { get; init; } public int PretranslateCount { get; init; } + public bool IsTrainFilteredByChapter { get; init; } + public bool IsPretranslateFilteredByChapter { get; init; } + public Dictionary>? TrainVerseCount { get; init; } + public Dictionary>? PretranslateVerseCount { get; init; } public IReadOnlyList Warnings { get; init; } = []; public string? EngineSourceLanguageTag { get; init; } public string? EngineTargetLanguageTag { get; init; } diff --git a/src/Serval/src/Serval.Translation/Models/ExecutionData.cs b/src/Serval/src/Serval.Translation/Models/ExecutionData.cs index 79c953e7..646b53e7 100644 --- a/src/Serval/src/Serval.Translation/Models/ExecutionData.cs +++ b/src/Serval/src/Serval.Translation/Models/ExecutionData.cs @@ -4,6 +4,10 @@ public record ExecutionData { public int? TrainCount { get; init; } public int? PretranslateCount { get; init; } + public bool? IsTrainFilteredByChapter { get; init; } + public bool? IsPretranslateFilteredByChapter { get; init; } + public Dictionary>? TrainVerseCount { get; init; } + public Dictionary>? PretranslateVerseCount { get; init; } public IReadOnlyList? Warnings { get; init; } public string? EngineSourceLanguageTag { get; init; } public string? EngineTargetLanguageTag { get; init; } diff --git a/src/Serval/src/Serval.Translation/Services/DtoMapper.cs b/src/Serval/src/Serval.Translation/Services/DtoMapper.cs index 8dd3e08f..ad09d355 100644 --- a/src/Serval/src/Serval.Translation/Services/DtoMapper.cs +++ b/src/Serval/src/Serval.Translation/Services/DtoMapper.cs @@ -231,6 +231,10 @@ private static ExecutionDataDto Map(ExecutionData source) => { TrainCount = source.TrainCount ?? 0, PretranslateCount = source.PretranslateCount ?? 0, + PretranslateVerseCount = source.PretranslateVerseCount, + TrainVerseCount = source.TrainVerseCount, + IsPretranslateFilteredByChapter = source.IsPretranslateFilteredByChapter ?? false, + IsTrainFilteredByChapter = source.IsTrainFilteredByChapter ?? false, Warnings = source.Warnings ?? [], EngineSourceLanguageTag = source.EngineSourceLanguageTag, EngineTargetLanguageTag = source.EngineTargetLanguageTag, diff --git a/src/Serval/src/Serval.Translation/Services/PlatformService.cs b/src/Serval/src/Serval.Translation/Services/PlatformService.cs index 1616bd6b..9b88daab 100644 --- a/src/Serval/src/Serval.Translation/Services/PlatformService.cs +++ b/src/Serval/src/Serval.Translation/Services/PlatformService.cs @@ -311,6 +311,10 @@ await _builds.UpdateAsync( { TrainCount = executionData.TrainCount, PretranslateCount = executionData.PretranslateCount, + TrainVerseCount = executionData.TrainVerseCount, + PretranslateVerseCount = executionData.PretranslateVerseCount, + IsTrainFilteredByChapter = executionData.IsTrainFilteredByChapter, + IsPretranslateFilteredByChapter = executionData.IsPretranslateFilteredByChapter, Warnings = executionData.Warnings?.ToList() ?? [], EngineSourceLanguageTag = executionData.EngineSourceLanguageTag, EngineTargetLanguageTag = executionData.EngineTargetLanguageTag, diff --git a/src/Serval/src/Serval.WordAlignment.Contracts/ExecutionDataContract.cs b/src/Serval/src/Serval.WordAlignment.Contracts/ExecutionDataContract.cs index 78e75672..0227a5ee 100644 --- a/src/Serval/src/Serval.WordAlignment.Contracts/ExecutionDataContract.cs +++ b/src/Serval/src/Serval.WordAlignment.Contracts/ExecutionDataContract.cs @@ -4,6 +4,10 @@ public record ExecutionDataContract { public int? TrainCount { get; init; } public int? WordAlignCount { get; init; } + public bool? IsTrainFilteredByChapter { get; init; } + public bool? IsWordAlignmentFilteredByChapter { get; init; } + public Dictionary>? TrainVerseCount { get; init; } + public Dictionary>? WordAlignVerseCount { get; init; } public IReadOnlyList? Warnings { get; init; } public string? EngineSourceLanguageTag { get; init; } public string? EngineTargetLanguageTag { get; init; } diff --git a/src/Serval/src/Serval.WordAlignment/Dtos/WordAlignmentExecutionDataDto.cs b/src/Serval/src/Serval.WordAlignment/Dtos/WordAlignmentExecutionDataDto.cs index 8f37edfc..6aa2764f 100644 --- a/src/Serval/src/Serval.WordAlignment/Dtos/WordAlignmentExecutionDataDto.cs +++ b/src/Serval/src/Serval.WordAlignment/Dtos/WordAlignmentExecutionDataDto.cs @@ -4,6 +4,10 @@ public record WordAlignmentExecutionDataDto { public int TrainCount { get; init; } public int WordAlignCount { get; init; } + public bool? IsTrainFilteredByChapter { get; init; } + public bool? IsWordAlignFilteredByChapter { get; init; } + public Dictionary>? TrainVerseCount { get; init; } + public Dictionary>? WordAlignVerseCount { get; init; } public IReadOnlyList Warnings { get; init; } = []; public string? EngineSourceLanguageTag { get; init; } public string? EngineTargetLanguageTag { get; init; } diff --git a/src/Serval/src/Serval.WordAlignment/Models/ExecutionData.cs b/src/Serval/src/Serval.WordAlignment/Models/ExecutionData.cs index c606e08f..5908a109 100644 --- a/src/Serval/src/Serval.WordAlignment/Models/ExecutionData.cs +++ b/src/Serval/src/Serval.WordAlignment/Models/ExecutionData.cs @@ -4,6 +4,10 @@ public record ExecutionData { public int? TrainCount { get; init; } public int? WordAlignCount { get; init; } + public bool? IsTrainFilteredByChapter { get; init; } + public bool? IsWordAlignFilteredByChapter { get; init; } + public Dictionary>? TrainVerseCount { get; init; } + public Dictionary>? WordAlignVerseCount { get; init; } public IReadOnlyList? Warnings { get; init; } public string? EngineSourceLanguageTag { get; init; } public string? EngineTargetLanguageTag { get; init; } diff --git a/src/Serval/src/Serval.WordAlignment/Services/DtoMapper.cs b/src/Serval/src/Serval.WordAlignment/Services/DtoMapper.cs index 0731b262..ec36d427 100644 --- a/src/Serval/src/Serval.WordAlignment/Services/DtoMapper.cs +++ b/src/Serval/src/Serval.WordAlignment/Services/DtoMapper.cs @@ -190,6 +190,10 @@ private static WordAlignmentExecutionDataDto Map(ExecutionData source) { TrainCount = source.TrainCount ?? 0, WordAlignCount = source.WordAlignCount ?? 0, + TrainVerseCount = source.TrainVerseCount ?? [], + WordAlignVerseCount = source.WordAlignVerseCount ?? [], + IsTrainFilteredByChapter = source.IsTrainFilteredByChapter, + IsWordAlignFilteredByChapter = source.IsWordAlignFilteredByChapter, Warnings = source.Warnings ?? [], EngineSourceLanguageTag = source.EngineSourceLanguageTag, EngineTargetLanguageTag = source.EngineTargetLanguageTag, diff --git a/src/Serval/src/Serval.WordAlignment/Services/PlatformService.cs b/src/Serval/src/Serval.WordAlignment/Services/PlatformService.cs index 100aaaee..2b2de631 100644 --- a/src/Serval/src/Serval.WordAlignment/Services/PlatformService.cs +++ b/src/Serval/src/Serval.WordAlignment/Services/PlatformService.cs @@ -363,6 +363,10 @@ await _builds.UpdateAsync( { TrainCount = executionData.TrainCount, WordAlignCount = executionData.WordAlignCount, + TrainVerseCount = executionData.TrainVerseCount, + WordAlignVerseCount = executionData.WordAlignVerseCount, + IsTrainFilteredByChapter = executionData.IsTrainFilteredByChapter, + IsWordAlignFilteredByChapter = executionData.IsWordAlignmentFilteredByChapter, Warnings = executionData.Warnings?.ToList() ?? [], EngineSourceLanguageTag = executionData.EngineSourceLanguageTag, EngineTargetLanguageTag = executionData.EngineTargetLanguageTag, diff --git a/src/Serval/test/Serval.Translation.Tests/Services/PlatformServiceTests.cs b/src/Serval/test/Serval.Translation.Tests/Services/PlatformServiceTests.cs index 9e0d0901..9a2ff36d 100644 --- a/src/Serval/test/Serval.Translation.Tests/Services/PlatformServiceTests.cs +++ b/src/Serval/test/Serval.Translation.Tests/Services/PlatformServiceTests.cs @@ -157,7 +157,27 @@ public async Task UpdateBuildExecutionData() await env.PlatformService.UpdateBuildExecutionDataAsync( engine.Id, "123", - new() { TrainCount = 4, PretranslateCount = 5 } + new() + { + TrainCount = 4, + PretranslateCount = 5, + TrainVerseCount = new() + { + { + "MAT", + new() { { "1", 4 } } + }, + }, + PretranslateVerseCount = new() + { + { + "MAT", + new() { { "1", 10 } } + }, + }, + IsTrainFilteredByChapter = false, + IsPretranslateFilteredByChapter = true, + } ); build = await env.Builds.GetAsync(c => c.Id == build.Id); @@ -167,6 +187,19 @@ await env.PlatformService.UpdateBuildExecutionDataAsync( Assert.That(executionData, Is.Not.Null); Assert.That(executionData.TrainCount, Is.GreaterThan(0)); Assert.That(executionData.PretranslateCount, Is.GreaterThan(0)); + + Assert.That(executionData.TrainVerseCount, Is.Not.Null); + Assert.That(executionData.TrainVerseCount, Has.Count.EqualTo(1)); + Assert.That(executionData.TrainVerseCount.First().Value, Has.Count.EqualTo(1)); + Assert.That(executionData.TrainVerseCount.First().Value.First().Value, Is.EqualTo(4)); + + Assert.That(executionData.PretranslateVerseCount, Is.Not.Null); + Assert.That(executionData.PretranslateVerseCount, Has.Count.EqualTo(1)); + Assert.That(executionData.PretranslateVerseCount.First().Value, Has.Count.EqualTo(1)); + Assert.That(executionData.PretranslateVerseCount.First().Value.First().Value, Is.EqualTo(10)); + + Assert.That(executionData.IsTrainFilteredByChapter, Is.False); + Assert.That(executionData.IsPretranslateFilteredByChapter, Is.True); } [Test]