diff --git a/src/Echo/src/EchoEngine/TranslationEngineService.cs b/src/Echo/src/EchoEngine/TranslationEngineService.cs index 4c2e173a..e8a83338 100644 --- a/src/Echo/src/EchoEngine/TranslationEngineService.cs +++ b/src/Echo/src/EchoEngine/TranslationEngineService.cs @@ -237,6 +237,7 @@ .. tokens.Select( await platform.InsertPretranslationsAsync( engineId, + buildId, pretranslations.ToAsyncEnumerable(), linkedCts.Token ); diff --git a/src/Machine/src/Serval.Machine.Shared/Services/IPlatformService.cs b/src/Machine/src/Serval.Machine.Shared/Services/IPlatformService.cs index 5658aa35..5e385045 100644 --- a/src/Machine/src/Serval.Machine.Shared/Services/IPlatformService.cs +++ b/src/Machine/src/Serval.Machine.Shared/Services/IPlatformService.cs @@ -28,6 +28,7 @@ Task BuildCompletedAsync( Task InsertInferenceResultsAsync( string engineId, + string buildId, Stream pretranslationsStream, CancellationToken cancellationToken = default ); diff --git a/src/Machine/src/Serval.Machine.Shared/Services/ServalTranslationPlatformService.cs b/src/Machine/src/Serval.Machine.Shared/Services/ServalTranslationPlatformService.cs index 7ea06bdf..1eb9ebb4 100644 --- a/src/Machine/src/Serval.Machine.Shared/Services/ServalTranslationPlatformService.cs +++ b/src/Machine/src/Serval.Machine.Shared/Services/ServalTranslationPlatformService.cs @@ -70,12 +70,14 @@ public Task UpdateBuildStatusAsync(string buildId, int step, CancellationToken c public async Task InsertInferenceResultsAsync( string engineId, + string buildId, Stream pretranslationsStream, CancellationToken cancellationToken = default ) { await _platformService.InsertPretranslationsAsync( engineId, + buildId, ReadPretranslationsAsync(pretranslationsStream, cancellationToken), cancellationToken ); diff --git a/src/Machine/src/Serval.Machine.Shared/Services/ServalWordAlignmentPlatformService.cs b/src/Machine/src/Serval.Machine.Shared/Services/ServalWordAlignmentPlatformService.cs index 204daca6..29217d33 100644 --- a/src/Machine/src/Serval.Machine.Shared/Services/ServalWordAlignmentPlatformService.cs +++ b/src/Machine/src/Serval.Machine.Shared/Services/ServalWordAlignmentPlatformService.cs @@ -70,6 +70,7 @@ public Task UpdateBuildStatusAsync(string buildId, int step, CancellationToken c public async Task InsertInferenceResultsAsync( string engineId, + string buildId, Stream wordAlignmentsStream, CancellationToken cancellationToken = default ) diff --git a/src/Machine/src/Serval.Machine.Shared/Services/StatisticalPostprocessBuildJob.cs b/src/Machine/src/Serval.Machine.Shared/Services/StatisticalPostprocessBuildJob.cs index 7630e87a..fb8cdd90 100644 --- a/src/Machine/src/Serval.Machine.Shared/Services/StatisticalPostprocessBuildJob.cs +++ b/src/Machine/src/Serval.Machine.Shared/Services/StatisticalPostprocessBuildJob.cs @@ -43,7 +43,12 @@ CancellationToken cancellationToken ) ) { - await PlatformService.InsertInferenceResultsAsync(engineId, wordAlignmentStream, cancellationToken); + await PlatformService.InsertInferenceResultsAsync( + engineId, + buildId, + wordAlignmentStream, + cancellationToken + ); } int additionalCorpusSize = await SaveModelAsync(engineId, buildId, cancellationToken); diff --git a/src/Machine/src/Serval.Machine.Shared/Services/TranslationPostprocessBuildJob.cs b/src/Machine/src/Serval.Machine.Shared/Services/TranslationPostprocessBuildJob.cs index 3d93603e..1c75b6e3 100644 --- a/src/Machine/src/Serval.Machine.Shared/Services/TranslationPostprocessBuildJob.cs +++ b/src/Machine/src/Serval.Machine.Shared/Services/TranslationPostprocessBuildJob.cs @@ -36,7 +36,12 @@ CancellationToken cancellationToken ) ) { - await PlatformService.InsertInferenceResultsAsync(engineId, pretranslationsStream, cancellationToken); + await PlatformService.InsertInferenceResultsAsync( + engineId, + buildId, + pretranslationsStream, + cancellationToken + ); } int additionalCorpusSize = await SaveModelAsync(engineId, buildId, cancellationToken); diff --git a/src/Machine/test/Serval.Machine.Shared.Tests/Services/ServalTranslationPlatformServiceTests.cs b/src/Machine/test/Serval.Machine.Shared.Tests/Services/ServalTranslationPlatformServiceTests.cs index 72c81c7f..6d501f81 100644 --- a/src/Machine/test/Serval.Machine.Shared.Tests/Services/ServalTranslationPlatformServiceTests.cs +++ b/src/Machine/test/Serval.Machine.Shared.Tests/Services/ServalTranslationPlatformServiceTests.cs @@ -24,13 +24,14 @@ await JsonSerializer.SerializeAsync( } ); stream.Seek(0, SeekOrigin.Begin); - await env.Service.InsertInferenceResultsAsync("engine1", stream); + await env.Service.InsertInferenceResultsAsync("engine1", "build1", stream); } await env .PlatformService.Received() .InsertPretranslationsAsync( "engine1", + "build1", Arg.Any>(), Arg.Any() ); @@ -79,13 +80,14 @@ await JsonSerializer.SerializeAsync( } ); stream.Seek(0, SeekOrigin.Begin); - await env.Service.InsertInferenceResultsAsync("engine1", stream); + await env.Service.InsertInferenceResultsAsync("engine1", "build1", stream); } await env .PlatformService.Received() .InsertPretranslationsAsync( "engine1", + "build1", Arg.Any>(), Arg.Any() ); @@ -125,6 +127,7 @@ public TestEnvironment() PlatformService = Substitute.For(); PlatformService .InsertPretranslationsAsync( + Arg.Any(), Arg.Any(), Arg.Any>(), Arg.Any() diff --git a/src/Machine/test/Serval.Machine.Shared.Tests/Services/ServalWordAlignmentPlatformServiceTests.cs b/src/Machine/test/Serval.Machine.Shared.Tests/Services/ServalWordAlignmentPlatformServiceTests.cs index 64a215e9..6bf3b680 100644 --- a/src/Machine/test/Serval.Machine.Shared.Tests/Services/ServalWordAlignmentPlatformServiceTests.cs +++ b/src/Machine/test/Serval.Machine.Shared.Tests/Services/ServalWordAlignmentPlatformServiceTests.cs @@ -25,7 +25,7 @@ await JsonSerializer.SerializeAsync( } ); stream.Seek(0, SeekOrigin.Begin); - await env.Service.InsertInferenceResultsAsync("engine1", stream); + await env.Service.InsertInferenceResultsAsync("engine1", "build1", stream); } await env @@ -85,7 +85,7 @@ await JsonSerializer.SerializeAsync( } ); stream.Seek(0, SeekOrigin.Begin); - await env.Service.InsertInferenceResultsAsync("engine1", stream); + await env.Service.InsertInferenceResultsAsync("engine1", "build1", stream); } await env diff --git a/src/Serval/src/Serval.Client/Client.g.cs b/src/Serval/src/Serval.Client/Client.g.cs index 4b9b0e32..1b173fbd 100644 --- a/src/Serval/src/Serval.Client/Client.g.cs +++ b/src/Serval/src/Serval.Client/Client.g.cs @@ -11521,6 +11521,9 @@ public partial class ExecutionData [Newtonsoft.Json.JsonProperty("resolvedTargetLanguage", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] public string? ResolvedTargetLanguage { get; set; } = default!; + [Newtonsoft.Json.JsonProperty("averagePretranslationConfidence", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + public double? AveragePretranslationConfidence { get; set; } = default!; + } [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.6.3.0 (NJsonSchema v11.5.2.0 (Newtonsoft.Json v13.0.0.0))")] diff --git a/src/Serval/src/Serval.Translation.Contracts/ITranslationPlatformService.cs b/src/Serval/src/Serval.Translation.Contracts/ITranslationPlatformService.cs index 320ebff2..458f146f 100644 --- a/src/Serval/src/Serval.Translation.Contracts/ITranslationPlatformService.cs +++ b/src/Serval/src/Serval.Translation.Contracts/ITranslationPlatformService.cs @@ -26,6 +26,7 @@ Task UpdateBuildStatusAsync( Task IncrementEngineCorpusSizeAsync(string engineId, int count = 1, CancellationToken cancellationToken = default); Task InsertPretranslationsAsync( string engineId, + string buildId, IAsyncEnumerable pretranslations, CancellationToken cancellationToken = default ); diff --git a/src/Serval/src/Serval.Translation/Dtos/ExecutionDataDto.cs b/src/Serval/src/Serval.Translation/Dtos/ExecutionDataDto.cs index 46384ed9..ef48a34a 100644 --- a/src/Serval/src/Serval.Translation/Dtos/ExecutionDataDto.cs +++ b/src/Serval/src/Serval.Translation/Dtos/ExecutionDataDto.cs @@ -9,4 +9,5 @@ public record ExecutionDataDto public string? EngineTargetLanguageTag { get; init; } public string? ResolvedSourceLanguage { get; init; } public string? ResolvedTargetLanguage { get; init; } + public double? AveragePretranslationConfidence { get; init; } } diff --git a/src/Serval/src/Serval.Translation/Models/ExecutionData.cs b/src/Serval/src/Serval.Translation/Models/ExecutionData.cs index d1d509d9..79c953e7 100644 --- a/src/Serval/src/Serval.Translation/Models/ExecutionData.cs +++ b/src/Serval/src/Serval.Translation/Models/ExecutionData.cs @@ -9,4 +9,5 @@ public record ExecutionData public string? EngineTargetLanguageTag { get; init; } public string? ResolvedSourceLanguage { get; init; } public string? ResolvedTargetLanguage { get; init; } + public double? AveragePretranslationConfidence { get; init; } } diff --git a/src/Serval/src/Serval.Translation/Services/DtoMapper.cs b/src/Serval/src/Serval.Translation/Services/DtoMapper.cs index de3be2e7..8dd3e08f 100644 --- a/src/Serval/src/Serval.Translation/Services/DtoMapper.cs +++ b/src/Serval/src/Serval.Translation/Services/DtoMapper.cs @@ -236,6 +236,7 @@ private static ExecutionDataDto Map(ExecutionData source) => EngineTargetLanguageTag = source.EngineTargetLanguageTag, ResolvedSourceLanguage = source.ResolvedSourceLanguage, ResolvedTargetLanguage = source.ResolvedTargetLanguage, + AveragePretranslationConfidence = source.AveragePretranslationConfidence, }; } diff --git a/src/Serval/src/Serval.Translation/Services/PlatformService.cs b/src/Serval/src/Serval.Translation/Services/PlatformService.cs index 506e6e83..e0a24184 100644 --- a/src/Serval/src/Serval.Translation/Services/PlatformService.cs +++ b/src/Serval/src/Serval.Translation/Services/PlatformService.cs @@ -365,6 +365,7 @@ await _engines.UpdateAsync( public async Task InsertPretranslationsAsync( string engineId, + string buildId, IAsyncEnumerable pretranslations, CancellationToken cancellationToken = default ) @@ -375,6 +376,8 @@ public async Task InsertPretranslationsAsync( int nextModelRevision = engine.ModelRevision + 1; var batch = new List(); + double confidenceTotal = 0.0; + int numPretranslations = 0; await foreach (PretranslationContract item in pretranslations.WithCancellation(cancellationToken)) { batch.Add( @@ -401,6 +404,8 @@ public async Task InsertPretranslationsAsync( Confidence = item.Confidence, } ); + confidenceTotal += item.Confidence ?? 0.0; + numPretranslations += 1; if (batch.Count == PretranslationInsertBatchSize) { await _pretranslations.InsertAllAsync(batch, cancellationToken); @@ -409,5 +414,11 @@ public async Task InsertPretranslationsAsync( } if (batch.Count > 0) await _pretranslations.InsertAllAsync(batch, CancellationToken.None); + + await _builds.UpdateAsync( + b => b.Id == buildId, + u => u.Set(b => b.ExecutionData.AveragePretranslationConfidence, confidenceTotal / numPretranslations), + cancellationToken: cancellationToken + ); } } diff --git a/src/Serval/test/Serval.E2ETests/ServalApiTests.cs b/src/Serval/test/Serval.E2ETests/ServalApiTests.cs index 7e2fd7f7..5401649c 100644 --- a/src/Serval/test/Serval.E2ETests/ServalApiTests.cs +++ b/src/Serval/test/Serval.E2ETests/ServalApiTests.cs @@ -292,6 +292,7 @@ public async Task Nmt_Paratext(bool withAdditionalFiles) string buildId = await _helperClient.BuildEngineAsync(engineId); TranslationBuild build = await _helperClient.TranslationEnginesClient.GetBuildAsync(engineId, buildId); Assert.That(build.State, Is.EqualTo(JobState.Completed), JsonSerializer.Serialize(build)); + Assert.That(build.ExecutionData.AveragePretranslationConfidence, Is.GreaterThan(0.3)); IList translations = await _helperClient.TranslationEnginesClient.GetAllPretranslationsAsync( engineId, diff --git a/src/Serval/test/Serval.Translation.Tests/Services/PlatformServiceTests.cs b/src/Serval/test/Serval.Translation.Tests/Services/PlatformServiceTests.cs index 25153972..0289ed71 100644 --- a/src/Serval/test/Serval.Translation.Tests/Services/PlatformServiceTests.cs +++ b/src/Serval/test/Serval.Translation.Tests/Services/PlatformServiceTests.cs @@ -39,7 +39,7 @@ await env.Builds.InsertAsync( Assert.That(env.Engines.Get("e0").IsBuilding, Is.False); Assert.That(env.Pretranslations.Count, Is.EqualTo(0)); - await env.PlatformService.InsertPretranslationsAsync("e0", GetTestPretranslations()); + await env.PlatformService.InsertPretranslationsAsync("e0", "b0", GetTestPretranslations()); Assert.That(env.Pretranslations.Count, Is.EqualTo(1)); await env.PlatformService.BuildFaultedAsync("b0", ""); @@ -48,14 +48,18 @@ await env.Builds.InsertAsync( Assert.That(env.Engines.Get("e0").IsBuilding, Is.False); await env.PlatformService.BuildRestartingAsync("b0"); - await env.PlatformService.InsertPretranslationsAsync("e0", GetTestPretranslations()); + await env.PlatformService.InsertPretranslationsAsync("e0", "b0", GetTestPretranslations()); Assert.That(env.Pretranslations.Count, Is.EqualTo(1)); await env.PlatformService.BuildCompletedAsync("b0", 0, 0.0); Assert.That(env.Pretranslations.Count, Is.EqualTo(1)); await env.PlatformService.BuildStartedAsync("b0"); - await env.PlatformService.InsertPretranslationsAsync("e0", GetTestPretranslations()); + await env.PlatformService.InsertPretranslationsAsync("e0", "b0", GetTestPretranslations()); await env.PlatformService.BuildCompletedAsync("b0", 0, 0.0); Assert.That(env.Pretranslations.Count, Is.EqualTo(1)); + Assert.That( + (await env.Builds.GetAsync(b => b.Id == "b0"))?.ExecutionData.AveragePretranslationConfidence, + Is.Zero + ); } [Test]