Skip to content

Add chapter-level data counts to execution data #990

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Enkidu93 wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Add chapter-level data counts to execution data #990

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 5 additions & 2 deletions src/Machine/src/Serval.Machine.Shared/Models/BuildExecutionData.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,11 @@ namespace Serval.Machine.Shared.Models;
public record BuildExecutionData
{
public int? TrainCount { get; init; }
public int? PretranslateCount { get; init; }
public int? WordAlignCount { get; init; }
public int? InferenceCount { get; init; }
public bool? IsTrainFilteredByChapter { get; init; }
public bool? IsInferenceFilteredByChapter { get; init; }
public Dictionary<string, Dictionary<string, int>>? TrainVerseCount { get; init; }
public Dictionary<string, Dictionary<string, int>>? InferenceVerseCount { get; init; }
public IReadOnlyList<string>? Warnings { get; init; }
public string? EngineSourceLanguageTag { get; init; }
public string? EngineTargetLanguageTag { get; init; }
Expand Down
21 changes: 12 additions & 9 deletions src/Machine/src/Serval.Machine.Shared/Services/NmtPreprocessBuildJob.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,8 +52,7 @@ await PlatformService.UpdateTargetQuoteConventionAsync(
protected override async Task UpdateBuildExecutionData(
string engineId,
string buildId,
int trainCount,
int pretranslateCount,
PreprocessStats stats,
string sourceLanguageTag,
string targetLanguageTag,
IReadOnlyList<ParallelCorpusContract> parallelCorpora,
Expand All @@ -63,16 +62,16 @@ CancellationToken cancellationToken
bool sourceLanguageHasNativeSupport = ResolveLanguageCode(sourceLanguageTag, out string resolvedSourceLanguage);
bool targetLanguageHasNativeSupport = ResolveLanguageCode(targetLanguageTag, out string resolvedTargetLanguage);

if (trainCount == 0 && (!sourceLanguageHasNativeSupport || !targetLanguageHasNativeSupport))
if (stats.TrainCount == 0 && (!sourceLanguageHasNativeSupport || !targetLanguageHasNativeSupport))
{
throw new InvalidOperationException(
$"At least one language code in build {buildId} is unknown to the base model, and the data specified for training was empty. Build canceled."
);
}

IReadOnlyList<string> warnings = GetWarnings(
trainCount,
pretranslateCount,
stats.TrainCount,
stats.InferenceCount,
sourceLanguageTag,
targetLanguageTag,
parallelCorpora
Expand All @@ -92,8 +91,8 @@ CancellationToken cancellationToken
{ "Event", "BuildPreprocess" },
{ "EngineId", engineId },
{ "BuildId", buildId },
{ "NumTrainRows", trainCount },
{ "NumPretranslateRows", pretranslateCount },
{ "NumTrainRows", stats.TrainCount },
{ "NumPretranslateRows", stats.InferenceCount },
{ "EngineSourceLanguageTag", sourceLanguageTag },
{ "EngineTargetLanguageTag", targetLanguageTag },
{ "SourceLanguageResolved", resolvedSourceLanguage },
Expand All @@ -103,8 +102,12 @@ CancellationToken cancellationToken
Logger.LogInformation("{summary}", buildPreprocessSummary.ToJsonString());
var executionData = new BuildExecutionData()
{
TrainCount = trainCount,
PretranslateCount = pretranslateCount,
TrainCount = stats.TrainCount,
InferenceCount = stats.InferenceCount,
TrainVerseCount = stats.TrainVerseCount,
InferenceVerseCount = stats.InferenceVerseCount,
IsTrainFilteredByChapter = stats.IsTrainFilteredByChapter,
IsInferenceFilteredByChapter = stats.IsInferenceFilteredByChapter,
Warnings = warnings,
EngineSourceLanguageTag = sourceLanguageTag,
EngineTargetLanguageTag = targetLanguageTag,
Expand Down
17 changes: 5 additions & 12 deletions src/Machine/src/Serval.Machine.Shared/Services/PreprocessBuildJob.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,18 +45,12 @@ CancellationToken cancellationToken
if (engine is null)
throw new OperationCanceledException($"Engine {engineId} does not exist. Build canceled.");

(int trainCount, int inferenceCount) = await WriteDataFilesAsync(
buildId,
data,
buildOptions,
cancellationToken
);
PreprocessStats stats = await WriteDataFilesAsync(buildId, data, buildOptions, cancellationToken);

await UpdateBuildExecutionData(
engineId,
buildId,
trainCount,
inferenceCount,
stats,
engine.SourceLanguage,
engine.TargetLanguage,
data,
Expand All @@ -65,7 +59,7 @@ await UpdateBuildExecutionData(

await UpdateTargetQuoteConventionAsync(engineId, buildId, data, cancellationToken);

if (inferenceCount == 0 && engine is TranslationEngine { IsModelPersisted: false })
if (stats.InferenceCount == 0 && engine is TranslationEngine { IsModelPersisted: false })
{
throw new InvalidOperationException(
$"There was no data specified for inferencing in build {buildId}. Build canceled."
Expand All @@ -90,8 +84,7 @@ await UpdateBuildExecutionData(
protected abstract Task UpdateBuildExecutionData(
string engineId,
string buildId,
int trainCount,
int inferenceCount,
PreprocessStats stats,
string sourceLanguageTag,
string targetLanguageTag,
IReadOnlyList<ParallelCorpusContract> parallelCorpora,
Expand All @@ -105,7 +98,7 @@ protected virtual Task UpdateTargetQuoteConventionAsync(
CancellationToken cancellationToken
) => Task.CompletedTask;

protected abstract Task<(int TrainCount, int InferenceCount)> WriteDataFilesAsync(
protected abstract Task<PreprocessStats> WriteDataFilesAsync(
string buildId,
IReadOnlyList<ParallelCorpusContract> parallelCorpora,
string? buildOptions,
Expand Down
11 changes: 11 additions & 0 deletions src/Machine/src/Serval.Machine.Shared/Services/PreprocessStats.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
namespace Serval.Machine.Shared.Services;

public record PreprocessStats
{
public required int TrainCount { get; init; }
public required int InferenceCount { get; init; }
public required bool IsTrainFilteredByChapter { get; init; }
public required bool IsInferenceFilteredByChapter { get; init; }
public required Dictionary<string, Dictionary<string, int>> TrainVerseCount { get; init; }
public required Dictionary<string, Dictionary<string, int>> InferenceVerseCount { get; init; }
}
6 changes: 5 additions & 1 deletion src/Machine/src/Serval.Machine.Shared/Services/ServalTranslationPlatformService.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -101,7 +101,11 @@ public Task UpdateBuildExecutionDataAsync(
new ExecutionDataContract
{
TrainCount = executionData.TrainCount,
PretranslateCount = executionData.PretranslateCount,
PretranslateCount = executionData.InferenceCount,
TrainVerseCount = executionData.TrainVerseCount,
PretranslateVerseCount = executionData.InferenceVerseCount,
IsTrainFilteredByChapter = executionData.IsTrainFilteredByChapter,
IsPretranslateFilteredByChapter = executionData.IsInferenceFilteredByChapter,
Warnings = executionData.Warnings,
EngineSourceLanguageTag = executionData.EngineSourceLanguageTag,
EngineTargetLanguageTag = executionData.EngineTargetLanguageTag,
Expand Down
6 changes: 5 additions & 1 deletion src/Machine/src/Serval.Machine.Shared/Services/ServalWordAlignmentPlatformService.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -100,7 +100,11 @@ public Task UpdateBuildExecutionDataAsync(
new ExecutionDataContract
{
TrainCount = executionData.TrainCount,
WordAlignCount = executionData.WordAlignCount,
WordAlignCount = executionData.InferenceCount,
IsTrainFilteredByChapter = executionData.IsTrainFilteredByChapter,
IsWordAlignmentFilteredByChapter = executionData.IsInferenceFilteredByChapter,
TrainVerseCount = executionData.TrainVerseCount,
WordAlignVerseCount = executionData.InferenceVerseCount,
Warnings = executionData.Warnings,
EngineSourceLanguageTag = executionData.EngineSourceLanguageTag,
EngineTargetLanguageTag = executionData.EngineTargetLanguageTag,
Expand Down
84 changes: 73 additions & 11 deletions src/Machine/src/Serval.Machine.Shared/Services/TranslationPreprocessBuildJob.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
using SIL.Extensions;

namespace Serval.Machine.Shared.Services;

public class TranslationPreprocessBuildJob(
Expand All @@ -21,7 +23,7 @@ IOptionsMonitor<BuildJobOptions> options
options
)
{
protected override async Task<(int TrainCount, int InferenceCount)> WriteDataFilesAsync(
protected override async Task<PreprocessStats> WriteDataFilesAsync(
string buildId,
IReadOnlyList<ParallelCorpusContract> parallelCorpora,
string? buildOptions,
Expand Down Expand Up @@ -51,9 +53,20 @@ await SharedFileService.OpenWriteAsync($"builds/{buildId}/train.key-terms.trg.tx
cancellationToken
);
await using Utf8JsonWriter pretranslateWriter = new(pretranslateStream, InferenceWriterOptions);

bool isTrainFilteredByChapter = parallelCorpora.Any(pc =>
pc.SourceCorpora.Any(c =>
c.TrainOnChapters is not null && c.TrainOnChapters.Values.Any(chapters => chapters.Count > 0)
)
);
bool isPretranslationFilteredByChapter = parallelCorpora.Any(pc =>
pc.SourceCorpora.Any(c =>
c.InferenceChapters is not null && c.InferenceChapters.Values.Any(chapters => chapters.Count > 0)
)
);
int trainCount = 0;
int pretranslateCount = 0;
Dictionary<string, Dictionary<string, int>> trainVerseCountByChapter = [];
Dictionary<string, Dictionary<string, int>> pretranslateVerseCountByChapter = [];
pretranslateWriter.WriteStartArray();
await ParallelCorpusService.PreprocessAsync(
parallelCorpora,
Expand All @@ -73,7 +86,27 @@ await ParallelCorpusService.PreprocessAsync(
}
}
if (row.SourceSegment.Length > 0 && row.TargetSegment.Length > 0)
{
trainCount++;
foreach (object? reference in row.SourceRefs)
{
if (reference is not null and ScriptureRef sr)
{
trainVerseCountByChapter.UpdateValue(
sr.Book,
() => [],
chapters =>
{
if (chapters.TryGetValue(sr.Chapter, out int count))
chapters[sr.Chapter] = count + 1;
else
chapters[sr.Chapter] = 1;
return chapters;
}
);
}
}
}
},
async (row, isInTrainingData, corpusId) =>
{
Expand All @@ -93,6 +126,24 @@ await ParallelCorpusService.PreprocessAsync(
pretranslateWriter.WriteString("translation", row.SourceSegment);
pretranslateWriter.WriteEndObject();
pretranslateCount++;
foreach (object? reference in row.SourceRefs)
{
if (reference is not null and ScriptureRef sr)
{
pretranslateVerseCountByChapter.UpdateValue(
sr.Book,
() => [],
chapters =>
{
if (chapters.TryGetValue(sr.Chapter, out int count))
chapters[sr.Chapter] = count + 1;
else
chapters[sr.Chapter] = 1;
return chapters;
}
);
}
}
}
if (pretranslateWriter.BytesPending > 1024 * 1024)
await pretranslateWriter.FlushAsync();
Expand All @@ -103,23 +154,30 @@ await ParallelCorpusService.PreprocessAsync(

pretranslateWriter.WriteEndArray();

return (trainCount, pretranslateCount);
return new PreprocessStats
{
TrainCount = trainCount,
InferenceCount = pretranslateCount,
IsTrainFilteredByChapter = isTrainFilteredByChapter,
IsInferenceFilteredByChapter = isPretranslationFilteredByChapter,
TrainVerseCount = trainVerseCountByChapter,
InferenceVerseCount = pretranslateVerseCountByChapter,
};
}

protected override async Task UpdateBuildExecutionData(
string engineId,
string buildId,
int trainCount,
int pretranslateCount,
PreprocessStats stats,
string sourceLanguageTag,
string targetLanguageTag,
IReadOnlyList<ParallelCorpusContract> parallelCorpora,
CancellationToken cancellationToken
)
{
IReadOnlyList<string> warnings = GetWarnings(
trainCount,
pretranslateCount,
stats.TrainCount,
stats.InferenceCount,
sourceLanguageTag,
targetLanguageTag,
parallelCorpora
Expand All @@ -131,17 +189,21 @@ CancellationToken cancellationToken
{ "Event", "BuildPreprocess" },
{ "EngineId", engineId },
{ "BuildId", buildId },
{ "NumTrainRows", trainCount },
{ "NumPretranslateRows", pretranslateCount },
{ "NumTrainRows", stats.TrainCount },
{ "NumPretranslateRows", stats.InferenceCount },
{ "EngineSourceLanguageTag", sourceLanguageTag },
{ "EngineTargetLanguageTag", targetLanguageTag },
{ "Warnings", new JsonArray(warnings.Select(w => JsonValue.Create(w)).ToArray()) },
};
Logger.LogInformation("{summary}", buildPreprocessSummary.ToJsonString());
var executionData = new BuildExecutionData()
{
TrainCount = trainCount,
PretranslateCount = pretranslateCount,
TrainCount = stats.TrainCount,
InferenceCount = stats.InferenceCount,
IsTrainFilteredByChapter = stats.IsTrainFilteredByChapter,
IsInferenceFilteredByChapter = stats.IsInferenceFilteredByChapter,
TrainVerseCount = stats.TrainVerseCount,
InferenceVerseCount = stats.InferenceVerseCount,
Warnings = warnings,
EngineSourceLanguageTag = sourceLanguageTag,
EngineTargetLanguageTag = targetLanguageTag,
Expand Down
Loading
Loading