diff --git a/src/Machine/src/Serval.Machine.Shared/Services/NmtClearMLBuildJobFactory.cs b/src/Machine/src/Serval.Machine.Shared/Services/NmtClearMLBuildJobFactory.cs index 4ab83dc4..4758a7b5 100644 --- a/src/Machine/src/Serval.Machine.Shared/Services/NmtClearMLBuildJobFactory.cs +++ b/src/Machine/src/Serval.Machine.Shared/Services/NmtClearMLBuildJobFactory.cs @@ -41,7 +41,11 @@ public async Task CreateJobScriptAsync( + $" 'trg_lang': '{trgLang}',\n" + $" 'shared_file_uri': '{baseUri}',\n" + $" 'shared_file_folder': '{folder}',\n" - + (buildOptions is not null ? $" 'build_options': '''{buildOptions}''',\n" : "") + + ( + buildOptions is not null + ? $" 'build_options': '''{Convert.ToBase64String(Encoding.UTF8.GetBytes(buildOptions))}''',\n" + : "" + ) // buildRevision + 1 because the build revision is incremented after the build job // is finished successfully but the file should be saved with the new revision number + (engine.IsModelPersisted ? $" 'save_model': '{engineId}_{engine.BuildRevision + 1}',\n" : $"") diff --git a/src/Machine/src/Serval.Machine.Shared/Services/SmtTransferClearMLBuildJobFactory.cs b/src/Machine/src/Serval.Machine.Shared/Services/SmtTransferClearMLBuildJobFactory.cs index 0552c093..ca5ba04b 100644 --- a/src/Machine/src/Serval.Machine.Shared/Services/SmtTransferClearMLBuildJobFactory.cs +++ b/src/Machine/src/Serval.Machine.Shared/Services/SmtTransferClearMLBuildJobFactory.cs @@ -35,7 +35,11 @@ public async Task CreateJobScriptAsync( + $" 'build_id': '{buildId}',\n" + $" 'shared_file_uri': '{baseUri}',\n" + $" 'shared_file_folder': '{folder}',\n" - + (buildOptions is not null ? $" 'build_options': '''{buildOptions}''',\n" : "") + + ( + buildOptions is not null + ? $" 'build_options': '''{Convert.ToBase64String(Encoding.UTF8.GetBytes(buildOptions))}''',\n" + : "" + ) + $" 'clearml': True\n" + "}\n" + "run(args)\n"; diff --git a/src/Machine/src/Serval.Machine.Shared/Services/StatsiticalClearMLBuildJobFactory.cs b/src/Machine/src/Serval.Machine.Shared/Services/StatisticalClearMLBuildJobFactory.cs similarity index 88% rename from src/Machine/src/Serval.Machine.Shared/Services/StatsiticalClearMLBuildJobFactory.cs rename to src/Machine/src/Serval.Machine.Shared/Services/StatisticalClearMLBuildJobFactory.cs index e9395857..47493378 100644 --- a/src/Machine/src/Serval.Machine.Shared/Services/StatsiticalClearMLBuildJobFactory.cs +++ b/src/Machine/src/Serval.Machine.Shared/Services/StatisticalClearMLBuildJobFactory.cs @@ -35,7 +35,11 @@ public async Task CreateJobScriptAsync( + $" 'build_id': '{buildId}',\n" + $" 'shared_file_uri': '{baseUri}',\n" + $" 'shared_file_folder': '{folder}',\n" - + (buildOptions is not null ? $" 'build_options': '''{buildOptions}''',\n" : "") + + ( + buildOptions is not null + ? $" 'build_options': '''{Convert.ToBase64String(Encoding.UTF8.GetBytes(buildOptions))}''',\n" + : "" + ) + $" 'clearml': True\n" + "}\n" + "run(args)\n"; diff --git a/src/Machine/test/Serval.Machine.Shared.Tests/Services/NmtClearMLBuildJobFactoryTests.cs b/src/Machine/test/Serval.Machine.Shared.Tests/Services/NmtClearMLBuildJobFactoryTests.cs index baa5cef5..e36dabc4 100644 --- a/src/Machine/test/Serval.Machine.Shared.Tests/Services/NmtClearMLBuildJobFactoryTests.cs +++ b/src/Machine/test/Serval.Machine.Shared.Tests/Services/NmtClearMLBuildJobFactoryTests.cs @@ -7,18 +7,20 @@ public class NmtClearMLBuildJobFactoryTests public async Task CreateJobScriptAsync_BuildOptions() { var env = new TestEnvironment(); + const string BuildOptions = "{ \"max_steps\": \"10\" }"; + string buildOptionsBase64 = Convert.ToBase64String(Encoding.UTF8.GetBytes(BuildOptions)); string script = await env.BuildJobFactory.CreateJobScriptAsync( "engine1", "build1", "test_model", BuildStage.Train, - buildOptions: "{ \"max_steps\": \"10\" }" + buildOptions: BuildOptions ); Assert.That( script, Is.EqualTo( - @"from machine.jobs.build_nmt_engine import run -args = { + $@"from machine.jobs.build_nmt_engine import run +args = {{ 'model_type': 'test_model', 'engine_id': 'engine1', 'build_id': 'build1', @@ -26,9 +28,9 @@ public async Task CreateJobScriptAsync_BuildOptions() 'trg_lang': 'eng_Latn', 'shared_file_uri': 's3://bucket', 'shared_file_folder': 'folder1/folder2', - 'build_options': '''{ ""max_steps"": ""10"" }''', + 'build_options': '''{buildOptionsBase64}''', 'clearml': True -} +}} run(args) ".ReplaceLineEndings("\n") ) diff --git a/src/Machine/test/Serval.Machine.Shared.Tests/Usings.cs b/src/Machine/test/Serval.Machine.Shared.Tests/Usings.cs index 4696e0f4..c1214b76 100644 --- a/src/Machine/test/Serval.Machine.Shared.Tests/Usings.cs +++ b/src/Machine/test/Serval.Machine.Shared.Tests/Usings.cs @@ -1,3 +1,4 @@ +global using System.Text; global using System.Text.Json; global using System.Text.Json.Nodes; global using Microsoft.Extensions.DependencyInjection; @@ -6,7 +7,6 @@ global using Microsoft.Extensions.Logging; global using Microsoft.Extensions.Options; global using NSubstitute; -global using NSubstitute.ReceivedExtensions; global using NUnit.Framework; global using NUnit.Framework.Constraints; global using RichardSzalay.MockHttp;