Seed canonical morph types and regenerate search index

backend/FwLite/FwDataMiniLcmBridge.Tests/CanonicalMorphTypeTests.cs

@@ -0,0 +1,51 @@
+using FwDataMiniLcmBridge.Api;
+using FwDataMiniLcmBridge.LcmUtils;
+using FwDataMiniLcmBridge.Tests.Fixtures;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Options;
+using MiniLcm.Models;
+
+namespace FwDataMiniLcmBridge.Tests;
+
+public class CanonicalMorphTypeTests : IDisposable
+{
+    private readonly ServiceProvider _serviceProvider;
+    private readonly FwDataMiniLcmApi _api;
+    private readonly FwDataProject _project;
+
+    public CanonicalMorphTypeTests()
+    {
+        var services = new ServiceCollection()
+            .AddTestFwDataBridge(mockProjectLoader: false)
+            .PostConfigure<FwDataBridgeConfig>(config =>
+                config.TemplatesFolder = Path.GetFullPath("Templates"))
+            .BuildServiceProvider();
+        _serviceProvider = services;
+
+        var config = services.GetRequiredService<IOptions<FwDataBridgeConfig>>();
+        Directory.CreateDirectory(config.Value.ProjectsFolder);
+        var projectName = $"canonical-morph-types-test_{Guid.NewGuid()}";
+        _project = new FwDataProject(projectName, config.Value.ProjectsFolder);
+        var projectLoader = services.GetRequiredService<IProjectLoader>();
+        projectLoader.NewProject(_project, "en", "en");
+
+        var fwDataFactory = services.GetRequiredService<FwDataFactory>();
+        _api = fwDataFactory.GetFwDataMiniLcmApi(_project, false);
+    }
+
+    public void Dispose()
+    {
+        _api.Dispose();
+        _serviceProvider.Dispose();
+        if (Directory.Exists(_project.ProjectFolder))
+            Directory.Delete(_project.ProjectFolder, true);
+    }
+
+    [Fact]
+    public async Task CanonicalMorphTypes_MatchNewLangProjMorphTypes()
+    {
+        var libLcmMorphTypes = await _api.GetMorphTypes().ToArrayAsync();
+        libLcmMorphTypes.Should().NotBeEmpty();
+        CanonicalMorphTypes.All.Values.Should().BeEquivalentTo(libLcmMorphTypes);
+    }
+}

backend/FwLite/FwDataMiniLcmBridge.Tests/FwDataMiniLcmBridge.Tests.csproj

@@ -35,5 +35,9 @@
   </ItemGroup>
   <ItemGroup>
     <Folder Include="TestData\" />
+    <PackageReference Include="SIL.LCModel" GeneratePathProperty="true" />
+    <Content Include="$(PkgSIL_LCModel)/contentFiles/Templates/*.*"
+      Link="Templates/%(Filename)%(Extension)"
+      CopyToOutputDirectory="PreserveNewest" />
   </ItemGroup>
-</Project>
+</Project>

backend/FwLite/FwDataMiniLcmBridge.Tests/MiniLcmTests/SortingTests.cs

@@ -22,9 +22,9 @@ public async Task SecondaryOrder_DefaultsToStem(string query, SortField sortFiel
     {
         var unknownMorphTypeEntryId = Guid.NewGuid();
         Entry[] expected = [
-            new() { Id = unknownMorphTypeEntryId, LexemeForm = { ["en"] = "aaaa" }, MorphType = MorphTypeKind.Unknown }, // SecondaryOrder defaults to Stem = 1
-            new() { Id = Guid.NewGuid(), LexemeForm = { ["en"] = "aaaa" }, MorphType = MorphTypeKind.BoundStem }, // SecondaryOrder = 2
-            new() { Id = Guid.NewGuid(), LexemeForm = { ["en"] = "aaaa" }, MorphType = MorphTypeKind.Suffix }, // SecondaryOrder = 6
+            new() { Id = unknownMorphTypeEntryId, LexemeForm = { ["en"] = "aaaa" }, MorphType = MorphTypeKind.Unknown }, // SecondaryOrder defaults to Stem = 0
+            new() { Id = Guid.NewGuid(), LexemeForm = { ["en"] = "aaaa" }, MorphType = MorphTypeKind.BoundStem }, // SecondaryOrder = 10
+            new() { Id = Guid.NewGuid(), LexemeForm = { ["en"] = "aaaa" }, MorphType = MorphTypeKind.Suffix }, // SecondaryOrder = 70
         ];
 
         var ids = expected.Select(e => e.Id).ToHashSet();

backend/FwLite/FwDataMiniLcmBridge/Api/Sorting.cs

@@ -33,12 +33,16 @@ public static IEnumerable<ILexEntry> ApplyHeadwordOrder(this IEnumerable<ILexEnt
     /// </summary>
     public static IEnumerable<ILexEntry> ApplyRoughBestMatchOrder(this IEnumerable<ILexEntry> entries, SortOptions order, int sortWsHandle, int stemSecondaryOrder, string? query = null)
     {
-        var projected = entries.Select(e => (Entry: e, Headword: e.LexEntryHeadword(sortWsHandle, applyMorphTokens: false)));
+        var projected = entries.Select(e => (
+            Entry: e,
+            Headword: e.LexEntryHeadword(sortWsHandle, applyMorphTokens: false),
+            HeadwordWithTokens: e.LexEntryHeadword(sortWsHandle, applyMorphTokens: true)
+        ));
         if (order.Ascending)
         {
             return projected
-                .OrderByDescending(x => !string.IsNullOrEmpty(query) && (x.Headword?.StartsWithDiacriticMatch(query!) ?? false))
-                .ThenByDescending(x => !string.IsNullOrEmpty(query) && (x.Headword?.ContainsDiacriticMatch(query!) ?? false))
+                .OrderByDescending(x => !string.IsNullOrEmpty(query) && (x.HeadwordWithTokens?.StartsWithDiacriticMatch(query!) ?? false))
+                .ThenByDescending(x => !string.IsNullOrEmpty(query) && (x.HeadwordWithTokens?.ContainsDiacriticMatch(query!) ?? false))
                 .ThenBy(x => x.Headword?.Length ?? 0)
                 .ThenBy(x => x.Headword)
                 .ThenBy(x => x.Entry.PrimaryMorphType?.SecondaryOrder ?? stemSecondaryOrder)
@@ -49,8 +53,8 @@ public static IEnumerable<ILexEntry> ApplyRoughBestMatchOrder(this IEnumerable<I
         else
         {
             return projected
-                .OrderBy(x => !string.IsNullOrEmpty(query) && (x.Headword?.StartsWithDiacriticMatch(query!) ?? false))
-                .ThenBy(x => !string.IsNullOrEmpty(query) && (x.Headword?.ContainsDiacriticMatch(query!) ?? false))
+                .OrderBy(x => !string.IsNullOrEmpty(query) && (x.HeadwordWithTokens?.StartsWithDiacriticMatch(query!) ?? false))
+                .ThenBy(x => !string.IsNullOrEmpty(query) && (x.HeadwordWithTokens?.ContainsDiacriticMatch(query!) ?? false))
                 .ThenByDescending(x => x.Headword?.Length ?? 0)
                 .ThenByDescending(x => x.Headword)
                 .ThenByDescending(x => x.Entry.PrimaryMorphType?.SecondaryOrder ?? stemSecondaryOrder)

backend/FwLite/FwLiteProjectSync.Tests/FluentAssertGlobalConfig.cs

@@ -1,3 +1,4 @@
+using System.Runtime.CompilerServices;
 using FluentAssertions.Extensibility;
 using FwLiteProjectSync.Tests;
 
@@ -7,6 +8,12 @@ namespace FwLiteProjectSync.Tests;
 
 public static class FluentAssertGlobalConfig
 {
+    [ModuleInitializer]
+    internal static void InitVerify()
+    {
+        VerifierSettings.OmitContentFromException();
+    }
+
     public static void Initialize()
     {
         MiniLcm.Tests.FluentAssertGlobalConfig.Initialize();

backend/FwLite/FwLiteProjectSync.Tests/Import/ResumableTests.cs

@@ -38,18 +38,7 @@ public async Task ImportProject_IsResumable_AcrossRandomFailures()
             }).ToList();
         var expectedPartsOfSpeech = Enumerable.Range(1, 10)
             .Select(i => new PartOfSpeech { Id = Guid.NewGuid(), Name = { ["en"] = $"pos{i}" } }).ToList();
-        var expectedMorphTypes = Enum.GetValues<MorphTypeKind>()
-            .Select(typ => new MorphType()
-            {
-                Id = Guid.NewGuid(),
-                Name = new() { ["en"] = $"Test Morph Type {(int)typ} {typ}" },
-                Abbreviation = new() { ["en"] = $"Tst MrphTyp{(int)typ}" },
-                Description = new() { { "en", new RichString($"test desc for {typ}") } },
-                Prefix = null,
-                Postfix = null,
-                Kind = typ,
-                SecondaryOrder = 0
-            }).ToList();
+        var expectedMorphTypes = CanonicalMorphTypes.All.Values;
 
         var mockFrom = new Mock<IMiniLcmApi>();
         IMiniLcmApi mockTo = new UnreliableApi(
@@ -132,7 +121,6 @@ public async Task ImportProject_IsResumable_AcrossRandomFailures()
         createdEntries.Select(e => e.LexemeForm["en"]).Should().BeEquivalentTo(expectedEntries.Select(e => e.LexemeForm["en"]));
         createdMorphTypes.Select(e => e.Name["en"]).Should().BeEquivalentTo(expectedMorphTypes.Select(e => e.Name["en"]));
         createdMorphTypes.Select(e => e.Kind).Should().BeEquivalentTo(expectedMorphTypes.Select(e => e.Kind));
-
     }
 
 

backend/FwLite/FwLiteProjectSync/CrdtFwdataProjectSyncService.cs

@@ -76,6 +76,18 @@ private async Task<SyncResult> SyncOrImportInternal(IMiniLcmApi crdtApi, IMiniLc
         {
             // Repair any missing translation IDs before doing the full sync, so the sync doesn't have to deal with them
             var syncedIdCount = await CrdtRepairs.SyncMissingTranslationIds(projectSnapshot.Entries, fwdata, crdt, dryRun);
+
+            // Patch legacy snapshots that were created before morph-type support.
+            // After seeding, the CRDT has morph-types but the snapshot still has [].
+            // Without this patch, the diff would see all morph-types as "new" and try to re-add them.
+            if (projectSnapshot.MorphTypes.Length == 0)
+            {
+                var currentCrdtMorphTypes = await crdt.GetMorphTypes().ToArrayAsync();
+                if (currentCrdtMorphTypes.Length > 0)
+                {
+                    projectSnapshot = projectSnapshot with { MorphTypes = currentCrdtMorphTypes };
+                }
+            }
         }
 
         var syncResult = projectSnapshot is null

backend/FwLite/FwLiteProjectSync/MiniLcmImport.cs

@@ -8,6 +8,7 @@
 using MiniLcm;
 using MiniLcm.Models;
 using MiniLcm.Project;
+using MiniLcm.SyncHelpers;
 
 namespace FwLiteProjectSync;
 
@@ -70,6 +71,11 @@ public async Task ImportProject(IMiniLcmApi importTo, IMiniLcmApi importFrom, in
             logger.LogInformation("Imported complex form type {Id}", complexFormType.Id);
         }
 
+        // Morph types are created automatically for CRDT projects, so we update them instead of creating them
+        var importFromMorphTypes = await importFrom.GetMorphTypes().ToArrayAsync();
+        var existingMorphTypes = await importTo.GetMorphTypes().ToArrayAsync();
+        await MorphTypeSync.Sync(existingMorphTypes, importFromMorphTypes, importTo);
+
         await foreach (var morphType in importFrom.GetMorphTypes())
         {
             await importTo.CreateMorphType(morphType);

backend/FwLite/LcmCrdt.Tests/Data/FilteringTests.cs

@@ -12,9 +12,9 @@ public FilteringTests()
         _entries =
         [
             new Entry { LexemeForm = { { "en", "123" } }, },
-            new Entry { LexemeForm = { { "en", "456" } }, }
+            new Entry { LexemeForm = { { "en", "456" } }, },
         ];
-        _morphTypes = new MorphType[] { new() { Id = Guid.NewGuid(), Kind = MorphTypeKind.Stem, Name = { ["en"] = "Stem" }, SecondaryOrder = 1 } }.AsQueryable();
+        _morphTypes = CanonicalMorphTypes.All.Values.ToArray().AsQueryable();
     }
 
     [Theory]

backend/FwLite/LcmCrdt.Tests/Data/MigrationTests.cs

@@ -22,6 +22,7 @@ public class MigrationTests : IAsyncLifetime
     internal static void Init()
     {
         VerifySystemJson.Initialize();
+        VerifierSettings.OmitContentFromException();
     }
 
     public Task InitializeAsync()

backend/FwLite/LcmCrdt.Tests/Data/RegressionTestHelper.cs

@@ -73,6 +73,6 @@ private static string GetFilePath(string name, [CallerFilePath] string sourceFil
     public enum RegressionVersion
     {
         v1,
-        v2
+        v2,
     }
 }

backend/FwLite/LcmCrdt.Tests/Data/SnapshotDeserializationRegressionData.latest.verified.txt

@@ -1613,7 +1613,7 @@
       "Analysis": null
     },
     "Id": "72bcb2dd-4d73-486c-d5df-e5d7b9a6a3eb",
-    "DeletedAt": null,
+    "DeletedAt": null
   },
   {
     "$type": "MiniLcmCrdtAdapter",

backend/FwLite/LcmCrdt.Tests/DataModelSnapshotTests.VerifyChangeModels.verified.txt

@@ -192,6 +192,14 @@
       DerivedType: CreateCustomViewChange,
       TypeDiscriminator: CreateCustomViewChange
     },
+    {
+      DerivedType: EditCustomViewChange,
+      TypeDiscriminator: EditCustomViewChange
+    },
+    {
+      DerivedType: DeleteChange<CustomView>,
+      TypeDiscriminator: delete:CustomView
+    },
     {
       DerivedType: CreateMorphTypeChange,
       TypeDiscriminator: CreateMorphTypeChange

backend/FwLite/LcmCrdt.Tests/MiniLcmApiFixture.cs

@@ -1,5 +1,6 @@
 using System.Diagnostics;
 using LcmCrdt.MediaServer;
+using LcmCrdt.Objects;
 using Meziantou.Extensions.Logging.Xunit;
 using Microsoft.EntityFrameworkCore;
 using Microsoft.Extensions.DependencyInjection;
@@ -71,9 +72,12 @@ public async Task InitializeAsync(string projectName)
         _crdtDbContext = await _services.ServiceProvider.GetRequiredService<IDbContextFactory<LcmCrdtDbContext>>().CreateDbContextAsync();
         await _crdtDbContext.Database.OpenConnectionAsync();
         //can't use ProjectsService.CreateProject because it opens and closes the db context, this would wipe out the in memory db.
+        var projectData = new ProjectData("Sena 3", projectName, Guid.NewGuid(), null, Guid.NewGuid());
         await CrdtProjectsService.InitProjectDb(_crdtDbContext,
-            new ProjectData("Sena 3", projectName, Guid.NewGuid(), null, Guid.NewGuid()));
+            projectData);
         await currentProjectService.RefreshProjectData();
+        // also need to manually add morph-types, because we're not using CreateProject
+        await PreDefinedData.PredefinedMorphTypes(_services.ServiceProvider.GetRequiredService<DataModel>(), projectData.ClientId);
         if (_seedWs)
         {
             await Api.CreateWritingSystem(new WritingSystem()

backend/FwLite/LcmCrdt.Tests/MiniLcmTests/SortingTests.cs

@@ -24,19 +24,10 @@ public override async Task DisposeAsync()
     [InlineData("a", SortField.SearchRelevance)] // non-FTS
     public async Task SecondaryOrder_DefaultsToStem(string query, SortField sortField)
     {
-        MorphType[] morphTypes = [
-            new() { Id = Guid.NewGuid(), Kind = MorphTypeKind.Stem, Name = { ["en"] = "Stem" }, SecondaryOrder = 1 },
-            new() { Id = Guid.NewGuid(), Kind = MorphTypeKind.BoundStem, Name = { ["en"] = "BoundStem" }, SecondaryOrder = 2 },
-            new() { Id = Guid.NewGuid(), Kind = MorphTypeKind.Suffix, Name = { ["en"] = "Suffix" }, Postfix = "-", SecondaryOrder = 6 },
-        ];
-
-        foreach (var morphType in morphTypes)
-            await Api.CreateMorphType(morphType);
-
         Entry[] expected = [
-            new() { Id = Guid.NewGuid(), LexemeForm = { ["en"] = "aaaa" }, MorphType = MorphTypeKind.Unknown }, // SecondaryOrder defaults to Stem = 1
-            new() { Id = Guid.NewGuid(), LexemeForm = { ["en"] = "aaaa" }, MorphType = MorphTypeKind.BoundStem }, // SecondaryOrder = 2
-            new() { Id = Guid.NewGuid(), LexemeForm = { ["en"] = "aaaa" }, MorphType = MorphTypeKind.Suffix }, // SecondaryOrder = 6
+            new() { Id = Guid.NewGuid(), LexemeForm = { ["en"] = "aaaa" }, MorphType = MorphTypeKind.Unknown }, // SecondaryOrder defaults to Stem = 0
+            new() { Id = Guid.NewGuid(), LexemeForm = { ["en"] = "aaaa" }, MorphType = MorphTypeKind.BoundStem }, // SecondaryOrder = 10
+            new() { Id = Guid.NewGuid(), LexemeForm = { ["en"] = "aaaa" }, MorphType = MorphTypeKind.Suffix }, // SecondaryOrder = 70
         ];
 
         var ids = expected.Select(e => e.Id).ToHashSet();