diff --git a/CONTEXT-MAP.md b/CONTEXT-MAP.md index b9b94c0c41..181eb5568d 100644 --- a/CONTEXT-MAP.md +++ b/CONTEXT-MAP.md @@ -4,4 +4,4 @@ Domain glossaries for this monorepo. Each `CONTEXT.md` is a glossary only — no ## Contexts -- [FwLite Commenting](./backend/FwLite/CONTEXT.md) — collaborative comments on dictionary entries, senses, and example sentences +- [FwLite](./backend/FwLite/CONTEXT.md) — dictionary editing concerns including collation and collaborative comments diff --git a/backend/Directory.Packages.props b/backend/Directory.Packages.props index d717033990..3d869111b8 100644 --- a/backend/Directory.Packages.props +++ b/backend/Directory.Packages.props @@ -29,6 +29,7 @@ + diff --git a/backend/FwLite/CONTEXT.md b/backend/FwLite/CONTEXT.md index b414b42d0b..a5f61cb073 100644 --- a/backend/FwLite/CONTEXT.md +++ b/backend/FwLite/CONTEXT.md @@ -1,4 +1,46 @@ -# FwLite Commenting +# FwLite + +## Collation + +How strings in a writing system are compared when ordering dictionary content. Imported from FLEx per writing system; stored on `WritingSystem` without LDML at runtime. + +**Collation**: +The rules that govern string comparison for a writing system (ICU tailoring or .NET locale alias). Distinct from list position and from the user action of sorting entries. +_Avoid_: Sort rules (when meaning collation specifically), custom sorting (as a model term) + +**Writing system order**: +Where a writing system appears in the project's vernacular or analysis list (`WritingSystem.Order`). Not how strings alphabetize. +_Avoid_: Confusing with collation + +**Entry sort**: +The user-facing action of ordering the entry list (e.g. by headword in a chosen writing system). Uses the selected writing system's collation for headword comparison. +_Avoid_: Collation (when meaning the user action) + +**Collation scope (v1)**: +Collation governs headword entry sort (`SortField.Headword`) and the alphabetical tie-break when search results fall through to headword order (`SortField.SearchRelevance`). It does not govern search matching (whether a query matches text) or FTS ranking. +_Avoid_: Applying collation to `ContainsDiacriticMatch`, prefix/contains predicates, or FTS rank + +**Imported collation**: +Collation metadata synced from FLEx onto `WritingSystem`. Two optional fields: compiled ICU rules, or a .NET locale alias for "same as another language." No LDML at FwLite runtime. +_Avoid_: Storing LDML, simple-rule source text, or import metadata + +**Legacy collation fallback**: +When neither field is set, headword comparison uses the pre-existing FwLite behavior (`CultureInfo` from `WsId`, case-insensitive with lowercase-before-uppercase tie-break). FLEx "Default Ordering" is imported as this fallback too — FwLite does not replicate FLEx's empty-rule ICU default until we choose to. +_Avoid_: Treating `null` and `""` as different states for `IcuCollationRules` + +**Collation compare (imported)**: +When `IcuCollationRules` or `SystemCollationLocale` is set, use ICU4N (`RuleBasedCollator` / locale `Collator`) `Compare` with no legacy case tie-break layered on top. Matches FLEx for custom and other-language modes. FwData/FLEx bridge code still uses icu.net separately. +_Avoid_: Applying case-insensitive or lowercase-first logic on top of imported ICU4N collation + +**Collation import (from FLEx)**: +Populated when mapping `CoreWritingSystemDefinition` → `WritingSystem` in the FwData bridge. Custom modes: store non-empty compiled ICU rules only. Other-language: store .NET locale tag only. Default ordering: leave both fields null (legacy fallback). Import-only — no write-back to fwdata LDML. +_Avoid_: Parsing LDML in FwLite; persisting empty `IcuCollationRules` for default ordering + +**Collation write-back**: +Explicitly a no-op. The fwdata `UpdateWritingSystemProxy` overrides collation fields and does not write them to LDML. +_Avoid_: Assuming bidirectional collation sync + +## Commenting Collaborative discussion attached to dictionary data (entries, senses, example sentences). Comments sync via Harmony; read status is local-only per device (not synced, not per Lexbox user account). diff --git a/backend/FwLite/FwDataMiniLcmBridge/Api/FwDataMiniLcmApi.cs b/backend/FwLite/FwDataMiniLcmBridge/Api/FwDataMiniLcmApi.cs index 4d390f93f5..b24c9fa7f4 100644 --- a/backend/FwLite/FwDataMiniLcmBridge/Api/FwDataMiniLcmApi.cs +++ b/backend/FwLite/FwDataMiniLcmBridge/Api/FwDataMiniLcmApi.cs @@ -2,6 +2,7 @@ using System.Globalization; using System.Text; using FwDataMiniLcmBridge.Api.UpdateProxy; +using FwDataMiniLcmBridge.Collation; using FwDataMiniLcmBridge.LcmUtils; using FwDataMiniLcmBridge.Media; using Gridify; @@ -113,6 +114,7 @@ public Task GetWritingSystems() private WritingSystem FromLcmWritingSystem(CoreWritingSystemDefinition ws, WritingSystemType type, int index = default) { + var (icuCollationRules, systemCollationLocale) = WritingSystemCollationExtractor.Extract(ws); return new WritingSystem { Id = Guid.Empty, @@ -125,7 +127,9 @@ private WritingSystem FromLcmWritingSystem(CoreWritingSystemDefinition ws, Writi Name = ws.LanguageTag, Abbreviation = ws.Abbreviation, Font = ws.DefaultFontName, - Exemplars = ws.CharacterSets.FirstOrDefault(s => s.Type == "index")?.Characters.ToArray() ?? [] + Exemplars = ws.CharacterSets.FirstOrDefault(s => s.Type == "index")?.Characters.ToArray() ?? [], + IcuCollationRules = icuCollationRules, + SystemCollationLocale = systemCollationLocale, }; } diff --git a/backend/FwLite/FwDataMiniLcmBridge/Api/UpdateProxy/UpdateWritingSystemProxy.cs b/backend/FwLite/FwDataMiniLcmBridge/Api/UpdateProxy/UpdateWritingSystemProxy.cs index 857be0f458..4902f6e6b6 100644 --- a/backend/FwLite/FwDataMiniLcmBridge/Api/UpdateProxy/UpdateWritingSystemProxy.cs +++ b/backend/FwLite/FwDataMiniLcmBridge/Api/UpdateProxy/UpdateWritingSystemProxy.cs @@ -47,4 +47,17 @@ public override required string Font } } } + + // Collation is import-only from FLEx LDML; MiniLcm updates do not write back to fwdata. + public override string? IcuCollationRules + { + get => null; + set { } + } + + public override string? SystemCollationLocale + { + get => null; + set { } + } } diff --git a/backend/FwLite/FwDataMiniLcmBridge/Collation/WritingSystemCollationExtractor.cs b/backend/FwLite/FwDataMiniLcmBridge/Collation/WritingSystemCollationExtractor.cs new file mode 100644 index 0000000000..b105ea98f3 --- /dev/null +++ b/backend/FwLite/FwDataMiniLcmBridge/Collation/WritingSystemCollationExtractor.cs @@ -0,0 +1,21 @@ +using SIL.LCModel.Core.WritingSystems; +using SIL.WritingSystems; + +namespace FwDataMiniLcmBridge.Collation; + +public static class WritingSystemCollationExtractor +{ + public static (string? IcuCollationRules, string? SystemCollationLocale) Extract(CoreWritingSystemDefinition ws) + { + var cd = ws.DefaultCollation; + cd.Validate(out _); + + return cd switch + { + SystemCollationDefinition sys => (null, sys.LanguageTag), + RulesCollationDefinition rules when !string.IsNullOrEmpty(rules.CollationRules) + => (rules.CollationRules, null), + _ => (null, null) + }; + } +} diff --git a/backend/FwLite/FwLiteWeb/FwLiteWebServer.cs b/backend/FwLite/FwLiteWeb/FwLiteWebServer.cs index 29978b9dfa..a393fc17b4 100644 --- a/backend/FwLite/FwLiteWeb/FwLiteWebServer.cs +++ b/backend/FwLite/FwLiteWeb/FwLiteWebServer.cs @@ -24,11 +24,8 @@ public static WebApplication SetupAppServer(WebApplicationOptions options, Actio var builder = WebApplication.CreateBuilder(options); if (!builder.Environment.IsDevelopment() && options.Args?.Contains("--urls") != true && string.IsNullOrEmpty(builder.Configuration["http_ports"])) builder.WebHost.UseUrls("http://127.0.0.1:0"); - if (builder.Environment.IsDevelopment()) - { - //do this early so we catch bugs on startup - ProjectLoader.Init(); - } + // ICU required for FwData bridge (liblcm / icu.net), not for CRDT collation (ICU4N). + ProjectLoader.Init(); builder.ConfigureDev(config => config.LexboxServers = [ diff --git a/backend/FwLite/LcmCrdt.Tests/Changes/ChangeDeserializationRegressionData.latest.verified.txt b/backend/FwLite/LcmCrdt.Tests/Changes/ChangeDeserializationRegressionData.latest.verified.txt index b28168b1ba..5f2ad8eec7 100644 --- a/backend/FwLite/LcmCrdt.Tests/Changes/ChangeDeserializationRegressionData.latest.verified.txt +++ b/backend/FwLite/LcmCrdt.Tests/Changes/ChangeDeserializationRegressionData.latest.verified.txt @@ -756,10 +756,27 @@ "Exemplars": [ "Berkshire" ], + "IcuCollationRules": null, + "SystemCollationLocale": null, "Type": 1, "Order": 0.7988004089201804, "EntityId": "0fccfaaa-1e55-f6fb-966f-5c1e6caedf45" }, + { + "$type": "CreateWritingSystemChange", + "WsId": "prl", + "Name": "Investment Account", + "Abbreviation": "JBOD", + "Font": "SCSI", + "Exemplars": [ + "Games \u0026 Industrial" + ], + "IcuCollationRules": "parsing", + "SystemCollationLocale": "interactive", + "Type": 1, + "Order": 0.10446673530044903, + "EntityId": "f6928dbf-09ad-c0e8-a12a-23616641b071" + }, { "$type": "AddComplexFormTypeChange", "ComplexFormType": { diff --git a/backend/FwLite/LcmCrdt.Tests/Changes/ChangeDeserializationRegressionData.legacy.verified.txt b/backend/FwLite/LcmCrdt.Tests/Changes/ChangeDeserializationRegressionData.legacy.verified.txt index 19edc4c876..a20de6ece1 100644 --- a/backend/FwLite/LcmCrdt.Tests/Changes/ChangeDeserializationRegressionData.legacy.verified.txt +++ b/backend/FwLite/LcmCrdt.Tests/Changes/ChangeDeserializationRegressionData.legacy.verified.txt @@ -625,5 +625,65 @@ ], "EntityId": "f27a032b-e0f6-f389-b466-959bdb17086d" } + }, + { + "Input": { + "$type": "CreateWritingSystemChange", + "WsId": "zms", + "Name": "quantify", + "Abbreviation": "Ergonomic", + "Font": "software", + "Exemplars": [ + "Berkshire" + ], + "Type": 1, + "Order": 0.7988004089201804, + "EntityId": "0fccfaaa-1e55-f6fb-966f-5c1e6caedf45" + }, + "Output": { + "$type": "CreateWritingSystemChange", + "WsId": "zms", + "Name": "quantify", + "Abbreviation": "Ergonomic", + "Font": "software", + "Exemplars": [ + "Berkshire" + ], + "IcuCollationRules": null, + "SystemCollationLocale": null, + "Type": 1, + "Order": 0.7988004089201804, + "EntityId": "0fccfaaa-1e55-f6fb-966f-5c1e6caedf45" + } + }, + { + "Input": { + "$type": "CreateWritingSystemChange", + "WsId": "zms", + "Name": "quantify", + "Abbreviation": "Ergonomic", + "Font": "software", + "Exemplars": [ + "Berkshire" + ], + "Type": 1, + "Order": 0.7988004089201804, + "EntityId": "0fccfaaa-1e55-f6fb-966f-5c1e6caedf45" + }, + "Output": { + "$type": "CreateWritingSystemChange", + "WsId": "zms", + "Name": "quantify", + "Abbreviation": "Ergonomic", + "Font": "software", + "Exemplars": [ + "Berkshire" + ], + "IcuCollationRules": null, + "SystemCollationLocale": null, + "Type": 1, + "Order": 0.7988004089201804, + "EntityId": "0fccfaaa-1e55-f6fb-966f-5c1e6caedf45" + } } ] \ No newline at end of file diff --git a/backend/FwLite/LcmCrdt.Tests/Culture/WritingSystemCollatorProviderTests.cs b/backend/FwLite/LcmCrdt.Tests/Culture/WritingSystemCollatorProviderTests.cs new file mode 100644 index 0000000000..1984be1c08 --- /dev/null +++ b/backend/FwLite/LcmCrdt.Tests/Culture/WritingSystemCollatorProviderTests.cs @@ -0,0 +1,89 @@ +using LcmCrdt.Culture; +using Microsoft.Extensions.Caching.Memory; +using Microsoft.Extensions.DependencyInjection; +using MiniLcm.Culture; +using MiniLcm.Models; +using SIL.WritingSystems; + +namespace LcmCrdt.Tests.Culture; + +public class WritingSystemCollatorProviderTests +{ + private static IWritingSystemCollatorProvider CreateProvider() + { + var services = new ServiceCollection(); + services.AddLogging(); + services.AddMemoryCache(); + services.AddSingleton(); + services.AddSingleton(); + return services.BuildServiceProvider().GetRequiredService(); + } + + private static WritingSystem BaseWs() => new() + { + Id = Guid.NewGuid(), + WsId = "en", + Name = "English", + Abbreviation = "En", + Font = "Arial", + Type = WritingSystemType.Vernacular, + }; + + [Fact] + public void GetCollator_UsesIcu4NLocaleCollator_WhenLocaleSet() + { + var provider = CreateProvider(); + var collator = provider.GetCollator(BaseWs() with { SystemCollationLocale = "de" }); + collator.Should().BeOfType(); + } + + [Fact] + public void GetCollator_UsesIcu4NRulesCollator_WhenRulesSet() + { + var provider = CreateProvider(); + var collator = provider.GetCollator(BaseWs() with { IcuCollationRules = "&a < b" }); + collator.Should().BeOfType(); + } + + [Fact] + public void GetCollator_UsesLegacyFallback_WhenCollationUnset() + { + var provider = CreateProvider(); + var collator = provider.GetCollator(BaseWs()); + collator.Should().NotBeOfType(); + collator.Should().NotBeOfType(); + } + + [Fact] + public void Icu4NRulesCollator_SortsByCustomRules() + { + var provider = CreateProvider(); + var collator = provider.GetCollator(BaseWs() with { IcuCollationRules = "&z < a" }); + collator.Compare("z", "a").Should().BeLessThan(0); + } + + [Fact] + public void Icu4NRulesCollator_ReversesAAndB() + { + var provider = CreateProvider(); + var collator = provider.GetCollator(BaseWs() with { IcuCollationRules = "&b < a &B < A" }); + collator.Compare("Banane", "Apfel").Should().BeLessThan(0); + } + + [Fact] + public void LegacyCollator_PrefersLowercaseOnCaseInsensitiveTie() + { + var provider = CreateProvider(); + var collator = provider.GetCollator(BaseWs()); + collator.Compare("Ab", "ab").Should().BeGreaterThan(0); + } + + [Fact] + public void GetCollator_FallsBackToLegacy_WhenRulesInvalid() + { + var provider = CreateProvider(); + var collator = provider.GetCollator(BaseWs() with { IcuCollationRules = "not valid icu rules <<<<" }); + collator.Should().NotBeOfType(); + collator.Should().BeOfType(); + } +} diff --git a/backend/FwLite/LcmCrdt.Tests/Data/SnapshotDeserializationRegressionData.latest.verified.txt b/backend/FwLite/LcmCrdt.Tests/Data/SnapshotDeserializationRegressionData.latest.verified.txt index b7e45c5eb4..2390aa4435 100644 --- a/backend/FwLite/LcmCrdt.Tests/Data/SnapshotDeserializationRegressionData.latest.verified.txt +++ b/backend/FwLite/LcmCrdt.Tests/Data/SnapshotDeserializationRegressionData.latest.verified.txt @@ -3494,11 +3494,36 @@ "Y", "Z" ], + "IcuCollationRules": null, + "SystemCollationLocale": null, "Order": 0 }, "Id": "3ce0d5a6-5af0-4a08-8086-d0e827dfe2dd", "DeletedAt": null }, + { + "$type": "MiniLcmCrdtAdapter", + "Obj": { + "$type": "WritingSystem", + "Id": "3b85e4c8-6330-9167-2349-3bba70056346", + "MaybeId": "3b85e4c8-6330-9167-2349-3bba70056346", + "WsId": "syk", + "IsAudio": false, + "Name": "vortals", + "Abbreviation": "Granite", + "Font": "Iowa", + "DeletedAt": null, + "Type": 1, + "Exemplars": [ + "Incredible Metal Chair" + ], + "IcuCollationRules": "Customizable", + "SystemCollationLocale": "Borders", + "Order": 0.03765569767409571 + }, + "Id": "3b85e4c8-6330-9167-2349-3bba70056346", + "DeletedAt": null + }, { "$type": "MiniLcmCrdtAdapter", "Obj": { @@ -3515,11 +3540,36 @@ "Exemplars": [ "Planner" ], + "IcuCollationRules": null, + "SystemCollationLocale": null, "Order": 0.4320924489200455 }, "Id": "7cd09869-ce02-61b2-2dea-d3ed6e505109", "DeletedAt": null }, + { + "$type": "MiniLcmCrdtAdapter", + "Obj": { + "$type": "WritingSystem", + "Id": "9be8f583-63b4-cf64-4202-0e09481317c0", + "MaybeId": "9be8f583-63b4-cf64-4202-0e09481317c0", + "WsId": "enh", + "IsAudio": false, + "Name": "internet solution", + "Abbreviation": "Walk", + "Font": "payment", + "DeletedAt": null, + "Type": 0, + "Exemplars": [ + "Licensed Fresh Cheese" + ], + "IcuCollationRules": "connect", + "SystemCollationLocale": "Oregon", + "Order": 0.9633181157872764 + }, + "Id": "9be8f583-63b4-cf64-4202-0e09481317c0", + "DeletedAt": null + }, { "$type": "MiniLcmCrdtAdapter", "Obj": { @@ -3561,11 +3611,36 @@ "Y", "Z" ], + "IcuCollationRules": null, + "SystemCollationLocale": null, "Order": 0 }, "Id": "073c37cd-24cf-4d3d-ab33-19dfc94c96b7", "DeletedAt": null }, + { + "$type": "MiniLcmCrdtAdapter", + "Obj": { + "$type": "WritingSystem", + "Id": "3b9876d2-3b5d-a9f6-735d-7b6a8c232acd", + "MaybeId": "3b9876d2-3b5d-a9f6-735d-7b6a8c232acd", + "WsId": "ksh", + "IsAudio": false, + "Name": "Refined", + "Abbreviation": "Coordinator", + "Font": "indexing", + "DeletedAt": null, + "Type": 1, + "Exemplars": [ + "National" + ], + "IcuCollationRules": "Monitored", + "SystemCollationLocale": "bandwidth", + "Order": 0.2935114043495951 + }, + "Id": "3b9876d2-3b5d-a9f6-735d-7b6a8c232acd", + "DeletedAt": null + }, { "$type": "MiniLcmCrdtAdapter", "Obj": { @@ -3582,11 +3657,36 @@ "Exemplars": [ "blockchains" ], + "IcuCollationRules": null, + "SystemCollationLocale": null, "Order": 0.14806393624996905 }, "Id": "19023cf0-de25-9b0c-d5ad-94df308267c4", "DeletedAt": null }, + { + "$type": "MiniLcmCrdtAdapter", + "Obj": { + "$type": "WritingSystem", + "Id": "6e82adb2-46c1-7a00-e603-c18419f13d4f", + "MaybeId": "6e82adb2-46c1-7a00-e603-c18419f13d4f", + "WsId": "mnm", + "IsAudio": false, + "Name": "uniform", + "Abbreviation": "feed", + "Font": "Intelligent Plastic Chips", + "DeletedAt": null, + "Type": 1, + "Exemplars": [ + "Beauty, Games \u0026 Movies" + ], + "IcuCollationRules": "Assurance", + "SystemCollationLocale": "Kids \u0026 Electronics", + "Order": 0.34200726440404006 + }, + "Id": "6e82adb2-46c1-7a00-e603-c18419f13d4f", + "DeletedAt": null + }, { "$type": "MiniLcmCrdtAdapter", "Obj": { diff --git a/backend/FwLite/LcmCrdt.Tests/DataModelSnapshotTests.VerifyDbModel.verified.txt b/backend/FwLite/LcmCrdt.Tests/DataModelSnapshotTests.VerifyDbModel.verified.txt index 6eaccd9213..130f817d6a 100644 --- a/backend/FwLite/LcmCrdt.Tests/DataModelSnapshotTests.VerifyDbModel.verified.txt +++ b/backend/FwLite/LcmCrdt.Tests/DataModelSnapshotTests.VerifyDbModel.verified.txt @@ -353,9 +353,11 @@ Annotations: Relational:ColumnType: jsonb Font (string) Required + IcuCollationRules (string) Name (string) Required Order (double) Required SnapshotId (no field, Guid?) Shadow FK Index + SystemCollationLocale (string) Type (WritingSystemType) Required Index WsId (WritingSystemId) Required Index Keys: diff --git a/backend/FwLite/LcmCrdt.Tests/MiniLcmTests/CustomCollationSortingTests.cs b/backend/FwLite/LcmCrdt.Tests/MiniLcmTests/CustomCollationSortingTests.cs new file mode 100644 index 0000000000..f3c5d4245a --- /dev/null +++ b/backend/FwLite/LcmCrdt.Tests/MiniLcmTests/CustomCollationSortingTests.cs @@ -0,0 +1,61 @@ +namespace LcmCrdt.Tests.MiniLcmTests; + +public class CustomCollationSortingTests(MiniLcmApiFixture fixture) : IClassFixture +{ + [Fact] + public async Task HeadwordSort_UsesIcuCollationRules() + { + const string wsId = "en-x-icu-test"; + await fixture.Api.CreateWritingSystem(new() + { + Id = Guid.NewGuid(), + Type = WritingSystemType.Vernacular, + WsId = wsId, + Name = "Custom ICU", + Abbreviation = "Ci", + Font = "Arial", + IcuCollationRules = "&z < a", + }); + + var apple = await fixture.Api.CreateEntry(new() { LexemeForm = { { wsId, "apple" } } }); + var zebra = await fixture.Api.CreateEntry(new() { LexemeForm = { { wsId, "zebra" } } }); + var ids = new[] { apple.Id, zebra.Id }.ToHashSet(); + + var headwords = await fixture.Api + .GetEntries(new QueryOptions(new SortOptions(SortField.Headword, wsId))) + .Where(e => ids.Contains(e.Id)) + .Select(e => e.Headword()) + .ToArrayAsync(); + + headwords.Should().Equal("zebra", "apple"); + } + + [Fact] + public async Task HeadwordSort_UsesSystemCollationLocale() + { + const string wsId = "cs"; + await fixture.Api.CreateWritingSystem(new() + { + Id = Guid.NewGuid(), + Type = WritingSystemType.Vernacular, + WsId = wsId, + Name = "Czech", + Abbreviation = "Cs", + Font = "Arial", + SystemCollationLocale = "cs", + }); + + // English sorts c before h; Czech treats "ch" as a letter after h. + var cha = await fixture.Api.CreateEntry(new() { LexemeForm = { { wsId, "cha" } } }); + var ha = await fixture.Api.CreateEntry(new() { LexemeForm = { { wsId, "ha" } } }); + var ids = new[] { cha.Id, ha.Id }.ToHashSet(); + + var headwords = await fixture.Api + .GetEntries(new QueryOptions(new SortOptions(SortField.Headword, wsId))) + .Where(e => ids.Contains(e.Id)) + .Select(e => e.Headword()) + .ToArrayAsync(); + + headwords.Should().Equal("ha", "cha"); + } +} diff --git a/backend/FwLite/LcmCrdt.Tests/OpenProjectTests.cs b/backend/FwLite/LcmCrdt.Tests/OpenProjectTests.cs index a650a5f01a..703d4c362e 100644 --- a/backend/FwLite/LcmCrdt.Tests/OpenProjectTests.cs +++ b/backend/FwLite/LcmCrdt.Tests/OpenProjectTests.cs @@ -32,6 +32,7 @@ public async Task CanCreateExampleProject() .Contain(["de", "de-Zxxx-x-audio", "de-fonipa"]); writingSystems.Analysis.Select(ws => ws.WsId.Code).Should() .Contain(["en", "fr"]); + writingSystems.Vernacular.Single(ws => ws.WsId == "de").IcuCollationRules.Should().Be(ExampleProjectData.DemoIcuCollationRules); var entries = await api.GetEntries().ToArrayAsync(); entries.Select(e => e.LexemeForm["de"]).Should().Contain( @@ -41,6 +42,13 @@ public async Task CanCreateExampleProject() var complexForms = entries.Where(e => e.Components.Count > 0).ToArray(); complexForms.Select(e => e.LexemeForm["de"]).Should().Contain(["Erdbeere", "Heidelbeere"]); + var headwords = await api + .GetEntries(new QueryOptions(new SortOptions(SortField.Headword, "de"))) + .Select(e => e.LexemeForm["de"]) + .ToArrayAsync(); + headwords.Should().Equal( + "Banane", "Beere", "Apfel", "Erdbeere", "Heidelbeere", "Orange", "Traube"); + await using var dbContext = await asyncScope.ServiceProvider.GetRequiredService>().CreateDbContextAsync(); await dbContext.Database.EnsureDeletedAsync(); } diff --git a/backend/FwLite/LcmCrdt/Changes/CreateWritingSystemChange.cs b/backend/FwLite/LcmCrdt/Changes/CreateWritingSystemChange.cs index f2e11d1f75..8e5103f84d 100644 --- a/backend/FwLite/LcmCrdt/Changes/CreateWritingSystemChange.cs +++ b/backend/FwLite/LcmCrdt/Changes/CreateWritingSystemChange.cs @@ -15,6 +15,8 @@ public class CreateWritingSystemChange : CreateChange, ISelfNamed public required string Abbreviation { get; init; } public required string Font { get; init; } public required string[] Exemplars { get; init; } = []; + public string? IcuCollationRules { get; init; } + public string? SystemCollationLocale { get; init; } public required WritingSystemType Type { get; init; } public required double Order { get; init; } @@ -27,6 +29,8 @@ public CreateWritingSystemChange(WritingSystem writingSystem, Guid entityId, dou Abbreviation = writingSystem.Abbreviation; Font = writingSystem.Font; Exemplars = writingSystem.Exemplars; + IcuCollationRules = writingSystem.IcuCollationRules; + SystemCollationLocale = writingSystem.SystemCollationLocale; Type = writingSystem.Type; Order = order; } @@ -47,6 +51,8 @@ public override async ValueTask NewEntity(Commit commit, IChangeC Abbreviation = Abbreviation, Font = Font, Exemplars = Exemplars, + IcuCollationRules = IcuCollationRules, + SystemCollationLocale = SystemCollationLocale, Type = Type, Order = Order, DeletedAt = alreadyExists ? commit.DateTime : null diff --git a/backend/FwLite/LcmCrdt/CrdtProjectsService.cs b/backend/FwLite/LcmCrdt/CrdtProjectsService.cs index c4811e202e..d3692693b9 100644 --- a/backend/FwLite/LcmCrdt/CrdtProjectsService.cs +++ b/backend/FwLite/LcmCrdt/CrdtProjectsService.cs @@ -131,7 +131,10 @@ public record CreateProjectRequest( public async Task CreateExampleProject(string name) { // Code must satisfy the lowercase-only ProjectCode rule; the display name keeps its casing. - return await CreateProjectFromTemplate(new(name, name.ToLowerInvariant(), AfterCreate: ExampleProjectData.Seed, Role: UserProjectRole.Manager), vernacularWs: "de"); + return await CreateProjectFromTemplate( + new(name, name.ToLowerInvariant(), AfterCreate: ExampleProjectData.Seed, Role: UserProjectRole.Manager), + vernacularWs: "de", + configureVernacular: ws => ws with { IcuCollationRules = ExampleProjectData.DemoIcuCollationRules }); } /// @@ -144,7 +147,8 @@ public async Task CreateExampleProject(string name) public virtual async Task CreateProjectFromTemplate( CreateProjectRequest request, WritingSystemId vernacularWs, - WritingSystemId? analysisWs = null) + WritingSystemId? analysisWs = null, + Func? configureVernacular = null) { // Templated projects are created locally; this path has no way to associate one with a server // at creation time, so reject a Domain up front — the invariant holds where the project is born. @@ -160,7 +164,7 @@ public virtual async Task CreateProjectFromTemplate( { var api = provider.GetRequiredService(); var jsonOptions = provider.GetRequiredService>().Value.JsonSerializerOptions; - var snapshot = ProjectTemplate.CreateNewSnapshot(jsonOptions, vernacularWs, analysisWs); + var snapshot = ProjectTemplate.CreateNewSnapshot(jsonOptions, vernacularWs, analysisWs, configureVernacular); await projectImporter.ImportProject(api, snapshot); if (callerAfterCreate is not null) await callerAfterCreate(provider, project); } diff --git a/backend/FwLite/LcmCrdt/Culture/CultureCompareInfoCollator.cs b/backend/FwLite/LcmCrdt/Culture/CultureCompareInfoCollator.cs new file mode 100644 index 0000000000..eeb79e511f --- /dev/null +++ b/backend/FwLite/LcmCrdt/Culture/CultureCompareInfoCollator.cs @@ -0,0 +1,18 @@ +using System.Collections; +using System.Globalization; +using SIL.WritingSystems; + +namespace LcmCrdt.Culture; + +/// +/// Locale collation via .NET CompareInfo, without the legacy case tie-break. +/// +internal sealed class CultureCompareInfoCollator(CompareInfo compareInfo) : ICollator +{ + public int Compare(string? x, string? y) => + compareInfo.Compare(x ?? string.Empty, y ?? string.Empty, CompareOptions.None); + + public SortKey GetSortKey(string source) => compareInfo.GetSortKey(source); + + int IComparer.Compare(object? x, object? y) => Compare(x as string, y as string); +} diff --git a/backend/FwLite/LcmCrdt/Culture/Icu4NLocaleCollator.cs b/backend/FwLite/LcmCrdt/Culture/Icu4NLocaleCollator.cs new file mode 100644 index 0000000000..98d89bdd7c --- /dev/null +++ b/backend/FwLite/LcmCrdt/Culture/Icu4NLocaleCollator.cs @@ -0,0 +1,22 @@ +using System.Collections; +using System.Globalization; +using ICU4N.Text; +using SIL.WritingSystems; + +namespace LcmCrdt.Culture; + +internal sealed class Icu4NLocaleCollator : ICollator +{ + private readonly Collator _collator; + + public Icu4NLocaleCollator(string locale) => + _collator = Collator.GetInstance(CultureInfo.GetCultureInfo(locale)); + + public int Compare(string? x, string? y) => + _collator.Compare(x ?? string.Empty, y ?? string.Empty); + + public SortKey GetSortKey(string source) => + throw new NotSupportedException("ICU4N sort keys are not used by FwLite collation."); + + int IComparer.Compare(object? x, object? y) => Compare(x as string, y as string); +} diff --git a/backend/FwLite/LcmCrdt/Culture/Icu4NRulesCollator.cs b/backend/FwLite/LcmCrdt/Culture/Icu4NRulesCollator.cs new file mode 100644 index 0000000000..38df4cebc0 --- /dev/null +++ b/backend/FwLite/LcmCrdt/Culture/Icu4NRulesCollator.cs @@ -0,0 +1,21 @@ +using System.Collections; +using System.Globalization; +using ICU4N.Text; +using SIL.WritingSystems; + +namespace LcmCrdt.Culture; + +internal sealed class Icu4NRulesCollator : ICollator +{ + private readonly RuleBasedCollator _collator; + + public Icu4NRulesCollator(string rules) => _collator = new RuleBasedCollator(rules); + + public int Compare(string? x, string? y) => + _collator.Compare(x ?? string.Empty, y ?? string.Empty); + + public SortKey GetSortKey(string source) => + throw new NotSupportedException("ICU4N sort keys are not used by FwLite collation."); + + int IComparer.Compare(object? x, object? y) => Compare(x as string, y as string); +} diff --git a/backend/FwLite/LcmCrdt/Culture/LegacyCompareInfoCollator.cs b/backend/FwLite/LcmCrdt/Culture/LegacyCompareInfoCollator.cs new file mode 100644 index 0000000000..93c2f7dff8 --- /dev/null +++ b/backend/FwLite/LcmCrdt/Culture/LegacyCompareInfoCollator.cs @@ -0,0 +1,28 @@ +using System.Collections; +using System.Globalization; +using SIL.WritingSystems; + +namespace LcmCrdt.Culture; + +/// +/// Pre-collation-import fallback: case-insensitive compare with lowercase-before-uppercase tie-break. +/// +internal sealed class LegacyCompareInfoCollator(CompareInfo compareInfo) : ICollator +{ + public int Compare(string? x, string? y) + { + x ??= string.Empty; + y ??= string.Empty; + var caseInsensitiveResult = compareInfo.Compare(x, y, CompareOptions.IgnoreCase); + if (caseInsensitiveResult != 0) + { + return caseInsensitiveResult; + } + + return compareInfo.Compare(x, y, CompareOptions.None); + } + + public SortKey GetSortKey(string source) => compareInfo.GetSortKey(source); + + int IComparer.Compare(object? x, object? y) => Compare(x as string, y as string); +} diff --git a/backend/FwLite/LcmCrdt/Culture/WritingSystemCollatorProvider.cs b/backend/FwLite/LcmCrdt/Culture/WritingSystemCollatorProvider.cs new file mode 100644 index 0000000000..adc94359b2 --- /dev/null +++ b/backend/FwLite/LcmCrdt/Culture/WritingSystemCollatorProvider.cs @@ -0,0 +1,87 @@ +using System.Globalization; +using Microsoft.Extensions.Caching.Memory; +using Microsoft.Extensions.Logging; +using MiniLcm.Culture; +using MiniLcm.Models; +using SIL.WritingSystems; + +namespace LcmCrdt.Culture; + +public class WritingSystemCollatorProvider( + IMemoryCache cache, + IMiniLcmCultureProvider cultureProvider, + ILogger logger) : IWritingSystemCollatorProvider +{ + public ICollator GetCollator(WritingSystem writingSystem) + { + return cache.GetOrCreate(CacheKey(writingSystem), entry => + { + entry.SlidingExpiration = TimeSpan.FromHours(1); + return CreateCollator(writingSystem); + }) ?? CreateCollator(writingSystem); + } + + private ICollator CreateCollator(WritingSystem writingSystem) + { + if (!string.IsNullOrEmpty(writingSystem.SystemCollationLocale)) + { + return TryCreateLocaleCollator(writingSystem); + } + + if (!string.IsNullOrEmpty(writingSystem.IcuCollationRules)) + { + return TryCreateRulesCollator(writingSystem); + } + + return new LegacyCompareInfoCollator(cultureProvider.GetCompareInfo(writingSystem)); + } + + private ICollator TryCreateLocaleCollator(WritingSystem writingSystem) + { + try + { + return new Icu4NLocaleCollator(writingSystem.SystemCollationLocale!); + } + catch (Exception ex) + { + logger.LogWarning( + ex, + "Failed to create ICU4N locale collator for '{Locale}' on writing system '{WsId}'; using .NET collation fallback", + writingSystem.SystemCollationLocale, + writingSystem.WsId); + return CreateCultureCollator(writingSystem.SystemCollationLocale!); + } + } + + private ICollator TryCreateRulesCollator(WritingSystem writingSystem) + { + try + { + return new Icu4NRulesCollator(writingSystem.IcuCollationRules!); + } + catch (Exception ex) + { + logger.LogWarning( + ex, + "Failed to create ICU4N rules collator for writing system '{WsId}'; using legacy collation fallback", + writingSystem.WsId); + return new LegacyCompareInfoCollator(cultureProvider.GetCompareInfo(writingSystem)); + } + } + + private ICollator CreateCultureCollator(string locale) + { + try + { + return new CultureCompareInfoCollator(CultureInfo.GetCultureInfo(locale).CompareInfo); + } + catch (CultureNotFoundException ex) + { + logger.LogWarning(ex, "Unknown system collation locale '{Locale}'; using invariant collation", locale); + return new CultureCompareInfoCollator(CultureInfo.InvariantCulture.CompareInfo); + } + } + + private static string CacheKey(WritingSystem writingSystem) => + $"collator|{writingSystem.WsId}|{writingSystem.Type}|{writingSystem.IcuCollationRules}|{writingSystem.SystemCollationLocale}"; +} diff --git a/backend/FwLite/LcmCrdt/Data/SetupCollationInterceptor.cs b/backend/FwLite/LcmCrdt/Data/SetupCollationInterceptor.cs index de1a1c856e..9985440bb3 100644 --- a/backend/FwLite/LcmCrdt/Data/SetupCollationInterceptor.cs +++ b/backend/FwLite/LcmCrdt/Data/SetupCollationInterceptor.cs @@ -8,53 +8,63 @@ using Microsoft.EntityFrameworkCore.Diagnostics; using Microsoft.Extensions.Caching.Memory; using Microsoft.Extensions.Options; +using LcmCrdt.Culture; using MiniLcm.Culture; using SIL.Harmony; namespace LcmCrdt.Data; -public class SetupCollationInterceptor(IMemoryCache cache, IMiniLcmCultureProvider cultureProvider, IOptions crdtConfig) : IDbConnectionInterceptor, ISaveChangesInterceptor, IConnectionInterceptor +public class SetupCollationInterceptor( + IMemoryCache cache, + IMiniLcmCultureProvider cultureProvider, + IWritingSystemCollatorProvider collatorProvider, + IOptions crdtConfig) : IDbConnectionInterceptor, ISaveChangesInterceptor, IConnectionInterceptor { private static string? WsTableName = null; private WritingSystem[] GetWritingSystems(DbConnection connection, LcmCrdtDbContext? dbContext = null) { - return cache.GetOrCreate(CacheKey(connection), - entry => + var cacheKey = CacheKey(connection); + if (cache.TryGetValue(cacheKey, out WritingSystem[]? cached) && cached is not null) + { + return cached; + } + + try + { + var localContext = dbContext; + if (localContext is null) + { + var optionsBuilder = new DbContextOptionsBuilder(); + optionsBuilder.UseSqlite(connection); + localContext = new LcmCrdtDbContext(optionsBuilder.Options, crdtConfig); + } + + try { - entry.SlidingExpiration = TimeSpan.FromMinutes(30); - try + WsTableName ??= localContext.Model.FindRuntimeEntityType(typeof(WritingSystem))?.GetTableName() ?? "WritingSystem"; + if (!HasTable(localContext, WsTableName)) { - var localContext = dbContext; - if (localContext is null) - { - var optionsBuilder = new DbContextOptionsBuilder(); - optionsBuilder.UseSqlite(connection); - localContext = new LcmCrdtDbContext(optionsBuilder.Options, crdtConfig); - } - - try - { - WsTableName ??= localContext.Model.FindRuntimeEntityType(typeof(WritingSystem))?.GetTableName() ?? "WritingSystem"; - if (!HasTable(localContext, WsTableName)) - { - return []; - } - - return localContext.WritingSystems.ToArray(); - } - finally - { - if (dbContext is null) - { - localContext.Dispose(); - } - } + // Schema not migrated yet — don't cache so a later open can register collations. + return []; } - catch (SqliteException) + + var writingSystems = localContext.WritingSystems.ToArray(); + cache.Set(cacheKey, writingSystems, TimeSpan.FromMinutes(30)); + return writingSystems; + } + finally + { + if (dbContext is null) { - return []; + localContext.Dispose(); } - }) ?? []; + } + } + catch (SqliteException) + { + // Model/schema mismatch (e.g. connection opened mid-migration) — don't cache. + return []; + } } private bool HasTable(DbContext context, string tableName) @@ -186,9 +196,16 @@ private void SetupCollations(SqliteConnection connection, WritingSystem[] writin private void SetupCollation(SqliteConnection connection, WritingSystem writingSystem) { + if (HasImportedCollation(writingSystem)) + { + var collator = collatorProvider.GetCollator(writingSystem); + connection.CreateCollation(SqlSortingExtensions.CollationName(writingSystem.WsId), + (x, y) => collator.Compare(x, y)); + return; + } + var compareInfo = cultureProvider.GetCompareInfo(writingSystem); - //todo use custom comparison based on the writing system CreateSpanCollation(connection, SqlSortingExtensions.CollationName(writingSystem.WsId), compareInfo, static (compareInfo, x, y) => @@ -201,6 +218,10 @@ private void SetupCollation(SqliteConnection connection, WritingSystem writingSy }); } + private static bool HasImportedCollation(WritingSystem writingSystem) => + !string.IsNullOrEmpty(writingSystem.SystemCollationLocale) + || !string.IsNullOrEmpty(writingSystem.IcuCollationRules); + //this is a premature optimization, but it avoids creating strings for each comparison and instead uses spans which avoids allocations //if the new comparison function does not support spans then we can use SqliteConnection.CreateCollation instead which works with strings private void CreateSpanCollation(SqliteConnection connection, diff --git a/backend/FwLite/LcmCrdt/Data/Sorting.cs b/backend/FwLite/LcmCrdt/Data/Sorting.cs index 57822f2e99..65b8eef261 100644 --- a/backend/FwLite/LcmCrdt/Data/Sorting.cs +++ b/backend/FwLite/LcmCrdt/Data/Sorting.cs @@ -51,7 +51,7 @@ from mt in mtGroup.DefaultIfEmpty() !string.IsNullOrEmpty(query) && SqlHelpers.StartsWithIgnoreCaseAccents(e.HeadwordWithTokens(order.WritingSystem, mt.Prefix, mt.Postfix), query!) descending, !string.IsNullOrEmpty(query) && SqlHelpers.ContainsIgnoreCaseAccents(e.HeadwordWithTokens(order.WritingSystem, mt.Prefix, mt.Postfix), query!) descending, e.Headword(order.WritingSystem).Length, - e.Headword(order.WritingSystem), + e.Headword(order.WritingSystem).CollateUnicode(order.WritingSystem), mt != null ? mt.SecondaryOrder : stemOrder.FirstOrDefault(), e.HomographNumber, e.Id @@ -67,7 +67,7 @@ from mt in mtGroup.DefaultIfEmpty() !string.IsNullOrEmpty(query) && SqlHelpers.StartsWithIgnoreCaseAccents(e.HeadwordWithTokens(order.WritingSystem, mt.Prefix, mt.Postfix), query!), !string.IsNullOrEmpty(query) && SqlHelpers.ContainsIgnoreCaseAccents(e.HeadwordWithTokens(order.WritingSystem, mt.Prefix, mt.Postfix), query!), e.Headword(order.WritingSystem).Length descending, - e.Headword(order.WritingSystem) descending, + e.Headword(order.WritingSystem).CollateUnicode(order.WritingSystem) descending, (mt != null ? mt.SecondaryOrder : stemOrder.FirstOrDefault()) descending, e.HomographNumber descending, e.Id descending diff --git a/backend/FwLite/LcmCrdt/ExampleProjectData.cs b/backend/FwLite/LcmCrdt/ExampleProjectData.cs index 3035df9fce..9a0d369fb4 100644 --- a/backend/FwLite/LcmCrdt/ExampleProjectData.cs +++ b/backend/FwLite/LcmCrdt/ExampleProjectData.cs @@ -4,6 +4,9 @@ namespace LcmCrdt; internal static class ExampleProjectData { + /// ICU tailoring that sorts B before A so demo fruit headwords visibly differ from default order. + internal const string DemoIcuCollationRules = "&b < a &B < A"; + // Parts of speech and complex form types ship in the template (CreateProjectFromTemplate) with // liblcm's canonical Ids. We resolve the ones the demo needs (Noun, Compound) by name rather than // hard-coding those Ids, so this stays correct regardless of what the template assigns. @@ -20,8 +23,8 @@ public static async Task Seed(IServiceProvider provider, CrdtProject _) private static async Task CreateWritingSystems(IMiniLcmApi api) { - // The template path already provides vernacular "de" and analysis "en", so we only add the - // remaining demo writing systems here. They're appended after the template's, preserving the + // The template path already provides vernacular "de" (with demo ICU collation) and analysis "en". + // Add the remaining demo writing systems. They're appended after the template's, preserving the // order users see (de, de-audio, de-ipa / en, fr). await api.CreateWritingSystem(new() { diff --git a/backend/FwLite/LcmCrdt/LcmCrdt.csproj b/backend/FwLite/LcmCrdt/LcmCrdt.csproj index e5a86fa3ba..3c6e9fa324 100644 --- a/backend/FwLite/LcmCrdt/LcmCrdt.csproj +++ b/backend/FwLite/LcmCrdt/LcmCrdt.csproj @@ -24,6 +24,7 @@ + diff --git a/backend/FwLite/LcmCrdt/LcmCrdtKernel.cs b/backend/FwLite/LcmCrdt/LcmCrdtKernel.cs index 4aaa97f3b7..f48aeafe27 100644 --- a/backend/FwLite/LcmCrdt/LcmCrdtKernel.cs +++ b/backend/FwLite/LcmCrdt/LcmCrdtKernel.cs @@ -54,6 +54,7 @@ public static IServiceCollection AddLcmCrdtClientCore(this IServiceCollection se services.AddMemoryCache(); services.AddSingleton(); + services.AddSingleton(); services.AddSingleton(); services.AddScoped(); services.AddSingleton(); diff --git a/backend/FwLite/LcmCrdt/Migrations/20260702040259_AddWritingSystemCollation.Designer.cs b/backend/FwLite/LcmCrdt/Migrations/20260702040259_AddWritingSystemCollation.Designer.cs new file mode 100644 index 0000000000..245cdfba0f --- /dev/null +++ b/backend/FwLite/LcmCrdt/Migrations/20260702040259_AddWritingSystemCollation.Designer.cs @@ -0,0 +1,858 @@ +// +using System; +using System.Collections.Generic; +using LcmCrdt; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; + +#nullable disable + +namespace LcmCrdt.Migrations +{ + [DbContext(typeof(LcmCrdtDbContext))] + [Migration("20260702040259_AddWritingSystemCollation")] + partial class AddWritingSystemCollation + { + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder.HasAnnotation("ProductVersion", "10.0.8"); + + modelBuilder.Entity("LcmCrdt.FullTextSearch.EntrySearchRecord", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("TEXT"); + + b.Property("CitationForm") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("Definition") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("Gloss") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("Headword") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("LexemeForm") + .IsRequired() + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.ToTable("EntrySearchRecord", null, t => + { + t.ExcludeFromMigrations(); + }); + }); + + modelBuilder.Entity("LcmCrdt.ProjectData", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("TEXT"); + + b.Property("ClientId") + .HasColumnType("TEXT"); + + b.Property("Code") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("FwProjectId") + .HasColumnType("TEXT"); + + b.Property("LastUserId") + .HasColumnType("TEXT"); + + b.Property("LastUserName") + .HasColumnType("TEXT"); + + b.Property("Name") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("OriginDomain") + .HasColumnType("TEXT"); + + b.Property("Role") + .IsRequired() + .ValueGeneratedOnAdd() + .HasColumnType("TEXT") + .HasDefaultValue("Editor"); + + b.HasKey("Id"); + + b.ToTable("ProjectData"); + }); + + modelBuilder.Entity("MiniLcm.Models.ComplexFormComponent", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("TEXT"); + + b.Property("ComplexFormEntryId") + .HasColumnType("TEXT"); + + b.Property("ComplexFormHeadword") + .HasColumnType("TEXT"); + + b.Property("ComponentEntryId") + .HasColumnType("TEXT"); + + b.Property("ComponentHeadword") + .HasColumnType("TEXT"); + + b.Property("ComponentSenseId") + .HasColumnType("TEXT") + .HasColumnName("ComponentSenseId"); + + b.Property("DeletedAt") + .HasColumnType("TEXT"); + + b.Property("Order") + .HasColumnType("REAL"); + + b.Property("SnapshotId") + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.HasIndex("ComponentEntryId"); + + b.HasIndex("ComponentSenseId"); + + b.HasIndex("SnapshotId") + .IsUnique(); + + b.HasIndex("ComplexFormEntryId", "ComponentEntryId") + .IsUnique() + .HasFilter("ComponentSenseId IS NULL"); + + b.HasIndex("ComplexFormEntryId", "ComponentEntryId", "ComponentSenseId") + .IsUnique() + .HasFilter("ComponentSenseId IS NOT NULL"); + + b.ToTable("ComplexFormComponents", (string)null); + }); + + modelBuilder.Entity("MiniLcm.Models.ComplexFormType", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("TEXT"); + + b.Property("DeletedAt") + .HasColumnType("TEXT"); + + b.Property("Name") + .IsRequired() + .HasColumnType("jsonb"); + + b.Property("SnapshotId") + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.HasIndex("SnapshotId") + .IsUnique(); + + b.ToTable("ComplexFormType"); + }); + + modelBuilder.Entity("MiniLcm.Models.CustomView", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("TEXT"); + + b.Property("Analysis") + .HasColumnType("jsonb"); + + b.Property("Base") + .HasColumnType("INTEGER"); + + b.Property("DeletedAt") + .HasColumnType("TEXT"); + + b.Property("EntryFields") + .IsRequired() + .HasColumnType("jsonb"); + + b.Property("ExampleFields") + .IsRequired() + .HasColumnType("jsonb"); + + b.Property("Name") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("SenseFields") + .IsRequired() + .HasColumnType("jsonb"); + + b.Property("SnapshotId") + .HasColumnType("TEXT"); + + b.Property("Vernacular") + .HasColumnType("jsonb"); + + b.HasKey("Id"); + + b.HasIndex("SnapshotId") + .IsUnique(); + + b.ToTable("CustomView"); + }); + + modelBuilder.Entity("MiniLcm.Models.Entry", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("TEXT"); + + b.Property("CitationForm") + .IsRequired() + .HasColumnType("jsonb"); + + b.Property("ComplexFormTypes") + .IsRequired() + .HasColumnType("jsonb"); + + b.Property("DeletedAt") + .HasColumnType("TEXT"); + + b.Property("HomographNumber") + .HasColumnType("INTEGER"); + + b.Property("LexemeForm") + .IsRequired() + .HasColumnType("jsonb"); + + b.Property("LiteralMeaning") + .IsRequired() + .HasColumnType("jsonb"); + + b.Property("MorphType") + .HasColumnType("INTEGER"); + + b.Property("Note") + .IsRequired() + .HasColumnType("jsonb"); + + b.Property("PublishIn") + .IsRequired() + .HasColumnType("jsonb"); + + b.Property("SnapshotId") + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.HasIndex("SnapshotId") + .IsUnique(); + + b.ToTable("Entry"); + }); + + modelBuilder.Entity("MiniLcm.Models.ExampleSentence", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("TEXT"); + + b.Property("DeletedAt") + .HasColumnType("TEXT"); + + b.Property("Order") + .HasColumnType("REAL"); + + b.Property("Reference") + .HasColumnType("jsonb"); + + b.Property("SenseId") + .HasColumnType("TEXT"); + + b.Property("Sentence") + .IsRequired() + .HasColumnType("jsonb"); + + b.Property("SnapshotId") + .HasColumnType("TEXT"); + + b.Property("Translations") + .IsRequired() + .HasColumnType("jsonb"); + + b.HasKey("Id"); + + b.HasIndex("SenseId"); + + b.HasIndex("SnapshotId") + .IsUnique(); + + b.ToTable("ExampleSentence"); + }); + + modelBuilder.Entity("MiniLcm.Models.MorphType", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("TEXT"); + + b.Property("Abbreviation") + .IsRequired() + .HasColumnType("jsonb"); + + b.Property("DeletedAt") + .HasColumnType("TEXT"); + + b.Property("Description") + .IsRequired() + .HasColumnType("jsonb"); + + b.Property("Kind") + .HasColumnType("INTEGER"); + + b.Property("Name") + .IsRequired() + .HasColumnType("jsonb"); + + b.Property("Postfix") + .HasColumnType("TEXT"); + + b.Property("Prefix") + .HasColumnType("TEXT"); + + b.Property("SecondaryOrder") + .HasColumnType("INTEGER"); + + b.Property("SnapshotId") + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.HasIndex("Kind") + .IsUnique(); + + b.HasIndex("SnapshotId") + .IsUnique(); + + b.ToTable("MorphType"); + }); + + modelBuilder.Entity("MiniLcm.Models.PartOfSpeech", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("TEXT"); + + b.Property("DeletedAt") + .HasColumnType("TEXT"); + + b.Property("Name") + .IsRequired() + .HasColumnType("jsonb"); + + b.Property("Predefined") + .HasColumnType("INTEGER"); + + b.Property("SnapshotId") + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.HasIndex("SnapshotId") + .IsUnique(); + + b.ToTable("PartOfSpeech"); + }); + + modelBuilder.Entity("MiniLcm.Models.Publication", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("TEXT"); + + b.Property("DeletedAt") + .HasColumnType("TEXT"); + + b.Property("IsMain") + .HasColumnType("INTEGER"); + + b.Property("Name") + .IsRequired() + .HasColumnType("jsonb"); + + b.Property("SnapshotId") + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.HasIndex("SnapshotId") + .IsUnique(); + + b.ToTable("Publication"); + }); + + modelBuilder.Entity("MiniLcm.Models.SemanticDomain", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("TEXT"); + + b.Property("Code") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("DeletedAt") + .HasColumnType("TEXT"); + + b.Property("Name") + .IsRequired() + .HasColumnType("jsonb"); + + b.Property("Predefined") + .HasColumnType("INTEGER"); + + b.Property("SnapshotId") + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.HasIndex("SnapshotId") + .IsUnique(); + + b.ToTable("SemanticDomain"); + }); + + modelBuilder.Entity("MiniLcm.Models.Sense", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("TEXT"); + + b.Property("Definition") + .IsRequired() + .HasColumnType("jsonb"); + + b.Property("DeletedAt") + .HasColumnType("TEXT"); + + b.Property("EntryId") + .HasColumnType("TEXT"); + + b.Property("Gloss") + .IsRequired() + .HasColumnType("jsonb"); + + b.Property("Order") + .HasColumnType("REAL"); + + b.Property("PartOfSpeechId") + .HasColumnType("TEXT"); + + b.Property("Pictures") + .IsRequired() + .ValueGeneratedOnAdd() + .HasColumnType("jsonb") + .HasDefaultValueSql("'[]'"); + + b.Property("SemanticDomains") + .IsRequired() + .HasColumnType("jsonb"); + + b.Property("SnapshotId") + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.HasIndex("EntryId"); + + b.HasIndex("PartOfSpeechId"); + + b.HasIndex("SnapshotId") + .IsUnique(); + + b.ToTable("Sense"); + }); + + modelBuilder.Entity("MiniLcm.Models.WritingSystem", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("TEXT"); + + b.Property("Abbreviation") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("DeletedAt") + .HasColumnType("TEXT"); + + b.Property("Exemplars") + .IsRequired() + .HasColumnType("jsonb"); + + b.Property("Font") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("IcuCollationRules") + .HasColumnType("TEXT"); + + b.Property("Name") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("Order") + .HasColumnType("REAL"); + + b.Property("SnapshotId") + .HasColumnType("TEXT"); + + b.Property("SystemCollationLocale") + .HasColumnType("TEXT"); + + b.Property("Type") + .HasColumnType("INTEGER"); + + b.Property("WsId") + .IsRequired() + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.HasIndex("SnapshotId") + .IsUnique(); + + b.HasIndex("WsId", "Type") + .IsUnique(); + + b.ToTable("WritingSystem"); + }); + + modelBuilder.Entity("SIL.Harmony.Commit", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("TEXT"); + + b.Property("ClientId") + .HasColumnType("TEXT"); + + b.Property("Hash") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("Metadata") + .IsRequired() + .HasColumnType("jsonb"); + + b.Property("ParentHash") + .IsRequired() + .HasColumnType("TEXT"); + + b.ComplexProperty(typeof(Dictionary), "HybridDateTime", "SIL.Harmony.Commit.HybridDateTime#HybridDateTime", b1 => + { + b1.IsRequired(); + + b1.Property("Counter") + .HasColumnType("INTEGER") + .HasColumnName("Counter"); + + b1.Property("DateTime") + .HasColumnType("TEXT") + .HasColumnName("DateTime"); + }); + + b.HasKey("Id"); + + b.ToTable("Commits", (string)null); + + b.HasAnnotation("CustomIndex:CompositeIndexes", "[{\"paths\":[\"HybridDateTime.DateTime\",\"HybridDateTime.Counter\",\"Id\"],\"unique\":false,\"name\":\"IX_Commits_DateTime_Counter_Id\"}]"); + }); + + modelBuilder.Entity("SIL.Harmony.Core.ChangeEntity", b => + { + b.Property("CommitId") + .HasColumnType("TEXT"); + + b.Property("Index") + .HasColumnType("INTEGER"); + + b.Property("Change") + .HasColumnType("jsonb"); + + b.Property("EntityId") + .HasColumnType("TEXT"); + + b.HasKey("CommitId", "Index"); + + b.ToTable("ChangeEntities", (string)null); + }); + + modelBuilder.Entity("SIL.Harmony.Db.ObjectSnapshot", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("TEXT"); + + b.Property("CommitId") + .HasColumnType("TEXT"); + + b.Property("Entity") + .IsRequired() + .HasColumnType("jsonb"); + + b.Property("EntityId") + .HasColumnType("TEXT"); + + b.Property("EntityIsDeleted") + .HasColumnType("INTEGER"); + + b.Property("IsRoot") + .HasColumnType("INTEGER"); + + b.PrimitiveCollection("References") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("TypeName") + .IsRequired() + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.HasIndex("EntityId"); + + b.HasIndex("CommitId", "EntityId") + .IsUnique(); + + b.ToTable("Snapshots", (string)null); + }); + + modelBuilder.Entity("SIL.Harmony.Resource.LocalResource", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("TEXT"); + + b.Property("LocalPath") + .IsRequired() + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.ToTable("LocalResource"); + }); + + modelBuilder.Entity("SIL.Harmony.Resource.RemoteResource", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("TEXT"); + + b.Property("DeletedAt") + .HasColumnType("TEXT"); + + b.Property("RemoteId") + .HasColumnType("TEXT"); + + b.Property("SnapshotId") + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.HasIndex("SnapshotId") + .IsUnique(); + + b.ToTable("RemoteResource"); + }); + + modelBuilder.Entity("MiniLcm.Models.ComplexFormComponent", b => + { + b.HasOne("MiniLcm.Models.Entry", null) + .WithMany("Components") + .HasForeignKey("ComplexFormEntryId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("MiniLcm.Models.Entry", null) + .WithMany("ComplexForms") + .HasForeignKey("ComponentEntryId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("MiniLcm.Models.Sense", null) + .WithMany() + .HasForeignKey("ComponentSenseId") + .OnDelete(DeleteBehavior.Cascade); + + b.HasOne("SIL.Harmony.Db.ObjectSnapshot", null) + .WithOne() + .HasForeignKey("MiniLcm.Models.ComplexFormComponent", "SnapshotId") + .OnDelete(DeleteBehavior.SetNull); + }); + + modelBuilder.Entity("MiniLcm.Models.ComplexFormType", b => + { + b.HasOne("SIL.Harmony.Db.ObjectSnapshot", null) + .WithOne() + .HasForeignKey("MiniLcm.Models.ComplexFormType", "SnapshotId") + .OnDelete(DeleteBehavior.SetNull); + }); + + modelBuilder.Entity("MiniLcm.Models.CustomView", b => + { + b.HasOne("SIL.Harmony.Db.ObjectSnapshot", null) + .WithOne() + .HasForeignKey("MiniLcm.Models.CustomView", "SnapshotId") + .OnDelete(DeleteBehavior.SetNull); + }); + + modelBuilder.Entity("MiniLcm.Models.Entry", b => + { + b.HasOne("SIL.Harmony.Db.ObjectSnapshot", null) + .WithOne() + .HasForeignKey("MiniLcm.Models.Entry", "SnapshotId") + .OnDelete(DeleteBehavior.SetNull); + }); + + modelBuilder.Entity("MiniLcm.Models.ExampleSentence", b => + { + b.HasOne("MiniLcm.Models.Sense", null) + .WithMany("ExampleSentences") + .HasForeignKey("SenseId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("SIL.Harmony.Db.ObjectSnapshot", null) + .WithOne() + .HasForeignKey("MiniLcm.Models.ExampleSentence", "SnapshotId") + .OnDelete(DeleteBehavior.SetNull); + }); + + modelBuilder.Entity("MiniLcm.Models.MorphType", b => + { + b.HasOne("SIL.Harmony.Db.ObjectSnapshot", null) + .WithOne() + .HasForeignKey("MiniLcm.Models.MorphType", "SnapshotId") + .OnDelete(DeleteBehavior.SetNull); + }); + + modelBuilder.Entity("MiniLcm.Models.PartOfSpeech", b => + { + b.HasOne("SIL.Harmony.Db.ObjectSnapshot", null) + .WithOne() + .HasForeignKey("MiniLcm.Models.PartOfSpeech", "SnapshotId") + .OnDelete(DeleteBehavior.SetNull); + }); + + modelBuilder.Entity("MiniLcm.Models.Publication", b => + { + b.HasOne("SIL.Harmony.Db.ObjectSnapshot", null) + .WithOne() + .HasForeignKey("MiniLcm.Models.Publication", "SnapshotId") + .OnDelete(DeleteBehavior.SetNull); + }); + + modelBuilder.Entity("MiniLcm.Models.SemanticDomain", b => + { + b.HasOne("SIL.Harmony.Db.ObjectSnapshot", null) + .WithOne() + .HasForeignKey("MiniLcm.Models.SemanticDomain", "SnapshotId") + .OnDelete(DeleteBehavior.SetNull); + }); + + modelBuilder.Entity("MiniLcm.Models.Sense", b => + { + b.HasOne("MiniLcm.Models.Entry", null) + .WithMany("Senses") + .HasForeignKey("EntryId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("MiniLcm.Models.PartOfSpeech", "PartOfSpeech") + .WithMany() + .HasForeignKey("PartOfSpeechId") + .OnDelete(DeleteBehavior.SetNull); + + b.HasOne("SIL.Harmony.Db.ObjectSnapshot", null) + .WithOne() + .HasForeignKey("MiniLcm.Models.Sense", "SnapshotId") + .OnDelete(DeleteBehavior.SetNull); + + b.Navigation("PartOfSpeech"); + }); + + modelBuilder.Entity("MiniLcm.Models.WritingSystem", b => + { + b.HasOne("SIL.Harmony.Db.ObjectSnapshot", null) + .WithOne() + .HasForeignKey("MiniLcm.Models.WritingSystem", "SnapshotId") + .OnDelete(DeleteBehavior.SetNull); + }); + + modelBuilder.Entity("SIL.Harmony.Core.ChangeEntity", b => + { + b.HasOne("SIL.Harmony.Commit", null) + .WithMany("ChangeEntities") + .HasForeignKey("CommitId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("SIL.Harmony.Db.ObjectSnapshot", b => + { + b.HasOne("SIL.Harmony.Commit", "Commit") + .WithMany("Snapshots") + .HasForeignKey("CommitId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Commit"); + }); + + modelBuilder.Entity("SIL.Harmony.Resource.RemoteResource", b => + { + b.HasOne("SIL.Harmony.Db.ObjectSnapshot", null) + .WithOne() + .HasForeignKey("SIL.Harmony.Resource.RemoteResource", "SnapshotId") + .OnDelete(DeleteBehavior.SetNull); + }); + + modelBuilder.Entity("MiniLcm.Models.Entry", b => + { + b.Navigation("ComplexForms"); + + b.Navigation("Components"); + + b.Navigation("Senses"); + }); + + modelBuilder.Entity("MiniLcm.Models.Sense", b => + { + b.Navigation("ExampleSentences"); + }); + + modelBuilder.Entity("SIL.Harmony.Commit", b => + { + b.Navigation("ChangeEntities"); + + b.Navigation("Snapshots"); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/backend/FwLite/LcmCrdt/Migrations/20260702040259_AddWritingSystemCollation.cs b/backend/FwLite/LcmCrdt/Migrations/20260702040259_AddWritingSystemCollation.cs new file mode 100644 index 0000000000..d94cb92c31 --- /dev/null +++ b/backend/FwLite/LcmCrdt/Migrations/20260702040259_AddWritingSystemCollation.cs @@ -0,0 +1,38 @@ +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace LcmCrdt.Migrations +{ + /// + public partial class AddWritingSystemCollation : Migration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.AddColumn( + name: "IcuCollationRules", + table: "WritingSystem", + type: "TEXT", + nullable: true); + + migrationBuilder.AddColumn( + name: "SystemCollationLocale", + table: "WritingSystem", + type: "TEXT", + nullable: true); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropColumn( + name: "IcuCollationRules", + table: "WritingSystem"); + + migrationBuilder.DropColumn( + name: "SystemCollationLocale", + table: "WritingSystem"); + } + } +} diff --git a/backend/FwLite/LcmCrdt/Migrations/LcmCrdtDbContextModelSnapshot.cs b/backend/FwLite/LcmCrdt/Migrations/LcmCrdtDbContextModelSnapshot.cs index 650a5b9420..95ca317fd1 100644 --- a/backend/FwLite/LcmCrdt/Migrations/LcmCrdtDbContextModelSnapshot.cs +++ b/backend/FwLite/LcmCrdt/Migrations/LcmCrdtDbContextModelSnapshot.cs @@ -505,6 +505,9 @@ protected override void BuildModel(ModelBuilder modelBuilder) .IsRequired() .HasColumnType("TEXT"); + b.Property("IcuCollationRules") + .HasColumnType("TEXT"); + b.Property("Name") .IsRequired() .HasColumnType("TEXT"); @@ -515,6 +518,9 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.Property("SnapshotId") .HasColumnType("TEXT"); + b.Property("SystemCollationLocale") + .HasColumnType("TEXT"); + b.Property("Type") .HasColumnType("INTEGER"); diff --git a/backend/FwLite/LcmCrdt/Project/ProjectTemplate.cs b/backend/FwLite/LcmCrdt/Project/ProjectTemplate.cs index f99d24ddb4..f5ed42e42e 100644 --- a/backend/FwLite/LcmCrdt/Project/ProjectTemplate.cs +++ b/backend/FwLite/LcmCrdt/Project/ProjectTemplate.cs @@ -22,7 +22,8 @@ public static class ProjectTemplate public static ProjectSnapshot CreateNewSnapshot( JsonSerializerOptions jsonSerializerOptions, WritingSystemId vernacularWs, - WritingSystemId? analysisWs = null) + WritingSystemId? analysisWs = null, + Func? configureVernacular = null) { var assembly = typeof(ProjectTemplate).Assembly; using var stream = assembly.GetManifestResourceStream(EmbeddedResourceName) @@ -33,12 +34,19 @@ public static ProjectSnapshot CreateNewSnapshot( ?? throw new InvalidOperationException("Project template snapshot deserialized to null."); // Merge the requested writing systems into the snapshot so they're created with everything else // in dependency order, rather than tacked on after the import. - return snapshot with { WritingSystems = WithRequestedWritingSystems(snapshot.WritingSystems, vernacularWs, analysisWs) }; + return snapshot with { WritingSystems = WithRequestedWritingSystems(snapshot.WritingSystems, vernacularWs, analysisWs, configureVernacular) }; } - private static WritingSystems WithRequestedWritingSystems(WritingSystems template, WritingSystemId vernacularWs, WritingSystemId? analysisWs) + private static WritingSystems WithRequestedWritingSystems( + WritingSystems template, + WritingSystemId vernacularWs, + WritingSystemId? analysisWs, + Func? configureVernacular = null) { - WritingSystem[] vernacular = [.. template.Vernacular, DefaultWritingSystem(vernacularWs, WritingSystemType.Vernacular)]; + var vernacularWsEntity = DefaultWritingSystem(vernacularWs, WritingSystemType.Vernacular); + if (configureVernacular is not null) + vernacularWsEntity = configureVernacular(vernacularWsEntity); + WritingSystem[] vernacular = [.. template.Vernacular, vernacularWsEntity]; var analysis = template.Analysis; // The template already ships English analysis; only add the requested analysis WS if it's a different one. if (analysisWs is { } aws && !analysis.Any(ws => ws.WsId == aws)) diff --git a/backend/FwLite/MiniLcm.Tests/WritingSystemSyncTests.cs b/backend/FwLite/MiniLcm.Tests/WritingSystemSyncTests.cs new file mode 100644 index 0000000000..3b670a5bc4 --- /dev/null +++ b/backend/FwLite/MiniLcm.Tests/WritingSystemSyncTests.cs @@ -0,0 +1,54 @@ +using MiniLcm.Models; +using MiniLcm.SyncHelpers; + +namespace MiniLcm.Tests; + +public class WritingSystemSyncTests +{ + private static WritingSystem BaseWs() => new() + { + Id = Guid.NewGuid(), + WsId = "en", + Name = "English", + Abbreviation = "En", + Font = "Arial", + Type = WritingSystemType.Vernacular, + }; + + [Fact] + public void DiffToUpdate_IncludesIcuCollationRules() + { + var before = BaseWs(); + var after = before with { IcuCollationRules = "&a < b" }; + + var update = WritingSystemSync.WritingSystemDiffToUpdate(before, after); + + update.Should().NotBeNull(); + var op = update!.Patch.Operations.Single(o => o.Path == $"/{nameof(WritingSystem.IcuCollationRules)}"); + op.Value?.ToString().Should().Be("&a < b"); + } + + [Fact] + public void DiffToUpdate_IncludesSystemCollationLocale() + { + var before = BaseWs(); + var after = before with { SystemCollationLocale = "de" }; + + var update = WritingSystemSync.WritingSystemDiffToUpdate(before, after); + + update.Should().NotBeNull(); + var op = update!.Patch.Operations.Single(o => o.Path == $"/{nameof(WritingSystem.SystemCollationLocale)}"); + op.Value?.ToString().Should().Be("de"); + } + + [Fact] + public void DiffToUpdate_NoOpsWhenCollationUnchanged() + { + var before = BaseWs() with { IcuCollationRules = "&a < b" }; + var after = before with { }; + + var update = WritingSystemSync.WritingSystemDiffToUpdate(before, after); + + update.Should().BeNull(); + } +} diff --git a/backend/FwLite/MiniLcm/Culture/IWritingSystemCollatorProvider.cs b/backend/FwLite/MiniLcm/Culture/IWritingSystemCollatorProvider.cs new file mode 100644 index 0000000000..4c3139455b --- /dev/null +++ b/backend/FwLite/MiniLcm/Culture/IWritingSystemCollatorProvider.cs @@ -0,0 +1,9 @@ +using MiniLcm.Models; +using SIL.WritingSystems; + +namespace MiniLcm.Culture; + +public interface IWritingSystemCollatorProvider +{ + ICollator GetCollator(WritingSystem writingSystem); +} diff --git a/backend/FwLite/MiniLcm/Models/WritingSystem.cs b/backend/FwLite/MiniLcm/Models/WritingSystem.cs index b9eb655a9b..f5b3fcef2a 100644 --- a/backend/FwLite/MiniLcm/Models/WritingSystem.cs +++ b/backend/FwLite/MiniLcm/Models/WritingSystem.cs @@ -35,6 +35,17 @@ public required Guid Id public string[] Exemplars { get; set; } = []; //todo probably need more stuff here, see wesay for ideas + /// + /// Compiled ICU tailoring rules imported from FLEx (Custom Simple / Custom ICU). + /// Null uses legacy collation fallback at runtime. Not persisted as empty on import. + /// + public virtual string? IcuCollationRules { get; set; } + + /// + /// .NET locale tag for FLEx "same as another language" collation. + /// + public virtual string? SystemCollationLocale { get; set; } + public static string[] LatinExemplars => Enumerable.Range('A', 'Z' - 'A' + 1).Select(c => ((char)c).ToString()).ToArray(); [MiniLcmInternal] @@ -59,6 +70,8 @@ public WritingSystem Copy() Abbreviation = Abbreviation, Font = Font, Exemplars = [..Exemplars], + IcuCollationRules = IcuCollationRules, + SystemCollationLocale = SystemCollationLocale, DeletedAt = DeletedAt, Type = Type, Order = Order diff --git a/backend/FwLite/MiniLcm/SyncHelpers/WritingSystemSync.cs b/backend/FwLite/MiniLcm/SyncHelpers/WritingSystemSync.cs index cb58ec4e48..5f998055f2 100644 --- a/backend/FwLite/MiniLcm/SyncHelpers/WritingSystemSync.cs +++ b/backend/FwLite/MiniLcm/SyncHelpers/WritingSystemSync.cs @@ -44,6 +44,12 @@ public static async Task Sync(WritingSystem beforeWs, WritingSystem afterWs patchDocument.Operations.AddRange(SimpleStringDiff.GetStringDiff(nameof(WritingSystem.Font), beforeWritingSystem.Font, afterWritingSystem.Font)); + patchDocument.Operations.AddRange(SimpleStringDiff.GetStringDiff(nameof(WritingSystem.IcuCollationRules), + beforeWritingSystem.IcuCollationRules, + afterWritingSystem.IcuCollationRules)); + patchDocument.Operations.AddRange(SimpleStringDiff.GetStringDiff(nameof(WritingSystem.SystemCollationLocale), + beforeWritingSystem.SystemCollationLocale, + afterWritingSystem.SystemCollationLocale)); // TODO: Exemplars, Order, and do we need DeletedAt? if (patchDocument.Operations.Count == 0) return null; return new UpdateObjectInput(patchDocument); diff --git a/frontend/viewer/src/lib/dotnet-types/generated-types/MiniLcm/Models/IWritingSystem.ts b/frontend/viewer/src/lib/dotnet-types/generated-types/MiniLcm/Models/IWritingSystem.ts index 2e46bd6f9c..21824476a3 100644 --- a/frontend/viewer/src/lib/dotnet-types/generated-types/MiniLcm/Models/IWritingSystem.ts +++ b/frontend/viewer/src/lib/dotnet-types/generated-types/MiniLcm/Models/IWritingSystem.ts @@ -16,5 +16,7 @@ export interface IWritingSystem extends IObjectWithId deletedAt?: string; type: WritingSystemType; exemplars: string[]; + icuCollationRules?: string; + systemCollationLocale?: string; } /* eslint-enable */