From 713228e1ced0841d2473bc14482bfb717b5219c3 Mon Sep 17 00:00:00 2001 From: Kevin Hahn Date: Thu, 2 Jul 2026 11:43:14 +0700 Subject: [PATCH 1/3] Store writing system collation on import from FLEx. Persist compiled ICU rules and system locale on WritingSystem, extract them when mapping fwdata, sync into the CRDT, and add an EF migration. Collation is import-only via the fwdata update proxy; fix collation cache poisoning during mid-migration schema upgrades. Co-authored-by: Cursor --- CONTEXT-MAP.md | 2 +- backend/FwLite/CONTEXT.md | 44 +- .../Api/FwDataMiniLcmApi.cs | 6 +- .../UpdateProxy/UpdateWritingSystemProxy.cs | 13 + .../WritingSystemCollationExtractor.cs | 21 + ...lizationRegressionData.latest.verified.txt | 17 + ...lizationRegressionData.legacy.verified.txt | 60 ++ ...lizationRegressionData.latest.verified.txt | 100 ++ ...elSnapshotTests.VerifyDbModel.verified.txt | 2 + .../Changes/CreateWritingSystemChange.cs | 6 + .../LcmCrdt/Data/SetupCollationInterceptor.cs | 69 +- ...0259_AddWritingSystemCollation.Designer.cs | 858 ++++++++++++++++++ ...0260702040259_AddWritingSystemCollation.cs | 38 + .../LcmCrdtDbContextModelSnapshot.cs | 6 + .../MiniLcm.Tests/WritingSystemSyncTests.cs | 54 ++ .../FwLite/MiniLcm/Models/WritingSystem.cs | 13 + .../MiniLcm/SyncHelpers/WritingSystemSync.cs | 6 + .../MiniLcm/Models/IWritingSystem.ts | 2 + 18 files changed, 1282 insertions(+), 35 deletions(-) create mode 100644 backend/FwLite/FwDataMiniLcmBridge/Collation/WritingSystemCollationExtractor.cs create mode 100644 backend/FwLite/LcmCrdt/Migrations/20260702040259_AddWritingSystemCollation.Designer.cs create mode 100644 backend/FwLite/LcmCrdt/Migrations/20260702040259_AddWritingSystemCollation.cs create mode 100644 backend/FwLite/MiniLcm.Tests/WritingSystemSyncTests.cs diff --git a/CONTEXT-MAP.md b/CONTEXT-MAP.md index b9b94c0c41..181eb5568d 100644 --- a/CONTEXT-MAP.md +++ b/CONTEXT-MAP.md @@ -4,4 +4,4 @@ Domain glossaries for this monorepo. Each `CONTEXT.md` is a glossary only — no ## Contexts -- [FwLite Commenting](./backend/FwLite/CONTEXT.md) — collaborative comments on dictionary entries, senses, and example sentences +- [FwLite](./backend/FwLite/CONTEXT.md) — dictionary editing concerns including collation and collaborative comments diff --git a/backend/FwLite/CONTEXT.md b/backend/FwLite/CONTEXT.md index b414b42d0b..e749ce64ba 100644 --- a/backend/FwLite/CONTEXT.md +++ b/backend/FwLite/CONTEXT.md @@ -1,4 +1,46 @@ -# FwLite Commenting +# FwLite + +## Collation + +How strings in a writing system are compared when ordering dictionary content. Imported from FLEx per writing system; stored on `WritingSystem` without LDML at runtime. + +**Collation**: +The rules that govern string comparison for a writing system (ICU tailoring or .NET locale alias). Distinct from list position and from the user action of sorting entries. +_Avoid_: Sort rules (when meaning collation specifically), custom sorting (as a model term) + +**Writing system order**: +Where a writing system appears in the project's vernacular or analysis list (`WritingSystem.Order`). Not how strings alphabetize. +_Avoid_: Confusing with collation + +**Entry sort**: +The user-facing action of ordering the entry list (e.g. by headword in a chosen writing system). Uses the selected writing system's collation for headword comparison. +_Avoid_: Collation (when meaning the user action) + +**Collation scope (v1)**: +Collation governs headword entry sort (`SortField.Headword`) and the alphabetical tie-break when search results fall through to headword order (`SortField.SearchRelevance`). It does not govern search matching (whether a query matches text) or FTS ranking. +_Avoid_: Applying collation to `ContainsDiacriticMatch`, prefix/contains predicates, or FTS rank + +**Imported collation**: +Collation metadata synced from FLEx onto `WritingSystem`. Two optional fields: compiled ICU rules, or a .NET locale alias for "same as another language." No LDML at FwLite runtime. +_Avoid_: Storing LDML, simple-rule source text, or import metadata + +**Legacy collation fallback**: +When neither field is set, headword comparison uses the pre-existing FwLite behavior (`CultureInfo` from `WsId`, case-insensitive with lowercase-before-uppercase tie-break). FLEx "Default Ordering" is imported as this fallback too — FwLite does not replicate FLEx's empty-rule ICU default until we choose to. +_Avoid_: Treating `null` and `""` as different states for `IcuCollationRules` + +**Collation compare (imported)**: +When `IcuCollationRules` or `SystemCollationLocale` is set, use the libpalaso collator's native `Compare` with no legacy case tie-break layered on top. Matches FLEx for custom and other-language modes. +_Avoid_: Applying case-insensitive or lowercase-first logic on top of `IcuRulesCollator` or `SystemCollator` + +**Collation import (from FLEx)**: +Populated when mapping `CoreWritingSystemDefinition` → `WritingSystem` in the FwData bridge. Custom modes: store non-empty compiled ICU rules only. Other-language: store .NET locale tag only. Default ordering: leave both fields null (legacy fallback). Import-only — no write-back to fwdata LDML. +_Avoid_: Parsing LDML in FwLite; persisting empty `IcuCollationRules` for default ordering + +**Collation write-back**: +Explicitly a no-op. The fwdata `UpdateWritingSystemProxy` overrides collation fields and does not write them to LDML. +_Avoid_: Assuming bidirectional collation sync + +## Commenting Collaborative discussion attached to dictionary data (entries, senses, example sentences). Comments sync via Harmony; read status is local-only per device (not synced, not per Lexbox user account). diff --git a/backend/FwLite/FwDataMiniLcmBridge/Api/FwDataMiniLcmApi.cs b/backend/FwLite/FwDataMiniLcmBridge/Api/FwDataMiniLcmApi.cs index 4d390f93f5..b24c9fa7f4 100644 --- a/backend/FwLite/FwDataMiniLcmBridge/Api/FwDataMiniLcmApi.cs +++ b/backend/FwLite/FwDataMiniLcmBridge/Api/FwDataMiniLcmApi.cs @@ -2,6 +2,7 @@ using System.Globalization; using System.Text; using FwDataMiniLcmBridge.Api.UpdateProxy; +using FwDataMiniLcmBridge.Collation; using FwDataMiniLcmBridge.LcmUtils; using FwDataMiniLcmBridge.Media; using Gridify; @@ -113,6 +114,7 @@ public Task GetWritingSystems() private WritingSystem FromLcmWritingSystem(CoreWritingSystemDefinition ws, WritingSystemType type, int index = default) { + var (icuCollationRules, systemCollationLocale) = WritingSystemCollationExtractor.Extract(ws); return new WritingSystem { Id = Guid.Empty, @@ -125,7 +127,9 @@ private WritingSystem FromLcmWritingSystem(CoreWritingSystemDefinition ws, Writi Name = ws.LanguageTag, Abbreviation = ws.Abbreviation, Font = ws.DefaultFontName, - Exemplars = ws.CharacterSets.FirstOrDefault(s => s.Type == "index")?.Characters.ToArray() ?? [] + Exemplars = ws.CharacterSets.FirstOrDefault(s => s.Type == "index")?.Characters.ToArray() ?? [], + IcuCollationRules = icuCollationRules, + SystemCollationLocale = systemCollationLocale, }; } diff --git a/backend/FwLite/FwDataMiniLcmBridge/Api/UpdateProxy/UpdateWritingSystemProxy.cs b/backend/FwLite/FwDataMiniLcmBridge/Api/UpdateProxy/UpdateWritingSystemProxy.cs index 857be0f458..4902f6e6b6 100644 --- a/backend/FwLite/FwDataMiniLcmBridge/Api/UpdateProxy/UpdateWritingSystemProxy.cs +++ b/backend/FwLite/FwDataMiniLcmBridge/Api/UpdateProxy/UpdateWritingSystemProxy.cs @@ -47,4 +47,17 @@ public override required string Font } } } + + // Collation is import-only from FLEx LDML; MiniLcm updates do not write back to fwdata. + public override string? IcuCollationRules + { + get => null; + set { } + } + + public override string? SystemCollationLocale + { + get => null; + set { } + } } diff --git a/backend/FwLite/FwDataMiniLcmBridge/Collation/WritingSystemCollationExtractor.cs b/backend/FwLite/FwDataMiniLcmBridge/Collation/WritingSystemCollationExtractor.cs new file mode 100644 index 0000000000..b105ea98f3 --- /dev/null +++ b/backend/FwLite/FwDataMiniLcmBridge/Collation/WritingSystemCollationExtractor.cs @@ -0,0 +1,21 @@ +using SIL.LCModel.Core.WritingSystems; +using SIL.WritingSystems; + +namespace FwDataMiniLcmBridge.Collation; + +public static class WritingSystemCollationExtractor +{ + public static (string? IcuCollationRules, string? SystemCollationLocale) Extract(CoreWritingSystemDefinition ws) + { + var cd = ws.DefaultCollation; + cd.Validate(out _); + + return cd switch + { + SystemCollationDefinition sys => (null, sys.LanguageTag), + RulesCollationDefinition rules when !string.IsNullOrEmpty(rules.CollationRules) + => (rules.CollationRules, null), + _ => (null, null) + }; + } +} diff --git a/backend/FwLite/LcmCrdt.Tests/Changes/ChangeDeserializationRegressionData.latest.verified.txt b/backend/FwLite/LcmCrdt.Tests/Changes/ChangeDeserializationRegressionData.latest.verified.txt index b28168b1ba..5f2ad8eec7 100644 --- a/backend/FwLite/LcmCrdt.Tests/Changes/ChangeDeserializationRegressionData.latest.verified.txt +++ b/backend/FwLite/LcmCrdt.Tests/Changes/ChangeDeserializationRegressionData.latest.verified.txt @@ -756,10 +756,27 @@ "Exemplars": [ "Berkshire" ], + "IcuCollationRules": null, + "SystemCollationLocale": null, "Type": 1, "Order": 0.7988004089201804, "EntityId": "0fccfaaa-1e55-f6fb-966f-5c1e6caedf45" }, + { + "$type": "CreateWritingSystemChange", + "WsId": "prl", + "Name": "Investment Account", + "Abbreviation": "JBOD", + "Font": "SCSI", + "Exemplars": [ + "Games \u0026 Industrial" + ], + "IcuCollationRules": "parsing", + "SystemCollationLocale": "interactive", + "Type": 1, + "Order": 0.10446673530044903, + "EntityId": "f6928dbf-09ad-c0e8-a12a-23616641b071" + }, { "$type": "AddComplexFormTypeChange", "ComplexFormType": { diff --git a/backend/FwLite/LcmCrdt.Tests/Changes/ChangeDeserializationRegressionData.legacy.verified.txt b/backend/FwLite/LcmCrdt.Tests/Changes/ChangeDeserializationRegressionData.legacy.verified.txt index 19edc4c876..a20de6ece1 100644 --- a/backend/FwLite/LcmCrdt.Tests/Changes/ChangeDeserializationRegressionData.legacy.verified.txt +++ b/backend/FwLite/LcmCrdt.Tests/Changes/ChangeDeserializationRegressionData.legacy.verified.txt @@ -625,5 +625,65 @@ ], "EntityId": "f27a032b-e0f6-f389-b466-959bdb17086d" } + }, + { + "Input": { + "$type": "CreateWritingSystemChange", + "WsId": "zms", + "Name": "quantify", + "Abbreviation": "Ergonomic", + "Font": "software", + "Exemplars": [ + "Berkshire" + ], + "Type": 1, + "Order": 0.7988004089201804, + "EntityId": "0fccfaaa-1e55-f6fb-966f-5c1e6caedf45" + }, + "Output": { + "$type": "CreateWritingSystemChange", + "WsId": "zms", + "Name": "quantify", + "Abbreviation": "Ergonomic", + "Font": "software", + "Exemplars": [ + "Berkshire" + ], + "IcuCollationRules": null, + "SystemCollationLocale": null, + "Type": 1, + "Order": 0.7988004089201804, + "EntityId": "0fccfaaa-1e55-f6fb-966f-5c1e6caedf45" + } + }, + { + "Input": { + "$type": "CreateWritingSystemChange", + "WsId": "zms", + "Name": "quantify", + "Abbreviation": "Ergonomic", + "Font": "software", + "Exemplars": [ + "Berkshire" + ], + "Type": 1, + "Order": 0.7988004089201804, + "EntityId": "0fccfaaa-1e55-f6fb-966f-5c1e6caedf45" + }, + "Output": { + "$type": "CreateWritingSystemChange", + "WsId": "zms", + "Name": "quantify", + "Abbreviation": "Ergonomic", + "Font": "software", + "Exemplars": [ + "Berkshire" + ], + "IcuCollationRules": null, + "SystemCollationLocale": null, + "Type": 1, + "Order": 0.7988004089201804, + "EntityId": "0fccfaaa-1e55-f6fb-966f-5c1e6caedf45" + } } ] \ No newline at end of file diff --git a/backend/FwLite/LcmCrdt.Tests/Data/SnapshotDeserializationRegressionData.latest.verified.txt b/backend/FwLite/LcmCrdt.Tests/Data/SnapshotDeserializationRegressionData.latest.verified.txt index b7e45c5eb4..2390aa4435 100644 --- a/backend/FwLite/LcmCrdt.Tests/Data/SnapshotDeserializationRegressionData.latest.verified.txt +++ b/backend/FwLite/LcmCrdt.Tests/Data/SnapshotDeserializationRegressionData.latest.verified.txt @@ -3494,11 +3494,36 @@ "Y", "Z" ], + "IcuCollationRules": null, + "SystemCollationLocale": null, "Order": 0 }, "Id": "3ce0d5a6-5af0-4a08-8086-d0e827dfe2dd", "DeletedAt": null }, + { + "$type": "MiniLcmCrdtAdapter", + "Obj": { + "$type": "WritingSystem", + "Id": "3b85e4c8-6330-9167-2349-3bba70056346", + "MaybeId": "3b85e4c8-6330-9167-2349-3bba70056346", + "WsId": "syk", + "IsAudio": false, + "Name": "vortals", + "Abbreviation": "Granite", + "Font": "Iowa", + "DeletedAt": null, + "Type": 1, + "Exemplars": [ + "Incredible Metal Chair" + ], + "IcuCollationRules": "Customizable", + "SystemCollationLocale": "Borders", + "Order": 0.03765569767409571 + }, + "Id": "3b85e4c8-6330-9167-2349-3bba70056346", + "DeletedAt": null + }, { "$type": "MiniLcmCrdtAdapter", "Obj": { @@ -3515,11 +3540,36 @@ "Exemplars": [ "Planner" ], + "IcuCollationRules": null, + "SystemCollationLocale": null, "Order": 0.4320924489200455 }, "Id": "7cd09869-ce02-61b2-2dea-d3ed6e505109", "DeletedAt": null }, + { + "$type": "MiniLcmCrdtAdapter", + "Obj": { + "$type": "WritingSystem", + "Id": "9be8f583-63b4-cf64-4202-0e09481317c0", + "MaybeId": "9be8f583-63b4-cf64-4202-0e09481317c0", + "WsId": "enh", + "IsAudio": false, + "Name": "internet solution", + "Abbreviation": "Walk", + "Font": "payment", + "DeletedAt": null, + "Type": 0, + "Exemplars": [ + "Licensed Fresh Cheese" + ], + "IcuCollationRules": "connect", + "SystemCollationLocale": "Oregon", + "Order": 0.9633181157872764 + }, + "Id": "9be8f583-63b4-cf64-4202-0e09481317c0", + "DeletedAt": null + }, { "$type": "MiniLcmCrdtAdapter", "Obj": { @@ -3561,11 +3611,36 @@ "Y", "Z" ], + "IcuCollationRules": null, + "SystemCollationLocale": null, "Order": 0 }, "Id": "073c37cd-24cf-4d3d-ab33-19dfc94c96b7", "DeletedAt": null }, + { + "$type": "MiniLcmCrdtAdapter", + "Obj": { + "$type": "WritingSystem", + "Id": "3b9876d2-3b5d-a9f6-735d-7b6a8c232acd", + "MaybeId": "3b9876d2-3b5d-a9f6-735d-7b6a8c232acd", + "WsId": "ksh", + "IsAudio": false, + "Name": "Refined", + "Abbreviation": "Coordinator", + "Font": "indexing", + "DeletedAt": null, + "Type": 1, + "Exemplars": [ + "National" + ], + "IcuCollationRules": "Monitored", + "SystemCollationLocale": "bandwidth", + "Order": 0.2935114043495951 + }, + "Id": "3b9876d2-3b5d-a9f6-735d-7b6a8c232acd", + "DeletedAt": null + }, { "$type": "MiniLcmCrdtAdapter", "Obj": { @@ -3582,11 +3657,36 @@ "Exemplars": [ "blockchains" ], + "IcuCollationRules": null, + "SystemCollationLocale": null, "Order": 0.14806393624996905 }, "Id": "19023cf0-de25-9b0c-d5ad-94df308267c4", "DeletedAt": null }, + { + "$type": "MiniLcmCrdtAdapter", + "Obj": { + "$type": "WritingSystem", + "Id": "6e82adb2-46c1-7a00-e603-c18419f13d4f", + "MaybeId": "6e82adb2-46c1-7a00-e603-c18419f13d4f", + "WsId": "mnm", + "IsAudio": false, + "Name": "uniform", + "Abbreviation": "feed", + "Font": "Intelligent Plastic Chips", + "DeletedAt": null, + "Type": 1, + "Exemplars": [ + "Beauty, Games \u0026 Movies" + ], + "IcuCollationRules": "Assurance", + "SystemCollationLocale": "Kids \u0026 Electronics", + "Order": 0.34200726440404006 + }, + "Id": "6e82adb2-46c1-7a00-e603-c18419f13d4f", + "DeletedAt": null + }, { "$type": "MiniLcmCrdtAdapter", "Obj": { diff --git a/backend/FwLite/LcmCrdt.Tests/DataModelSnapshotTests.VerifyDbModel.verified.txt b/backend/FwLite/LcmCrdt.Tests/DataModelSnapshotTests.VerifyDbModel.verified.txt index 6eaccd9213..130f817d6a 100644 --- a/backend/FwLite/LcmCrdt.Tests/DataModelSnapshotTests.VerifyDbModel.verified.txt +++ b/backend/FwLite/LcmCrdt.Tests/DataModelSnapshotTests.VerifyDbModel.verified.txt @@ -353,9 +353,11 @@ Annotations: Relational:ColumnType: jsonb Font (string) Required + IcuCollationRules (string) Name (string) Required Order (double) Required SnapshotId (no field, Guid?) Shadow FK Index + SystemCollationLocale (string) Type (WritingSystemType) Required Index WsId (WritingSystemId) Required Index Keys: diff --git a/backend/FwLite/LcmCrdt/Changes/CreateWritingSystemChange.cs b/backend/FwLite/LcmCrdt/Changes/CreateWritingSystemChange.cs index f2e11d1f75..8e5103f84d 100644 --- a/backend/FwLite/LcmCrdt/Changes/CreateWritingSystemChange.cs +++ b/backend/FwLite/LcmCrdt/Changes/CreateWritingSystemChange.cs @@ -15,6 +15,8 @@ public class CreateWritingSystemChange : CreateChange, ISelfNamed public required string Abbreviation { get; init; } public required string Font { get; init; } public required string[] Exemplars { get; init; } = []; + public string? IcuCollationRules { get; init; } + public string? SystemCollationLocale { get; init; } public required WritingSystemType Type { get; init; } public required double Order { get; init; } @@ -27,6 +29,8 @@ public CreateWritingSystemChange(WritingSystem writingSystem, Guid entityId, dou Abbreviation = writingSystem.Abbreviation; Font = writingSystem.Font; Exemplars = writingSystem.Exemplars; + IcuCollationRules = writingSystem.IcuCollationRules; + SystemCollationLocale = writingSystem.SystemCollationLocale; Type = writingSystem.Type; Order = order; } @@ -47,6 +51,8 @@ public override async ValueTask NewEntity(Commit commit, IChangeC Abbreviation = Abbreviation, Font = Font, Exemplars = Exemplars, + IcuCollationRules = IcuCollationRules, + SystemCollationLocale = SystemCollationLocale, Type = Type, Order = Order, DeletedAt = alreadyExists ? commit.DateTime : null diff --git a/backend/FwLite/LcmCrdt/Data/SetupCollationInterceptor.cs b/backend/FwLite/LcmCrdt/Data/SetupCollationInterceptor.cs index de1a1c856e..61c319e241 100644 --- a/backend/FwLite/LcmCrdt/Data/SetupCollationInterceptor.cs +++ b/backend/FwLite/LcmCrdt/Data/SetupCollationInterceptor.cs @@ -18,43 +18,48 @@ public class SetupCollationInterceptor(IMemoryCache cache, IMiniLcmCultureProvid private static string? WsTableName = null; private WritingSystem[] GetWritingSystems(DbConnection connection, LcmCrdtDbContext? dbContext = null) { - return cache.GetOrCreate(CacheKey(connection), - entry => + var cacheKey = CacheKey(connection); + if (cache.TryGetValue(cacheKey, out WritingSystem[]? cached) && cached is not null) + { + return cached; + } + + try + { + var localContext = dbContext; + if (localContext is null) { - entry.SlidingExpiration = TimeSpan.FromMinutes(30); - try + var optionsBuilder = new DbContextOptionsBuilder(); + optionsBuilder.UseSqlite(connection); + localContext = new LcmCrdtDbContext(optionsBuilder.Options, crdtConfig); + } + + try + { + WsTableName ??= localContext.Model.FindRuntimeEntityType(typeof(WritingSystem))?.GetTableName() ?? "WritingSystem"; + if (!HasTable(localContext, WsTableName)) { - var localContext = dbContext; - if (localContext is null) - { - var optionsBuilder = new DbContextOptionsBuilder(); - optionsBuilder.UseSqlite(connection); - localContext = new LcmCrdtDbContext(optionsBuilder.Options, crdtConfig); - } - - try - { - WsTableName ??= localContext.Model.FindRuntimeEntityType(typeof(WritingSystem))?.GetTableName() ?? "WritingSystem"; - if (!HasTable(localContext, WsTableName)) - { - return []; - } - - return localContext.WritingSystems.ToArray(); - } - finally - { - if (dbContext is null) - { - localContext.Dispose(); - } - } + // Schema not migrated yet — don't cache so a later open can register collations. + return []; } - catch (SqliteException) + + var writingSystems = localContext.WritingSystems.ToArray(); + cache.Set(cacheKey, writingSystems, TimeSpan.FromMinutes(30)); + return writingSystems; + } + finally + { + if (dbContext is null) { - return []; + localContext.Dispose(); } - }) ?? []; + } + } + catch (SqliteException) + { + // Model/schema mismatch (e.g. connection opened mid-migration) — don't cache. + return []; + } } private bool HasTable(DbContext context, string tableName) diff --git a/backend/FwLite/LcmCrdt/Migrations/20260702040259_AddWritingSystemCollation.Designer.cs b/backend/FwLite/LcmCrdt/Migrations/20260702040259_AddWritingSystemCollation.Designer.cs new file mode 100644 index 0000000000..245cdfba0f --- /dev/null +++ b/backend/FwLite/LcmCrdt/Migrations/20260702040259_AddWritingSystemCollation.Designer.cs @@ -0,0 +1,858 @@ +// +using System; +using System.Collections.Generic; +using LcmCrdt; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; + +#nullable disable + +namespace LcmCrdt.Migrations +{ + [DbContext(typeof(LcmCrdtDbContext))] + [Migration("20260702040259_AddWritingSystemCollation")] + partial class AddWritingSystemCollation + { + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder.HasAnnotation("ProductVersion", "10.0.8"); + + modelBuilder.Entity("LcmCrdt.FullTextSearch.EntrySearchRecord", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("TEXT"); + + b.Property("CitationForm") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("Definition") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("Gloss") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("Headword") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("LexemeForm") + .IsRequired() + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.ToTable("EntrySearchRecord", null, t => + { + t.ExcludeFromMigrations(); + }); + }); + + modelBuilder.Entity("LcmCrdt.ProjectData", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("TEXT"); + + b.Property("ClientId") + .HasColumnType("TEXT"); + + b.Property("Code") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("FwProjectId") + .HasColumnType("TEXT"); + + b.Property("LastUserId") + .HasColumnType("TEXT"); + + b.Property("LastUserName") + .HasColumnType("TEXT"); + + b.Property("Name") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("OriginDomain") + .HasColumnType("TEXT"); + + b.Property("Role") + .IsRequired() + .ValueGeneratedOnAdd() + .HasColumnType("TEXT") + .HasDefaultValue("Editor"); + + b.HasKey("Id"); + + b.ToTable("ProjectData"); + }); + + modelBuilder.Entity("MiniLcm.Models.ComplexFormComponent", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("TEXT"); + + b.Property("ComplexFormEntryId") + .HasColumnType("TEXT"); + + b.Property("ComplexFormHeadword") + .HasColumnType("TEXT"); + + b.Property("ComponentEntryId") + .HasColumnType("TEXT"); + + b.Property("ComponentHeadword") + .HasColumnType("TEXT"); + + b.Property("ComponentSenseId") + .HasColumnType("TEXT") + .HasColumnName("ComponentSenseId"); + + b.Property("DeletedAt") + .HasColumnType("TEXT"); + + b.Property("Order") + .HasColumnType("REAL"); + + b.Property("SnapshotId") + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.HasIndex("ComponentEntryId"); + + b.HasIndex("ComponentSenseId"); + + b.HasIndex("SnapshotId") + .IsUnique(); + + b.HasIndex("ComplexFormEntryId", "ComponentEntryId") + .IsUnique() + .HasFilter("ComponentSenseId IS NULL"); + + b.HasIndex("ComplexFormEntryId", "ComponentEntryId", "ComponentSenseId") + .IsUnique() + .HasFilter("ComponentSenseId IS NOT NULL"); + + b.ToTable("ComplexFormComponents", (string)null); + }); + + modelBuilder.Entity("MiniLcm.Models.ComplexFormType", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("TEXT"); + + b.Property("DeletedAt") + .HasColumnType("TEXT"); + + b.Property("Name") + .IsRequired() + .HasColumnType("jsonb"); + + b.Property("SnapshotId") + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.HasIndex("SnapshotId") + .IsUnique(); + + b.ToTable("ComplexFormType"); + }); + + modelBuilder.Entity("MiniLcm.Models.CustomView", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("TEXT"); + + b.Property("Analysis") + .HasColumnType("jsonb"); + + b.Property("Base") + .HasColumnType("INTEGER"); + + b.Property("DeletedAt") + .HasColumnType("TEXT"); + + b.Property("EntryFields") + .IsRequired() + .HasColumnType("jsonb"); + + b.Property("ExampleFields") + .IsRequired() + .HasColumnType("jsonb"); + + b.Property("Name") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("SenseFields") + .IsRequired() + .HasColumnType("jsonb"); + + b.Property("SnapshotId") + .HasColumnType("TEXT"); + + b.Property("Vernacular") + .HasColumnType("jsonb"); + + b.HasKey("Id"); + + b.HasIndex("SnapshotId") + .IsUnique(); + + b.ToTable("CustomView"); + }); + + modelBuilder.Entity("MiniLcm.Models.Entry", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("TEXT"); + + b.Property("CitationForm") + .IsRequired() + .HasColumnType("jsonb"); + + b.Property("ComplexFormTypes") + .IsRequired() + .HasColumnType("jsonb"); + + b.Property("DeletedAt") + .HasColumnType("TEXT"); + + b.Property("HomographNumber") + .HasColumnType("INTEGER"); + + b.Property("LexemeForm") + .IsRequired() + .HasColumnType("jsonb"); + + b.Property("LiteralMeaning") + .IsRequired() + .HasColumnType("jsonb"); + + b.Property("MorphType") + .HasColumnType("INTEGER"); + + b.Property("Note") + .IsRequired() + .HasColumnType("jsonb"); + + b.Property("PublishIn") + .IsRequired() + .HasColumnType("jsonb"); + + b.Property("SnapshotId") + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.HasIndex("SnapshotId") + .IsUnique(); + + b.ToTable("Entry"); + }); + + modelBuilder.Entity("MiniLcm.Models.ExampleSentence", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("TEXT"); + + b.Property("DeletedAt") + .HasColumnType("TEXT"); + + b.Property("Order") + .HasColumnType("REAL"); + + b.Property("Reference") + .HasColumnType("jsonb"); + + b.Property("SenseId") + .HasColumnType("TEXT"); + + b.Property("Sentence") + .IsRequired() + .HasColumnType("jsonb"); + + b.Property("SnapshotId") + .HasColumnType("TEXT"); + + b.Property("Translations") + .IsRequired() + .HasColumnType("jsonb"); + + b.HasKey("Id"); + + b.HasIndex("SenseId"); + + b.HasIndex("SnapshotId") + .IsUnique(); + + b.ToTable("ExampleSentence"); + }); + + modelBuilder.Entity("MiniLcm.Models.MorphType", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("TEXT"); + + b.Property("Abbreviation") + .IsRequired() + .HasColumnType("jsonb"); + + b.Property("DeletedAt") + .HasColumnType("TEXT"); + + b.Property("Description") + .IsRequired() + .HasColumnType("jsonb"); + + b.Property("Kind") + .HasColumnType("INTEGER"); + + b.Property("Name") + .IsRequired() + .HasColumnType("jsonb"); + + b.Property("Postfix") + .HasColumnType("TEXT"); + + b.Property("Prefix") + .HasColumnType("TEXT"); + + b.Property("SecondaryOrder") + .HasColumnType("INTEGER"); + + b.Property("SnapshotId") + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.HasIndex("Kind") + .IsUnique(); + + b.HasIndex("SnapshotId") + .IsUnique(); + + b.ToTable("MorphType"); + }); + + modelBuilder.Entity("MiniLcm.Models.PartOfSpeech", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("TEXT"); + + b.Property("DeletedAt") + .HasColumnType("TEXT"); + + b.Property("Name") + .IsRequired() + .HasColumnType("jsonb"); + + b.Property("Predefined") + .HasColumnType("INTEGER"); + + b.Property("SnapshotId") + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.HasIndex("SnapshotId") + .IsUnique(); + + b.ToTable("PartOfSpeech"); + }); + + modelBuilder.Entity("MiniLcm.Models.Publication", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("TEXT"); + + b.Property("DeletedAt") + .HasColumnType("TEXT"); + + b.Property("IsMain") + .HasColumnType("INTEGER"); + + b.Property("Name") + .IsRequired() + .HasColumnType("jsonb"); + + b.Property("SnapshotId") + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.HasIndex("SnapshotId") + .IsUnique(); + + b.ToTable("Publication"); + }); + + modelBuilder.Entity("MiniLcm.Models.SemanticDomain", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("TEXT"); + + b.Property("Code") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("DeletedAt") + .HasColumnType("TEXT"); + + b.Property("Name") + .IsRequired() + .HasColumnType("jsonb"); + + b.Property("Predefined") + .HasColumnType("INTEGER"); + + b.Property("SnapshotId") + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.HasIndex("SnapshotId") + .IsUnique(); + + b.ToTable("SemanticDomain"); + }); + + modelBuilder.Entity("MiniLcm.Models.Sense", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("TEXT"); + + b.Property("Definition") + .IsRequired() + .HasColumnType("jsonb"); + + b.Property("DeletedAt") + .HasColumnType("TEXT"); + + b.Property("EntryId") + .HasColumnType("TEXT"); + + b.Property("Gloss") + .IsRequired() + .HasColumnType("jsonb"); + + b.Property("Order") + .HasColumnType("REAL"); + + b.Property("PartOfSpeechId") + .HasColumnType("TEXT"); + + b.Property("Pictures") + .IsRequired() + .ValueGeneratedOnAdd() + .HasColumnType("jsonb") + .HasDefaultValueSql("'[]'"); + + b.Property("SemanticDomains") + .IsRequired() + .HasColumnType("jsonb"); + + b.Property("SnapshotId") + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.HasIndex("EntryId"); + + b.HasIndex("PartOfSpeechId"); + + b.HasIndex("SnapshotId") + .IsUnique(); + + b.ToTable("Sense"); + }); + + modelBuilder.Entity("MiniLcm.Models.WritingSystem", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("TEXT"); + + b.Property("Abbreviation") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("DeletedAt") + .HasColumnType("TEXT"); + + b.Property("Exemplars") + .IsRequired() + .HasColumnType("jsonb"); + + b.Property("Font") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("IcuCollationRules") + .HasColumnType("TEXT"); + + b.Property("Name") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("Order") + .HasColumnType("REAL"); + + b.Property("SnapshotId") + .HasColumnType("TEXT"); + + b.Property("SystemCollationLocale") + .HasColumnType("TEXT"); + + b.Property("Type") + .HasColumnType("INTEGER"); + + b.Property("WsId") + .IsRequired() + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.HasIndex("SnapshotId") + .IsUnique(); + + b.HasIndex("WsId", "Type") + .IsUnique(); + + b.ToTable("WritingSystem"); + }); + + modelBuilder.Entity("SIL.Harmony.Commit", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("TEXT"); + + b.Property("ClientId") + .HasColumnType("TEXT"); + + b.Property("Hash") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("Metadata") + .IsRequired() + .HasColumnType("jsonb"); + + b.Property("ParentHash") + .IsRequired() + .HasColumnType("TEXT"); + + b.ComplexProperty(typeof(Dictionary), "HybridDateTime", "SIL.Harmony.Commit.HybridDateTime#HybridDateTime", b1 => + { + b1.IsRequired(); + + b1.Property("Counter") + .HasColumnType("INTEGER") + .HasColumnName("Counter"); + + b1.Property("DateTime") + .HasColumnType("TEXT") + .HasColumnName("DateTime"); + }); + + b.HasKey("Id"); + + b.ToTable("Commits", (string)null); + + b.HasAnnotation("CustomIndex:CompositeIndexes", "[{\"paths\":[\"HybridDateTime.DateTime\",\"HybridDateTime.Counter\",\"Id\"],\"unique\":false,\"name\":\"IX_Commits_DateTime_Counter_Id\"}]"); + }); + + modelBuilder.Entity("SIL.Harmony.Core.ChangeEntity", b => + { + b.Property("CommitId") + .HasColumnType("TEXT"); + + b.Property("Index") + .HasColumnType("INTEGER"); + + b.Property("Change") + .HasColumnType("jsonb"); + + b.Property("EntityId") + .HasColumnType("TEXT"); + + b.HasKey("CommitId", "Index"); + + b.ToTable("ChangeEntities", (string)null); + }); + + modelBuilder.Entity("SIL.Harmony.Db.ObjectSnapshot", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("TEXT"); + + b.Property("CommitId") + .HasColumnType("TEXT"); + + b.Property("Entity") + .IsRequired() + .HasColumnType("jsonb"); + + b.Property("EntityId") + .HasColumnType("TEXT"); + + b.Property("EntityIsDeleted") + .HasColumnType("INTEGER"); + + b.Property("IsRoot") + .HasColumnType("INTEGER"); + + b.PrimitiveCollection("References") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("TypeName") + .IsRequired() + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.HasIndex("EntityId"); + + b.HasIndex("CommitId", "EntityId") + .IsUnique(); + + b.ToTable("Snapshots", (string)null); + }); + + modelBuilder.Entity("SIL.Harmony.Resource.LocalResource", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("TEXT"); + + b.Property("LocalPath") + .IsRequired() + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.ToTable("LocalResource"); + }); + + modelBuilder.Entity("SIL.Harmony.Resource.RemoteResource", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("TEXT"); + + b.Property("DeletedAt") + .HasColumnType("TEXT"); + + b.Property("RemoteId") + .HasColumnType("TEXT"); + + b.Property("SnapshotId") + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.HasIndex("SnapshotId") + .IsUnique(); + + b.ToTable("RemoteResource"); + }); + + modelBuilder.Entity("MiniLcm.Models.ComplexFormComponent", b => + { + b.HasOne("MiniLcm.Models.Entry", null) + .WithMany("Components") + .HasForeignKey("ComplexFormEntryId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("MiniLcm.Models.Entry", null) + .WithMany("ComplexForms") + .HasForeignKey("ComponentEntryId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("MiniLcm.Models.Sense", null) + .WithMany() + .HasForeignKey("ComponentSenseId") + .OnDelete(DeleteBehavior.Cascade); + + b.HasOne("SIL.Harmony.Db.ObjectSnapshot", null) + .WithOne() + .HasForeignKey("MiniLcm.Models.ComplexFormComponent", "SnapshotId") + .OnDelete(DeleteBehavior.SetNull); + }); + + modelBuilder.Entity("MiniLcm.Models.ComplexFormType", b => + { + b.HasOne("SIL.Harmony.Db.ObjectSnapshot", null) + .WithOne() + .HasForeignKey("MiniLcm.Models.ComplexFormType", "SnapshotId") + .OnDelete(DeleteBehavior.SetNull); + }); + + modelBuilder.Entity("MiniLcm.Models.CustomView", b => + { + b.HasOne("SIL.Harmony.Db.ObjectSnapshot", null) + .WithOne() + .HasForeignKey("MiniLcm.Models.CustomView", "SnapshotId") + .OnDelete(DeleteBehavior.SetNull); + }); + + modelBuilder.Entity("MiniLcm.Models.Entry", b => + { + b.HasOne("SIL.Harmony.Db.ObjectSnapshot", null) + .WithOne() + .HasForeignKey("MiniLcm.Models.Entry", "SnapshotId") + .OnDelete(DeleteBehavior.SetNull); + }); + + modelBuilder.Entity("MiniLcm.Models.ExampleSentence", b => + { + b.HasOne("MiniLcm.Models.Sense", null) + .WithMany("ExampleSentences") + .HasForeignKey("SenseId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("SIL.Harmony.Db.ObjectSnapshot", null) + .WithOne() + .HasForeignKey("MiniLcm.Models.ExampleSentence", "SnapshotId") + .OnDelete(DeleteBehavior.SetNull); + }); + + modelBuilder.Entity("MiniLcm.Models.MorphType", b => + { + b.HasOne("SIL.Harmony.Db.ObjectSnapshot", null) + .WithOne() + .HasForeignKey("MiniLcm.Models.MorphType", "SnapshotId") + .OnDelete(DeleteBehavior.SetNull); + }); + + modelBuilder.Entity("MiniLcm.Models.PartOfSpeech", b => + { + b.HasOne("SIL.Harmony.Db.ObjectSnapshot", null) + .WithOne() + .HasForeignKey("MiniLcm.Models.PartOfSpeech", "SnapshotId") + .OnDelete(DeleteBehavior.SetNull); + }); + + modelBuilder.Entity("MiniLcm.Models.Publication", b => + { + b.HasOne("SIL.Harmony.Db.ObjectSnapshot", null) + .WithOne() + .HasForeignKey("MiniLcm.Models.Publication", "SnapshotId") + .OnDelete(DeleteBehavior.SetNull); + }); + + modelBuilder.Entity("MiniLcm.Models.SemanticDomain", b => + { + b.HasOne("SIL.Harmony.Db.ObjectSnapshot", null) + .WithOne() + .HasForeignKey("MiniLcm.Models.SemanticDomain", "SnapshotId") + .OnDelete(DeleteBehavior.SetNull); + }); + + modelBuilder.Entity("MiniLcm.Models.Sense", b => + { + b.HasOne("MiniLcm.Models.Entry", null) + .WithMany("Senses") + .HasForeignKey("EntryId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("MiniLcm.Models.PartOfSpeech", "PartOfSpeech") + .WithMany() + .HasForeignKey("PartOfSpeechId") + .OnDelete(DeleteBehavior.SetNull); + + b.HasOne("SIL.Harmony.Db.ObjectSnapshot", null) + .WithOne() + .HasForeignKey("MiniLcm.Models.Sense", "SnapshotId") + .OnDelete(DeleteBehavior.SetNull); + + b.Navigation("PartOfSpeech"); + }); + + modelBuilder.Entity("MiniLcm.Models.WritingSystem", b => + { + b.HasOne("SIL.Harmony.Db.ObjectSnapshot", null) + .WithOne() + .HasForeignKey("MiniLcm.Models.WritingSystem", "SnapshotId") + .OnDelete(DeleteBehavior.SetNull); + }); + + modelBuilder.Entity("SIL.Harmony.Core.ChangeEntity", b => + { + b.HasOne("SIL.Harmony.Commit", null) + .WithMany("ChangeEntities") + .HasForeignKey("CommitId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("SIL.Harmony.Db.ObjectSnapshot", b => + { + b.HasOne("SIL.Harmony.Commit", "Commit") + .WithMany("Snapshots") + .HasForeignKey("CommitId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Commit"); + }); + + modelBuilder.Entity("SIL.Harmony.Resource.RemoteResource", b => + { + b.HasOne("SIL.Harmony.Db.ObjectSnapshot", null) + .WithOne() + .HasForeignKey("SIL.Harmony.Resource.RemoteResource", "SnapshotId") + .OnDelete(DeleteBehavior.SetNull); + }); + + modelBuilder.Entity("MiniLcm.Models.Entry", b => + { + b.Navigation("ComplexForms"); + + b.Navigation("Components"); + + b.Navigation("Senses"); + }); + + modelBuilder.Entity("MiniLcm.Models.Sense", b => + { + b.Navigation("ExampleSentences"); + }); + + modelBuilder.Entity("SIL.Harmony.Commit", b => + { + b.Navigation("ChangeEntities"); + + b.Navigation("Snapshots"); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/backend/FwLite/LcmCrdt/Migrations/20260702040259_AddWritingSystemCollation.cs b/backend/FwLite/LcmCrdt/Migrations/20260702040259_AddWritingSystemCollation.cs new file mode 100644 index 0000000000..d94cb92c31 --- /dev/null +++ b/backend/FwLite/LcmCrdt/Migrations/20260702040259_AddWritingSystemCollation.cs @@ -0,0 +1,38 @@ +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace LcmCrdt.Migrations +{ + /// + public partial class AddWritingSystemCollation : Migration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.AddColumn( + name: "IcuCollationRules", + table: "WritingSystem", + type: "TEXT", + nullable: true); + + migrationBuilder.AddColumn( + name: "SystemCollationLocale", + table: "WritingSystem", + type: "TEXT", + nullable: true); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropColumn( + name: "IcuCollationRules", + table: "WritingSystem"); + + migrationBuilder.DropColumn( + name: "SystemCollationLocale", + table: "WritingSystem"); + } + } +} diff --git a/backend/FwLite/LcmCrdt/Migrations/LcmCrdtDbContextModelSnapshot.cs b/backend/FwLite/LcmCrdt/Migrations/LcmCrdtDbContextModelSnapshot.cs index 650a5b9420..95ca317fd1 100644 --- a/backend/FwLite/LcmCrdt/Migrations/LcmCrdtDbContextModelSnapshot.cs +++ b/backend/FwLite/LcmCrdt/Migrations/LcmCrdtDbContextModelSnapshot.cs @@ -505,6 +505,9 @@ protected override void BuildModel(ModelBuilder modelBuilder) .IsRequired() .HasColumnType("TEXT"); + b.Property("IcuCollationRules") + .HasColumnType("TEXT"); + b.Property("Name") .IsRequired() .HasColumnType("TEXT"); @@ -515,6 +518,9 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.Property("SnapshotId") .HasColumnType("TEXT"); + b.Property("SystemCollationLocale") + .HasColumnType("TEXT"); + b.Property("Type") .HasColumnType("INTEGER"); diff --git a/backend/FwLite/MiniLcm.Tests/WritingSystemSyncTests.cs b/backend/FwLite/MiniLcm.Tests/WritingSystemSyncTests.cs new file mode 100644 index 0000000000..3b670a5bc4 --- /dev/null +++ b/backend/FwLite/MiniLcm.Tests/WritingSystemSyncTests.cs @@ -0,0 +1,54 @@ +using MiniLcm.Models; +using MiniLcm.SyncHelpers; + +namespace MiniLcm.Tests; + +public class WritingSystemSyncTests +{ + private static WritingSystem BaseWs() => new() + { + Id = Guid.NewGuid(), + WsId = "en", + Name = "English", + Abbreviation = "En", + Font = "Arial", + Type = WritingSystemType.Vernacular, + }; + + [Fact] + public void DiffToUpdate_IncludesIcuCollationRules() + { + var before = BaseWs(); + var after = before with { IcuCollationRules = "&a < b" }; + + var update = WritingSystemSync.WritingSystemDiffToUpdate(before, after); + + update.Should().NotBeNull(); + var op = update!.Patch.Operations.Single(o => o.Path == $"/{nameof(WritingSystem.IcuCollationRules)}"); + op.Value?.ToString().Should().Be("&a < b"); + } + + [Fact] + public void DiffToUpdate_IncludesSystemCollationLocale() + { + var before = BaseWs(); + var after = before with { SystemCollationLocale = "de" }; + + var update = WritingSystemSync.WritingSystemDiffToUpdate(before, after); + + update.Should().NotBeNull(); + var op = update!.Patch.Operations.Single(o => o.Path == $"/{nameof(WritingSystem.SystemCollationLocale)}"); + op.Value?.ToString().Should().Be("de"); + } + + [Fact] + public void DiffToUpdate_NoOpsWhenCollationUnchanged() + { + var before = BaseWs() with { IcuCollationRules = "&a < b" }; + var after = before with { }; + + var update = WritingSystemSync.WritingSystemDiffToUpdate(before, after); + + update.Should().BeNull(); + } +} diff --git a/backend/FwLite/MiniLcm/Models/WritingSystem.cs b/backend/FwLite/MiniLcm/Models/WritingSystem.cs index b9eb655a9b..f5b3fcef2a 100644 --- a/backend/FwLite/MiniLcm/Models/WritingSystem.cs +++ b/backend/FwLite/MiniLcm/Models/WritingSystem.cs @@ -35,6 +35,17 @@ public required Guid Id public string[] Exemplars { get; set; } = []; //todo probably need more stuff here, see wesay for ideas + /// + /// Compiled ICU tailoring rules imported from FLEx (Custom Simple / Custom ICU). + /// Null uses legacy collation fallback at runtime. Not persisted as empty on import. + /// + public virtual string? IcuCollationRules { get; set; } + + /// + /// .NET locale tag for FLEx "same as another language" collation. + /// + public virtual string? SystemCollationLocale { get; set; } + public static string[] LatinExemplars => Enumerable.Range('A', 'Z' - 'A' + 1).Select(c => ((char)c).ToString()).ToArray(); [MiniLcmInternal] @@ -59,6 +70,8 @@ public WritingSystem Copy() Abbreviation = Abbreviation, Font = Font, Exemplars = [..Exemplars], + IcuCollationRules = IcuCollationRules, + SystemCollationLocale = SystemCollationLocale, DeletedAt = DeletedAt, Type = Type, Order = Order diff --git a/backend/FwLite/MiniLcm/SyncHelpers/WritingSystemSync.cs b/backend/FwLite/MiniLcm/SyncHelpers/WritingSystemSync.cs index cb58ec4e48..5f998055f2 100644 --- a/backend/FwLite/MiniLcm/SyncHelpers/WritingSystemSync.cs +++ b/backend/FwLite/MiniLcm/SyncHelpers/WritingSystemSync.cs @@ -44,6 +44,12 @@ public static async Task Sync(WritingSystem beforeWs, WritingSystem afterWs patchDocument.Operations.AddRange(SimpleStringDiff.GetStringDiff(nameof(WritingSystem.Font), beforeWritingSystem.Font, afterWritingSystem.Font)); + patchDocument.Operations.AddRange(SimpleStringDiff.GetStringDiff(nameof(WritingSystem.IcuCollationRules), + beforeWritingSystem.IcuCollationRules, + afterWritingSystem.IcuCollationRules)); + patchDocument.Operations.AddRange(SimpleStringDiff.GetStringDiff(nameof(WritingSystem.SystemCollationLocale), + beforeWritingSystem.SystemCollationLocale, + afterWritingSystem.SystemCollationLocale)); // TODO: Exemplars, Order, and do we need DeletedAt? if (patchDocument.Operations.Count == 0) return null; return new UpdateObjectInput(patchDocument); diff --git a/frontend/viewer/src/lib/dotnet-types/generated-types/MiniLcm/Models/IWritingSystem.ts b/frontend/viewer/src/lib/dotnet-types/generated-types/MiniLcm/Models/IWritingSystem.ts index 2e46bd6f9c..21824476a3 100644 --- a/frontend/viewer/src/lib/dotnet-types/generated-types/MiniLcm/Models/IWritingSystem.ts +++ b/frontend/viewer/src/lib/dotnet-types/generated-types/MiniLcm/Models/IWritingSystem.ts @@ -16,5 +16,7 @@ export interface IWritingSystem extends IObjectWithId deletedAt?: string; type: WritingSystemType; exemplars: string[]; + icuCollationRules?: string; + systemCollationLocale?: string; } /* eslint-enable */ From 1ae1982ed095a21c136c18220327afd9ed539d6d Mon Sep 17 00:00:00 2001 From: Kevin Hahn Date: Thu, 2 Jul 2026 12:56:12 +0700 Subject: [PATCH 2/3] Add writing system collator provider and use it for LcmCrdt sorting. Register ICU-backed collators in DI, initialize ICU in FwLiteWeb, and apply custom collation in SetupCollationInterceptor with Unicode tie-break sorting. Co-authored-by: Cursor --- backend/FwLite/FwLiteWeb/FwLiteWebServer.cs | 7 +- .../WritingSystemCollatorProviderTests.cs | 72 +++++++++++++++++++ .../CustomCollationSortingTests.cs | 61 ++++++++++++++++ .../Culture/LegacyCompareInfoCollator.cs | 28 ++++++++ .../Culture/WritingSystemCollationInit.cs | 26 +++++++ .../Culture/WritingSystemCollatorProvider.cs | 40 +++++++++++ .../LcmCrdt/Data/SetupCollationInterceptor.cs | 20 +++++- backend/FwLite/LcmCrdt/Data/Sorting.cs | 4 +- backend/FwLite/LcmCrdt/LcmCrdt.csproj | 1 + backend/FwLite/LcmCrdt/LcmCrdtKernel.cs | 1 + .../Culture/IWritingSystemCollatorProvider.cs | 9 +++ 11 files changed, 260 insertions(+), 9 deletions(-) create mode 100644 backend/FwLite/LcmCrdt.Tests/Culture/WritingSystemCollatorProviderTests.cs create mode 100644 backend/FwLite/LcmCrdt.Tests/MiniLcmTests/CustomCollationSortingTests.cs create mode 100644 backend/FwLite/LcmCrdt/Culture/LegacyCompareInfoCollator.cs create mode 100644 backend/FwLite/LcmCrdt/Culture/WritingSystemCollationInit.cs create mode 100644 backend/FwLite/LcmCrdt/Culture/WritingSystemCollatorProvider.cs create mode 100644 backend/FwLite/MiniLcm/Culture/IWritingSystemCollatorProvider.cs diff --git a/backend/FwLite/FwLiteWeb/FwLiteWebServer.cs b/backend/FwLite/FwLiteWeb/FwLiteWebServer.cs index 29978b9dfa..9e1d51bc29 100644 --- a/backend/FwLite/FwLiteWeb/FwLiteWebServer.cs +++ b/backend/FwLite/FwLiteWeb/FwLiteWebServer.cs @@ -24,11 +24,8 @@ public static WebApplication SetupAppServer(WebApplicationOptions options, Actio var builder = WebApplication.CreateBuilder(options); if (!builder.Environment.IsDevelopment() && options.Args?.Contains("--urls") != true && string.IsNullOrEmpty(builder.Configuration["http_ports"])) builder.WebHost.UseUrls("http://127.0.0.1:0"); - if (builder.Environment.IsDevelopment()) - { - //do this early so we catch bugs on startup - ProjectLoader.Init(); - } + // ICU required for imported writing-system collations (IcuRulesCollator). + ProjectLoader.Init(); builder.ConfigureDev(config => config.LexboxServers = [ diff --git a/backend/FwLite/LcmCrdt.Tests/Culture/WritingSystemCollatorProviderTests.cs b/backend/FwLite/LcmCrdt.Tests/Culture/WritingSystemCollatorProviderTests.cs new file mode 100644 index 0000000000..54dfe8020b --- /dev/null +++ b/backend/FwLite/LcmCrdt.Tests/Culture/WritingSystemCollatorProviderTests.cs @@ -0,0 +1,72 @@ +using LcmCrdt.Culture; +using Microsoft.Extensions.Caching.Memory; +using Microsoft.Extensions.DependencyInjection; +using MiniLcm.Culture; +using MiniLcm.Models; +using SIL.WritingSystems; + +namespace LcmCrdt.Tests.Culture; + +public class WritingSystemCollatorProviderTests +{ + private static IWritingSystemCollatorProvider CreateProvider() + { + var services = new ServiceCollection(); + services.AddLogging(); + services.AddMemoryCache(); + services.AddSingleton(); + services.AddSingleton(); + return services.BuildServiceProvider().GetRequiredService(); + } + + private static WritingSystem BaseWs() => new() + { + Id = Guid.NewGuid(), + WsId = "en", + Name = "English", + Abbreviation = "En", + Font = "Arial", + Type = WritingSystemType.Vernacular, + }; + + [Fact] + public void GetCollator_UsesSystemCollator_WhenLocaleSet() + { + var provider = CreateProvider(); + var collator = provider.GetCollator(BaseWs() with { SystemCollationLocale = "de" }); + collator.Should().BeOfType(); + } + + [Fact] + public void GetCollator_UsesIcuRulesCollator_WhenRulesSet() + { + var provider = CreateProvider(); + var collator = provider.GetCollator(BaseWs() with { IcuCollationRules = "&a < b" }); + collator.Should().BeOfType(); + } + + [Fact] + public void GetCollator_UsesLegacyFallback_WhenCollationUnset() + { + var provider = CreateProvider(); + var collator = provider.GetCollator(BaseWs()); + collator.Should().NotBeOfType(); + collator.Should().NotBeOfType(); + } + + [Fact] + public void IcuRulesCollator_SortsByCustomRules() + { + var provider = CreateProvider(); + var collator = provider.GetCollator(BaseWs() with { IcuCollationRules = "&z < a" }); + collator.Compare("z", "a").Should().BeLessThan(0); + } + + [Fact] + public void LegacyCollator_PrefersLowercaseOnCaseInsensitiveTie() + { + var provider = CreateProvider(); + var collator = provider.GetCollator(BaseWs()); + collator.Compare("Ab", "ab").Should().BeGreaterThan(0); + } +} diff --git a/backend/FwLite/LcmCrdt.Tests/MiniLcmTests/CustomCollationSortingTests.cs b/backend/FwLite/LcmCrdt.Tests/MiniLcmTests/CustomCollationSortingTests.cs new file mode 100644 index 0000000000..f3c5d4245a --- /dev/null +++ b/backend/FwLite/LcmCrdt.Tests/MiniLcmTests/CustomCollationSortingTests.cs @@ -0,0 +1,61 @@ +namespace LcmCrdt.Tests.MiniLcmTests; + +public class CustomCollationSortingTests(MiniLcmApiFixture fixture) : IClassFixture +{ + [Fact] + public async Task HeadwordSort_UsesIcuCollationRules() + { + const string wsId = "en-x-icu-test"; + await fixture.Api.CreateWritingSystem(new() + { + Id = Guid.NewGuid(), + Type = WritingSystemType.Vernacular, + WsId = wsId, + Name = "Custom ICU", + Abbreviation = "Ci", + Font = "Arial", + IcuCollationRules = "&z < a", + }); + + var apple = await fixture.Api.CreateEntry(new() { LexemeForm = { { wsId, "apple" } } }); + var zebra = await fixture.Api.CreateEntry(new() { LexemeForm = { { wsId, "zebra" } } }); + var ids = new[] { apple.Id, zebra.Id }.ToHashSet(); + + var headwords = await fixture.Api + .GetEntries(new QueryOptions(new SortOptions(SortField.Headword, wsId))) + .Where(e => ids.Contains(e.Id)) + .Select(e => e.Headword()) + .ToArrayAsync(); + + headwords.Should().Equal("zebra", "apple"); + } + + [Fact] + public async Task HeadwordSort_UsesSystemCollationLocale() + { + const string wsId = "cs"; + await fixture.Api.CreateWritingSystem(new() + { + Id = Guid.NewGuid(), + Type = WritingSystemType.Vernacular, + WsId = wsId, + Name = "Czech", + Abbreviation = "Cs", + Font = "Arial", + SystemCollationLocale = "cs", + }); + + // English sorts c before h; Czech treats "ch" as a letter after h. + var cha = await fixture.Api.CreateEntry(new() { LexemeForm = { { wsId, "cha" } } }); + var ha = await fixture.Api.CreateEntry(new() { LexemeForm = { { wsId, "ha" } } }); + var ids = new[] { cha.Id, ha.Id }.ToHashSet(); + + var headwords = await fixture.Api + .GetEntries(new QueryOptions(new SortOptions(SortField.Headword, wsId))) + .Where(e => ids.Contains(e.Id)) + .Select(e => e.Headword()) + .ToArrayAsync(); + + headwords.Should().Equal("ha", "cha"); + } +} diff --git a/backend/FwLite/LcmCrdt/Culture/LegacyCompareInfoCollator.cs b/backend/FwLite/LcmCrdt/Culture/LegacyCompareInfoCollator.cs new file mode 100644 index 0000000000..93c2f7dff8 --- /dev/null +++ b/backend/FwLite/LcmCrdt/Culture/LegacyCompareInfoCollator.cs @@ -0,0 +1,28 @@ +using System.Collections; +using System.Globalization; +using SIL.WritingSystems; + +namespace LcmCrdt.Culture; + +/// +/// Pre-collation-import fallback: case-insensitive compare with lowercase-before-uppercase tie-break. +/// +internal sealed class LegacyCompareInfoCollator(CompareInfo compareInfo) : ICollator +{ + public int Compare(string? x, string? y) + { + x ??= string.Empty; + y ??= string.Empty; + var caseInsensitiveResult = compareInfo.Compare(x, y, CompareOptions.IgnoreCase); + if (caseInsensitiveResult != 0) + { + return caseInsensitiveResult; + } + + return compareInfo.Compare(x, y, CompareOptions.None); + } + + public SortKey GetSortKey(string source) => compareInfo.GetSortKey(source); + + int IComparer.Compare(object? x, object? y) => Compare(x as string, y as string); +} diff --git a/backend/FwLite/LcmCrdt/Culture/WritingSystemCollationInit.cs b/backend/FwLite/LcmCrdt/Culture/WritingSystemCollationInit.cs new file mode 100644 index 0000000000..073d61d5e4 --- /dev/null +++ b/backend/FwLite/LcmCrdt/Culture/WritingSystemCollationInit.cs @@ -0,0 +1,26 @@ +namespace LcmCrdt.Culture; + +internal static class WritingSystemCollationInit +{ + private static bool _initialized; + private static readonly Lock _lock = new(); + + internal static void EnsureInitialized() + { + if (_initialized) + { + return; + } + + lock (_lock) + { + if (_initialized) + { + return; + } + + Icu.Wrapper.Init(); + _initialized = true; + } + } +} diff --git a/backend/FwLite/LcmCrdt/Culture/WritingSystemCollatorProvider.cs b/backend/FwLite/LcmCrdt/Culture/WritingSystemCollatorProvider.cs new file mode 100644 index 0000000000..31557d9d28 --- /dev/null +++ b/backend/FwLite/LcmCrdt/Culture/WritingSystemCollatorProvider.cs @@ -0,0 +1,40 @@ +using Microsoft.Extensions.Caching.Memory; +using MiniLcm.Culture; +using MiniLcm.Models; +using SIL.WritingSystems; + +namespace LcmCrdt.Culture; + +public class WritingSystemCollatorProvider( + IMemoryCache cache, + IMiniLcmCultureProvider cultureProvider) : IWritingSystemCollatorProvider +{ + public ICollator GetCollator(WritingSystem writingSystem) + { + return cache.GetOrCreate(CacheKey(writingSystem), entry => + { + entry.SlidingExpiration = TimeSpan.FromHours(1); + return CreateCollator(writingSystem); + }) ?? CreateCollator(writingSystem); + } + + private ICollator CreateCollator(WritingSystem writingSystem) + { + if (!string.IsNullOrEmpty(writingSystem.SystemCollationLocale)) + { + return new SystemCollator(writingSystem.SystemCollationLocale); + } + + if (!string.IsNullOrEmpty(writingSystem.IcuCollationRules)) + { + WritingSystemCollationInit.EnsureInitialized(); + return new IcuRulesCollator(writingSystem.IcuCollationRules); + } + + // FLEx default ordering also lands here until we choose to match its empty-rule ICU default. + return new LegacyCompareInfoCollator(cultureProvider.GetCompareInfo(writingSystem)); + } + + private static string CacheKey(WritingSystem writingSystem) => + $"collator|{writingSystem.WsId}|{writingSystem.Type}|{writingSystem.IcuCollationRules}|{writingSystem.SystemCollationLocale}"; +} diff --git a/backend/FwLite/LcmCrdt/Data/SetupCollationInterceptor.cs b/backend/FwLite/LcmCrdt/Data/SetupCollationInterceptor.cs index 61c319e241..9985440bb3 100644 --- a/backend/FwLite/LcmCrdt/Data/SetupCollationInterceptor.cs +++ b/backend/FwLite/LcmCrdt/Data/SetupCollationInterceptor.cs @@ -8,12 +8,17 @@ using Microsoft.EntityFrameworkCore.Diagnostics; using Microsoft.Extensions.Caching.Memory; using Microsoft.Extensions.Options; +using LcmCrdt.Culture; using MiniLcm.Culture; using SIL.Harmony; namespace LcmCrdt.Data; -public class SetupCollationInterceptor(IMemoryCache cache, IMiniLcmCultureProvider cultureProvider, IOptions crdtConfig) : IDbConnectionInterceptor, ISaveChangesInterceptor, IConnectionInterceptor +public class SetupCollationInterceptor( + IMemoryCache cache, + IMiniLcmCultureProvider cultureProvider, + IWritingSystemCollatorProvider collatorProvider, + IOptions crdtConfig) : IDbConnectionInterceptor, ISaveChangesInterceptor, IConnectionInterceptor { private static string? WsTableName = null; private WritingSystem[] GetWritingSystems(DbConnection connection, LcmCrdtDbContext? dbContext = null) @@ -191,9 +196,16 @@ private void SetupCollations(SqliteConnection connection, WritingSystem[] writin private void SetupCollation(SqliteConnection connection, WritingSystem writingSystem) { + if (HasImportedCollation(writingSystem)) + { + var collator = collatorProvider.GetCollator(writingSystem); + connection.CreateCollation(SqlSortingExtensions.CollationName(writingSystem.WsId), + (x, y) => collator.Compare(x, y)); + return; + } + var compareInfo = cultureProvider.GetCompareInfo(writingSystem); - //todo use custom comparison based on the writing system CreateSpanCollation(connection, SqlSortingExtensions.CollationName(writingSystem.WsId), compareInfo, static (compareInfo, x, y) => @@ -206,6 +218,10 @@ private void SetupCollation(SqliteConnection connection, WritingSystem writingSy }); } + private static bool HasImportedCollation(WritingSystem writingSystem) => + !string.IsNullOrEmpty(writingSystem.SystemCollationLocale) + || !string.IsNullOrEmpty(writingSystem.IcuCollationRules); + //this is a premature optimization, but it avoids creating strings for each comparison and instead uses spans which avoids allocations //if the new comparison function does not support spans then we can use SqliteConnection.CreateCollation instead which works with strings private void CreateSpanCollation(SqliteConnection connection, diff --git a/backend/FwLite/LcmCrdt/Data/Sorting.cs b/backend/FwLite/LcmCrdt/Data/Sorting.cs index 57822f2e99..65b8eef261 100644 --- a/backend/FwLite/LcmCrdt/Data/Sorting.cs +++ b/backend/FwLite/LcmCrdt/Data/Sorting.cs @@ -51,7 +51,7 @@ from mt in mtGroup.DefaultIfEmpty() !string.IsNullOrEmpty(query) && SqlHelpers.StartsWithIgnoreCaseAccents(e.HeadwordWithTokens(order.WritingSystem, mt.Prefix, mt.Postfix), query!) descending, !string.IsNullOrEmpty(query) && SqlHelpers.ContainsIgnoreCaseAccents(e.HeadwordWithTokens(order.WritingSystem, mt.Prefix, mt.Postfix), query!) descending, e.Headword(order.WritingSystem).Length, - e.Headword(order.WritingSystem), + e.Headword(order.WritingSystem).CollateUnicode(order.WritingSystem), mt != null ? mt.SecondaryOrder : stemOrder.FirstOrDefault(), e.HomographNumber, e.Id @@ -67,7 +67,7 @@ from mt in mtGroup.DefaultIfEmpty() !string.IsNullOrEmpty(query) && SqlHelpers.StartsWithIgnoreCaseAccents(e.HeadwordWithTokens(order.WritingSystem, mt.Prefix, mt.Postfix), query!), !string.IsNullOrEmpty(query) && SqlHelpers.ContainsIgnoreCaseAccents(e.HeadwordWithTokens(order.WritingSystem, mt.Prefix, mt.Postfix), query!), e.Headword(order.WritingSystem).Length descending, - e.Headword(order.WritingSystem) descending, + e.Headword(order.WritingSystem).CollateUnicode(order.WritingSystem) descending, (mt != null ? mt.SecondaryOrder : stemOrder.FirstOrDefault()) descending, e.HomographNumber descending, e.Id descending diff --git a/backend/FwLite/LcmCrdt/LcmCrdt.csproj b/backend/FwLite/LcmCrdt/LcmCrdt.csproj index e5a86fa3ba..c0942b7af5 100644 --- a/backend/FwLite/LcmCrdt/LcmCrdt.csproj +++ b/backend/FwLite/LcmCrdt/LcmCrdt.csproj @@ -24,6 +24,7 @@ + diff --git a/backend/FwLite/LcmCrdt/LcmCrdtKernel.cs b/backend/FwLite/LcmCrdt/LcmCrdtKernel.cs index 4aaa97f3b7..f48aeafe27 100644 --- a/backend/FwLite/LcmCrdt/LcmCrdtKernel.cs +++ b/backend/FwLite/LcmCrdt/LcmCrdtKernel.cs @@ -54,6 +54,7 @@ public static IServiceCollection AddLcmCrdtClientCore(this IServiceCollection se services.AddMemoryCache(); services.AddSingleton(); + services.AddSingleton(); services.AddSingleton(); services.AddScoped(); services.AddSingleton(); diff --git a/backend/FwLite/MiniLcm/Culture/IWritingSystemCollatorProvider.cs b/backend/FwLite/MiniLcm/Culture/IWritingSystemCollatorProvider.cs new file mode 100644 index 0000000000..4c3139455b --- /dev/null +++ b/backend/FwLite/MiniLcm/Culture/IWritingSystemCollatorProvider.cs @@ -0,0 +1,9 @@ +using MiniLcm.Models; +using SIL.WritingSystems; + +namespace MiniLcm.Culture; + +public interface IWritingSystemCollatorProvider +{ + ICollator GetCollator(WritingSystem writingSystem); +} From 5d5dfbf326fe5fee17cc9e700ddde6790a22af1d Mon Sep 17 00:00:00 2001 From: Kevin Hahn Date: Fri, 3 Jul 2026 16:37:53 +0700 Subject: [PATCH 3/3] Use ICU4N for FwLite collation and seed demo sort on example project. Replace icu.net/libpalaso collators in LcmCrdt with managed ICU4N adapters so custom writing-system collation works on Android. Seed the example project's German vernacular with reversed A/B ICU rules for manual testing. Co-authored-by: Cursor --- backend/Directory.Packages.props | 1 + backend/FwLite/CONTEXT.md | 4 +- backend/FwLite/FwLiteWeb/FwLiteWebServer.cs | 2 +- .../WritingSystemCollatorProviderTests.cs | 31 +++++++--- .../FwLite/LcmCrdt.Tests/OpenProjectTests.cs | 8 +++ backend/FwLite/LcmCrdt/CrdtProjectsService.cs | 10 +++- .../Culture/CultureCompareInfoCollator.cs | 18 ++++++ .../LcmCrdt/Culture/Icu4NLocaleCollator.cs | 22 +++++++ .../LcmCrdt/Culture/Icu4NRulesCollator.cs | 21 +++++++ .../Culture/WritingSystemCollationInit.cs | 26 --------- .../Culture/WritingSystemCollatorProvider.cs | 57 +++++++++++++++++-- backend/FwLite/LcmCrdt/ExampleProjectData.cs | 7 ++- backend/FwLite/LcmCrdt/LcmCrdt.csproj | 2 +- .../FwLite/LcmCrdt/Project/ProjectTemplate.cs | 16 ++++-- 14 files changed, 174 insertions(+), 51 deletions(-) create mode 100644 backend/FwLite/LcmCrdt/Culture/CultureCompareInfoCollator.cs create mode 100644 backend/FwLite/LcmCrdt/Culture/Icu4NLocaleCollator.cs create mode 100644 backend/FwLite/LcmCrdt/Culture/Icu4NRulesCollator.cs delete mode 100644 backend/FwLite/LcmCrdt/Culture/WritingSystemCollationInit.cs diff --git a/backend/Directory.Packages.props b/backend/Directory.Packages.props index d717033990..3d869111b8 100644 --- a/backend/Directory.Packages.props +++ b/backend/Directory.Packages.props @@ -29,6 +29,7 @@ + diff --git a/backend/FwLite/CONTEXT.md b/backend/FwLite/CONTEXT.md index e749ce64ba..a5f61cb073 100644 --- a/backend/FwLite/CONTEXT.md +++ b/backend/FwLite/CONTEXT.md @@ -29,8 +29,8 @@ When neither field is set, headword comparison uses the pre-existing FwLite beha _Avoid_: Treating `null` and `""` as different states for `IcuCollationRules` **Collation compare (imported)**: -When `IcuCollationRules` or `SystemCollationLocale` is set, use the libpalaso collator's native `Compare` with no legacy case tie-break layered on top. Matches FLEx for custom and other-language modes. -_Avoid_: Applying case-insensitive or lowercase-first logic on top of `IcuRulesCollator` or `SystemCollator` +When `IcuCollationRules` or `SystemCollationLocale` is set, use ICU4N (`RuleBasedCollator` / locale `Collator`) `Compare` with no legacy case tie-break layered on top. Matches FLEx for custom and other-language modes. FwData/FLEx bridge code still uses icu.net separately. +_Avoid_: Applying case-insensitive or lowercase-first logic on top of imported ICU4N collation **Collation import (from FLEx)**: Populated when mapping `CoreWritingSystemDefinition` → `WritingSystem` in the FwData bridge. Custom modes: store non-empty compiled ICU rules only. Other-language: store .NET locale tag only. Default ordering: leave both fields null (legacy fallback). Import-only — no write-back to fwdata LDML. diff --git a/backend/FwLite/FwLiteWeb/FwLiteWebServer.cs b/backend/FwLite/FwLiteWeb/FwLiteWebServer.cs index 9e1d51bc29..a393fc17b4 100644 --- a/backend/FwLite/FwLiteWeb/FwLiteWebServer.cs +++ b/backend/FwLite/FwLiteWeb/FwLiteWebServer.cs @@ -24,7 +24,7 @@ public static WebApplication SetupAppServer(WebApplicationOptions options, Actio var builder = WebApplication.CreateBuilder(options); if (!builder.Environment.IsDevelopment() && options.Args?.Contains("--urls") != true && string.IsNullOrEmpty(builder.Configuration["http_ports"])) builder.WebHost.UseUrls("http://127.0.0.1:0"); - // ICU required for imported writing-system collations (IcuRulesCollator). + // ICU required for FwData bridge (liblcm / icu.net), not for CRDT collation (ICU4N). ProjectLoader.Init(); builder.ConfigureDev(config => diff --git a/backend/FwLite/LcmCrdt.Tests/Culture/WritingSystemCollatorProviderTests.cs b/backend/FwLite/LcmCrdt.Tests/Culture/WritingSystemCollatorProviderTests.cs index 54dfe8020b..1984be1c08 100644 --- a/backend/FwLite/LcmCrdt.Tests/Culture/WritingSystemCollatorProviderTests.cs +++ b/backend/FwLite/LcmCrdt.Tests/Culture/WritingSystemCollatorProviderTests.cs @@ -30,19 +30,19 @@ private static IWritingSystemCollatorProvider CreateProvider() }; [Fact] - public void GetCollator_UsesSystemCollator_WhenLocaleSet() + public void GetCollator_UsesIcu4NLocaleCollator_WhenLocaleSet() { var provider = CreateProvider(); var collator = provider.GetCollator(BaseWs() with { SystemCollationLocale = "de" }); - collator.Should().BeOfType(); + collator.Should().BeOfType(); } [Fact] - public void GetCollator_UsesIcuRulesCollator_WhenRulesSet() + public void GetCollator_UsesIcu4NRulesCollator_WhenRulesSet() { var provider = CreateProvider(); var collator = provider.GetCollator(BaseWs() with { IcuCollationRules = "&a < b" }); - collator.Should().BeOfType(); + collator.Should().BeOfType(); } [Fact] @@ -50,18 +50,26 @@ public void GetCollator_UsesLegacyFallback_WhenCollationUnset() { var provider = CreateProvider(); var collator = provider.GetCollator(BaseWs()); - collator.Should().NotBeOfType(); - collator.Should().NotBeOfType(); + collator.Should().NotBeOfType(); + collator.Should().NotBeOfType(); } [Fact] - public void IcuRulesCollator_SortsByCustomRules() + public void Icu4NRulesCollator_SortsByCustomRules() { var provider = CreateProvider(); var collator = provider.GetCollator(BaseWs() with { IcuCollationRules = "&z < a" }); collator.Compare("z", "a").Should().BeLessThan(0); } + [Fact] + public void Icu4NRulesCollator_ReversesAAndB() + { + var provider = CreateProvider(); + var collator = provider.GetCollator(BaseWs() with { IcuCollationRules = "&b < a &B < A" }); + collator.Compare("Banane", "Apfel").Should().BeLessThan(0); + } + [Fact] public void LegacyCollator_PrefersLowercaseOnCaseInsensitiveTie() { @@ -69,4 +77,13 @@ public void LegacyCollator_PrefersLowercaseOnCaseInsensitiveTie() var collator = provider.GetCollator(BaseWs()); collator.Compare("Ab", "ab").Should().BeGreaterThan(0); } + + [Fact] + public void GetCollator_FallsBackToLegacy_WhenRulesInvalid() + { + var provider = CreateProvider(); + var collator = provider.GetCollator(BaseWs() with { IcuCollationRules = "not valid icu rules <<<<" }); + collator.Should().NotBeOfType(); + collator.Should().BeOfType(); + } } diff --git a/backend/FwLite/LcmCrdt.Tests/OpenProjectTests.cs b/backend/FwLite/LcmCrdt.Tests/OpenProjectTests.cs index a650a5f01a..703d4c362e 100644 --- a/backend/FwLite/LcmCrdt.Tests/OpenProjectTests.cs +++ b/backend/FwLite/LcmCrdt.Tests/OpenProjectTests.cs @@ -32,6 +32,7 @@ public async Task CanCreateExampleProject() .Contain(["de", "de-Zxxx-x-audio", "de-fonipa"]); writingSystems.Analysis.Select(ws => ws.WsId.Code).Should() .Contain(["en", "fr"]); + writingSystems.Vernacular.Single(ws => ws.WsId == "de").IcuCollationRules.Should().Be(ExampleProjectData.DemoIcuCollationRules); var entries = await api.GetEntries().ToArrayAsync(); entries.Select(e => e.LexemeForm["de"]).Should().Contain( @@ -41,6 +42,13 @@ public async Task CanCreateExampleProject() var complexForms = entries.Where(e => e.Components.Count > 0).ToArray(); complexForms.Select(e => e.LexemeForm["de"]).Should().Contain(["Erdbeere", "Heidelbeere"]); + var headwords = await api + .GetEntries(new QueryOptions(new SortOptions(SortField.Headword, "de"))) + .Select(e => e.LexemeForm["de"]) + .ToArrayAsync(); + headwords.Should().Equal( + "Banane", "Beere", "Apfel", "Erdbeere", "Heidelbeere", "Orange", "Traube"); + await using var dbContext = await asyncScope.ServiceProvider.GetRequiredService>().CreateDbContextAsync(); await dbContext.Database.EnsureDeletedAsync(); } diff --git a/backend/FwLite/LcmCrdt/CrdtProjectsService.cs b/backend/FwLite/LcmCrdt/CrdtProjectsService.cs index c4811e202e..d3692693b9 100644 --- a/backend/FwLite/LcmCrdt/CrdtProjectsService.cs +++ b/backend/FwLite/LcmCrdt/CrdtProjectsService.cs @@ -131,7 +131,10 @@ public record CreateProjectRequest( public async Task CreateExampleProject(string name) { // Code must satisfy the lowercase-only ProjectCode rule; the display name keeps its casing. - return await CreateProjectFromTemplate(new(name, name.ToLowerInvariant(), AfterCreate: ExampleProjectData.Seed, Role: UserProjectRole.Manager), vernacularWs: "de"); + return await CreateProjectFromTemplate( + new(name, name.ToLowerInvariant(), AfterCreate: ExampleProjectData.Seed, Role: UserProjectRole.Manager), + vernacularWs: "de", + configureVernacular: ws => ws with { IcuCollationRules = ExampleProjectData.DemoIcuCollationRules }); } /// @@ -144,7 +147,8 @@ public async Task CreateExampleProject(string name) public virtual async Task CreateProjectFromTemplate( CreateProjectRequest request, WritingSystemId vernacularWs, - WritingSystemId? analysisWs = null) + WritingSystemId? analysisWs = null, + Func? configureVernacular = null) { // Templated projects are created locally; this path has no way to associate one with a server // at creation time, so reject a Domain up front — the invariant holds where the project is born. @@ -160,7 +164,7 @@ public virtual async Task CreateProjectFromTemplate( { var api = provider.GetRequiredService(); var jsonOptions = provider.GetRequiredService>().Value.JsonSerializerOptions; - var snapshot = ProjectTemplate.CreateNewSnapshot(jsonOptions, vernacularWs, analysisWs); + var snapshot = ProjectTemplate.CreateNewSnapshot(jsonOptions, vernacularWs, analysisWs, configureVernacular); await projectImporter.ImportProject(api, snapshot); if (callerAfterCreate is not null) await callerAfterCreate(provider, project); } diff --git a/backend/FwLite/LcmCrdt/Culture/CultureCompareInfoCollator.cs b/backend/FwLite/LcmCrdt/Culture/CultureCompareInfoCollator.cs new file mode 100644 index 0000000000..eeb79e511f --- /dev/null +++ b/backend/FwLite/LcmCrdt/Culture/CultureCompareInfoCollator.cs @@ -0,0 +1,18 @@ +using System.Collections; +using System.Globalization; +using SIL.WritingSystems; + +namespace LcmCrdt.Culture; + +/// +/// Locale collation via .NET CompareInfo, without the legacy case tie-break. +/// +internal sealed class CultureCompareInfoCollator(CompareInfo compareInfo) : ICollator +{ + public int Compare(string? x, string? y) => + compareInfo.Compare(x ?? string.Empty, y ?? string.Empty, CompareOptions.None); + + public SortKey GetSortKey(string source) => compareInfo.GetSortKey(source); + + int IComparer.Compare(object? x, object? y) => Compare(x as string, y as string); +} diff --git a/backend/FwLite/LcmCrdt/Culture/Icu4NLocaleCollator.cs b/backend/FwLite/LcmCrdt/Culture/Icu4NLocaleCollator.cs new file mode 100644 index 0000000000..98d89bdd7c --- /dev/null +++ b/backend/FwLite/LcmCrdt/Culture/Icu4NLocaleCollator.cs @@ -0,0 +1,22 @@ +using System.Collections; +using System.Globalization; +using ICU4N.Text; +using SIL.WritingSystems; + +namespace LcmCrdt.Culture; + +internal sealed class Icu4NLocaleCollator : ICollator +{ + private readonly Collator _collator; + + public Icu4NLocaleCollator(string locale) => + _collator = Collator.GetInstance(CultureInfo.GetCultureInfo(locale)); + + public int Compare(string? x, string? y) => + _collator.Compare(x ?? string.Empty, y ?? string.Empty); + + public SortKey GetSortKey(string source) => + throw new NotSupportedException("ICU4N sort keys are not used by FwLite collation."); + + int IComparer.Compare(object? x, object? y) => Compare(x as string, y as string); +} diff --git a/backend/FwLite/LcmCrdt/Culture/Icu4NRulesCollator.cs b/backend/FwLite/LcmCrdt/Culture/Icu4NRulesCollator.cs new file mode 100644 index 0000000000..38df4cebc0 --- /dev/null +++ b/backend/FwLite/LcmCrdt/Culture/Icu4NRulesCollator.cs @@ -0,0 +1,21 @@ +using System.Collections; +using System.Globalization; +using ICU4N.Text; +using SIL.WritingSystems; + +namespace LcmCrdt.Culture; + +internal sealed class Icu4NRulesCollator : ICollator +{ + private readonly RuleBasedCollator _collator; + + public Icu4NRulesCollator(string rules) => _collator = new RuleBasedCollator(rules); + + public int Compare(string? x, string? y) => + _collator.Compare(x ?? string.Empty, y ?? string.Empty); + + public SortKey GetSortKey(string source) => + throw new NotSupportedException("ICU4N sort keys are not used by FwLite collation."); + + int IComparer.Compare(object? x, object? y) => Compare(x as string, y as string); +} diff --git a/backend/FwLite/LcmCrdt/Culture/WritingSystemCollationInit.cs b/backend/FwLite/LcmCrdt/Culture/WritingSystemCollationInit.cs deleted file mode 100644 index 073d61d5e4..0000000000 --- a/backend/FwLite/LcmCrdt/Culture/WritingSystemCollationInit.cs +++ /dev/null @@ -1,26 +0,0 @@ -namespace LcmCrdt.Culture; - -internal static class WritingSystemCollationInit -{ - private static bool _initialized; - private static readonly Lock _lock = new(); - - internal static void EnsureInitialized() - { - if (_initialized) - { - return; - } - - lock (_lock) - { - if (_initialized) - { - return; - } - - Icu.Wrapper.Init(); - _initialized = true; - } - } -} diff --git a/backend/FwLite/LcmCrdt/Culture/WritingSystemCollatorProvider.cs b/backend/FwLite/LcmCrdt/Culture/WritingSystemCollatorProvider.cs index 31557d9d28..adc94359b2 100644 --- a/backend/FwLite/LcmCrdt/Culture/WritingSystemCollatorProvider.cs +++ b/backend/FwLite/LcmCrdt/Culture/WritingSystemCollatorProvider.cs @@ -1,4 +1,6 @@ +using System.Globalization; using Microsoft.Extensions.Caching.Memory; +using Microsoft.Extensions.Logging; using MiniLcm.Culture; using MiniLcm.Models; using SIL.WritingSystems; @@ -7,7 +9,8 @@ namespace LcmCrdt.Culture; public class WritingSystemCollatorProvider( IMemoryCache cache, - IMiniLcmCultureProvider cultureProvider) : IWritingSystemCollatorProvider + IMiniLcmCultureProvider cultureProvider, + ILogger logger) : IWritingSystemCollatorProvider { public ICollator GetCollator(WritingSystem writingSystem) { @@ -22,19 +25,63 @@ private ICollator CreateCollator(WritingSystem writingSystem) { if (!string.IsNullOrEmpty(writingSystem.SystemCollationLocale)) { - return new SystemCollator(writingSystem.SystemCollationLocale); + return TryCreateLocaleCollator(writingSystem); } if (!string.IsNullOrEmpty(writingSystem.IcuCollationRules)) { - WritingSystemCollationInit.EnsureInitialized(); - return new IcuRulesCollator(writingSystem.IcuCollationRules); + return TryCreateRulesCollator(writingSystem); } - // FLEx default ordering also lands here until we choose to match its empty-rule ICU default. return new LegacyCompareInfoCollator(cultureProvider.GetCompareInfo(writingSystem)); } + private ICollator TryCreateLocaleCollator(WritingSystem writingSystem) + { + try + { + return new Icu4NLocaleCollator(writingSystem.SystemCollationLocale!); + } + catch (Exception ex) + { + logger.LogWarning( + ex, + "Failed to create ICU4N locale collator for '{Locale}' on writing system '{WsId}'; using .NET collation fallback", + writingSystem.SystemCollationLocale, + writingSystem.WsId); + return CreateCultureCollator(writingSystem.SystemCollationLocale!); + } + } + + private ICollator TryCreateRulesCollator(WritingSystem writingSystem) + { + try + { + return new Icu4NRulesCollator(writingSystem.IcuCollationRules!); + } + catch (Exception ex) + { + logger.LogWarning( + ex, + "Failed to create ICU4N rules collator for writing system '{WsId}'; using legacy collation fallback", + writingSystem.WsId); + return new LegacyCompareInfoCollator(cultureProvider.GetCompareInfo(writingSystem)); + } + } + + private ICollator CreateCultureCollator(string locale) + { + try + { + return new CultureCompareInfoCollator(CultureInfo.GetCultureInfo(locale).CompareInfo); + } + catch (CultureNotFoundException ex) + { + logger.LogWarning(ex, "Unknown system collation locale '{Locale}'; using invariant collation", locale); + return new CultureCompareInfoCollator(CultureInfo.InvariantCulture.CompareInfo); + } + } + private static string CacheKey(WritingSystem writingSystem) => $"collator|{writingSystem.WsId}|{writingSystem.Type}|{writingSystem.IcuCollationRules}|{writingSystem.SystemCollationLocale}"; } diff --git a/backend/FwLite/LcmCrdt/ExampleProjectData.cs b/backend/FwLite/LcmCrdt/ExampleProjectData.cs index 3035df9fce..9a0d369fb4 100644 --- a/backend/FwLite/LcmCrdt/ExampleProjectData.cs +++ b/backend/FwLite/LcmCrdt/ExampleProjectData.cs @@ -4,6 +4,9 @@ namespace LcmCrdt; internal static class ExampleProjectData { + /// ICU tailoring that sorts B before A so demo fruit headwords visibly differ from default order. + internal const string DemoIcuCollationRules = "&b < a &B < A"; + // Parts of speech and complex form types ship in the template (CreateProjectFromTemplate) with // liblcm's canonical Ids. We resolve the ones the demo needs (Noun, Compound) by name rather than // hard-coding those Ids, so this stays correct regardless of what the template assigns. @@ -20,8 +23,8 @@ public static async Task Seed(IServiceProvider provider, CrdtProject _) private static async Task CreateWritingSystems(IMiniLcmApi api) { - // The template path already provides vernacular "de" and analysis "en", so we only add the - // remaining demo writing systems here. They're appended after the template's, preserving the + // The template path already provides vernacular "de" (with demo ICU collation) and analysis "en". + // Add the remaining demo writing systems. They're appended after the template's, preserving the // order users see (de, de-audio, de-ipa / en, fr). await api.CreateWritingSystem(new() { diff --git a/backend/FwLite/LcmCrdt/LcmCrdt.csproj b/backend/FwLite/LcmCrdt/LcmCrdt.csproj index c0942b7af5..3c6e9fa324 100644 --- a/backend/FwLite/LcmCrdt/LcmCrdt.csproj +++ b/backend/FwLite/LcmCrdt/LcmCrdt.csproj @@ -24,7 +24,7 @@ - + diff --git a/backend/FwLite/LcmCrdt/Project/ProjectTemplate.cs b/backend/FwLite/LcmCrdt/Project/ProjectTemplate.cs index f99d24ddb4..f5ed42e42e 100644 --- a/backend/FwLite/LcmCrdt/Project/ProjectTemplate.cs +++ b/backend/FwLite/LcmCrdt/Project/ProjectTemplate.cs @@ -22,7 +22,8 @@ public static class ProjectTemplate public static ProjectSnapshot CreateNewSnapshot( JsonSerializerOptions jsonSerializerOptions, WritingSystemId vernacularWs, - WritingSystemId? analysisWs = null) + WritingSystemId? analysisWs = null, + Func? configureVernacular = null) { var assembly = typeof(ProjectTemplate).Assembly; using var stream = assembly.GetManifestResourceStream(EmbeddedResourceName) @@ -33,12 +34,19 @@ public static ProjectSnapshot CreateNewSnapshot( ?? throw new InvalidOperationException("Project template snapshot deserialized to null."); // Merge the requested writing systems into the snapshot so they're created with everything else // in dependency order, rather than tacked on after the import. - return snapshot with { WritingSystems = WithRequestedWritingSystems(snapshot.WritingSystems, vernacularWs, analysisWs) }; + return snapshot with { WritingSystems = WithRequestedWritingSystems(snapshot.WritingSystems, vernacularWs, analysisWs, configureVernacular) }; } - private static WritingSystems WithRequestedWritingSystems(WritingSystems template, WritingSystemId vernacularWs, WritingSystemId? analysisWs) + private static WritingSystems WithRequestedWritingSystems( + WritingSystems template, + WritingSystemId vernacularWs, + WritingSystemId? analysisWs, + Func? configureVernacular = null) { - WritingSystem[] vernacular = [.. template.Vernacular, DefaultWritingSystem(vernacularWs, WritingSystemType.Vernacular)]; + var vernacularWsEntity = DefaultWritingSystem(vernacularWs, WritingSystemType.Vernacular); + if (configureVernacular is not null) + vernacularWsEntity = configureVernacular(vernacularWsEntity); + WritingSystem[] vernacular = [.. template.Vernacular, vernacularWsEntity]; var analysis = template.Analysis; // The template already ships English analysis; only add the requested analysis WS if it's a different one. if (analysisWs is { } aws && !analysis.Any(ws => ws.WsId == aws))