Add FLEx-compatible per-writing-system collation to FwLite

CONTEXT-MAP.md

@@ -4,4 +4,4 @@ Domain glossaries for this monorepo. Each `CONTEXT.md` is a glossary only — no
 
 ## Contexts
 
-- [FwLite Commenting](./backend/FwLite/CONTEXT.md) — collaborative comments on dictionary entries, senses, and example sentences
+- [FwLite](./backend/FwLite/CONTEXT.md) — dictionary editing concerns including collation and collaborative comments

backend/Directory.Packages.props

@@ -29,6 +29,7 @@
     <PackageVersion Include="HotChocolate.Types.OffsetPagination" Version="$(HotChocolateVersion)" />
     <PackageVersion Include="Humanizer.Core" Version="3.0.10" />
     <PackageVersion Include="icu.net" Version="3.0.1" />
+    <PackageVersion Include="ICU4N.Collation" Version="60.1.0-alpha.356" />
     <PackageVersion Include="linq2db.Extensions" Version="6.3.0" />
     <PackageVersion Include="linq2db.EntityFrameworkCore" Version="10.4.0" />
     <PackageVersion Include="L10NSharp" Version="10.0.0-beta0012" />

backend/FwLite/CONTEXT.md

@@ -1,4 +1,46 @@
-# FwLite Commenting
+# FwLite
+
+## Collation
+
+How strings in a writing system are compared when ordering dictionary content. Imported from FLEx per writing system; stored on `WritingSystem` without LDML at runtime.
+
+**Collation**:
+The rules that govern string comparison for a writing system (ICU tailoring or .NET locale alias). Distinct from list position and from the user action of sorting entries.
+_Avoid_: Sort rules (when meaning collation specifically), custom sorting (as a model term)
+
+**Writing system order**:
+Where a writing system appears in the project's vernacular or analysis list (`WritingSystem.Order`). Not how strings alphabetize.
+_Avoid_: Confusing with collation
+
+**Entry sort**:
+The user-facing action of ordering the entry list (e.g. by headword in a chosen writing system). Uses the selected writing system's collation for headword comparison.
+_Avoid_: Collation (when meaning the user action)
+
+**Collation scope (v1)**:
+Collation governs headword entry sort (`SortField.Headword`) and the alphabetical tie-break when search results fall through to headword order (`SortField.SearchRelevance`). It does not govern search matching (whether a query matches text) or FTS ranking.
+_Avoid_: Applying collation to `ContainsDiacriticMatch`, prefix/contains predicates, or FTS rank
+
+**Imported collation**:
+Collation metadata synced from FLEx onto `WritingSystem`. Two optional fields: compiled ICU rules, or a .NET locale alias for "same as another language." No LDML at FwLite runtime.
+_Avoid_: Storing LDML, simple-rule source text, or import metadata
+
+**Legacy collation fallback**:
+When neither field is set, headword comparison uses the pre-existing FwLite behavior (`CultureInfo` from `WsId`, case-insensitive with lowercase-before-uppercase tie-break). FLEx "Default Ordering" is imported as this fallback too — FwLite does not replicate FLEx's empty-rule ICU default until we choose to.
+_Avoid_: Treating `null` and `""` as different states for `IcuCollationRules`
+
+**Collation compare (imported)**:
+When `IcuCollationRules` or `SystemCollationLocale` is set, use ICU4N (`RuleBasedCollator` / locale `Collator`) `Compare` with no legacy case tie-break layered on top. Matches FLEx for custom and other-language modes. FwData/FLEx bridge code still uses icu.net separately.
+_Avoid_: Applying case-insensitive or lowercase-first logic on top of imported ICU4N collation
+
+**Collation import (from FLEx)**:
+Populated when mapping `CoreWritingSystemDefinition` → `WritingSystem` in the FwData bridge. Custom modes: store non-empty compiled ICU rules only. Other-language: store .NET locale tag only. Default ordering: leave both fields null (legacy fallback). Import-only — no write-back to fwdata LDML.
+_Avoid_: Parsing LDML in FwLite; persisting empty `IcuCollationRules` for default ordering
+
+**Collation write-back**:
+Explicitly a no-op. The fwdata `UpdateWritingSystemProxy` overrides collation fields and does not write them to LDML.
+_Avoid_: Assuming bidirectional collation sync
+
+## Commenting
 
 Collaborative discussion attached to dictionary data (entries, senses, example sentences). Comments sync via Harmony; read status is local-only per device (not synced, not per Lexbox user account).
 

backend/FwLite/FwDataMiniLcmBridge/Api/FwDataMiniLcmApi.cs

@@ -2,6 +2,7 @@
 using System.Globalization;
 using System.Text;
 using FwDataMiniLcmBridge.Api.UpdateProxy;
+using FwDataMiniLcmBridge.Collation;
 using FwDataMiniLcmBridge.LcmUtils;
 using FwDataMiniLcmBridge.Media;
 using Gridify;
@@ -113,6 +114,7 @@ public Task<WritingSystems> GetWritingSystems()
 
     private WritingSystem FromLcmWritingSystem(CoreWritingSystemDefinition ws, WritingSystemType type, int index = default)
     {
+        var (icuCollationRules, systemCollationLocale) = WritingSystemCollationExtractor.Extract(ws);
         return new WritingSystem
         {
             Id = Guid.Empty,
@@ -125,7 +127,9 @@ private WritingSystem FromLcmWritingSystem(CoreWritingSystemDefinition ws, Writi
             Name = ws.LanguageTag,
             Abbreviation = ws.Abbreviation,
             Font = ws.DefaultFontName,
-            Exemplars = ws.CharacterSets.FirstOrDefault(s => s.Type == "index")?.Characters.ToArray() ?? []
+            Exemplars = ws.CharacterSets.FirstOrDefault(s => s.Type == "index")?.Characters.ToArray() ?? [],
+            IcuCollationRules = icuCollationRules,
+            SystemCollationLocale = systemCollationLocale,
         };
     }
 

backend/FwLite/FwDataMiniLcmBridge/Api/UpdateProxy/UpdateWritingSystemProxy.cs

@@ -47,4 +47,17 @@ public override required string Font
             }
         }
     }
+
+    // Collation is import-only from FLEx LDML; MiniLcm updates do not write back to fwdata.
+    public override string? IcuCollationRules
+    {
+        get => null;
+        set { }
+    }
+
+    public override string? SystemCollationLocale
+    {
+        get => null;
+        set { }
+    }
 }

backend/FwLite/FwDataMiniLcmBridge/Collation/WritingSystemCollationExtractor.cs

@@ -0,0 +1,21 @@
+using SIL.LCModel.Core.WritingSystems;
+using SIL.WritingSystems;
+
+namespace FwDataMiniLcmBridge.Collation;
+
+public static class WritingSystemCollationExtractor
+{
+    public static (string? IcuCollationRules, string? SystemCollationLocale) Extract(CoreWritingSystemDefinition ws)
+    {
+        var cd = ws.DefaultCollation;
+        cd.Validate(out _);
+
+        return cd switch
+        {
+            SystemCollationDefinition sys => (null, sys.LanguageTag),
+            RulesCollationDefinition rules when !string.IsNullOrEmpty(rules.CollationRules)
+                => (rules.CollationRules, null),
+            _ => (null, null)
+        };
+    }
+}

backend/FwLite/FwLiteWeb/FwLiteWebServer.cs

@@ -24,11 +24,8 @@ public static WebApplication SetupAppServer(WebApplicationOptions options, Actio
         var builder = WebApplication.CreateBuilder(options);
         if (!builder.Environment.IsDevelopment() && options.Args?.Contains("--urls") != true && string.IsNullOrEmpty(builder.Configuration["http_ports"]))
             builder.WebHost.UseUrls("http://127.0.0.1:0");
-        if (builder.Environment.IsDevelopment())
-        {
-            //do this early so we catch bugs on startup
-            ProjectLoader.Init();
-        }
+        // ICU required for FwData bridge (liblcm / icu.net), not for CRDT collation (ICU4N).
+        ProjectLoader.Init();
 
         builder.ConfigureDev<AuthConfig>(config =>
             config.LexboxServers = [

backend/FwLite/LcmCrdt.Tests/Changes/ChangeDeserializationRegressionData.latest.verified.txt

@@ -756,10 +756,27 @@
     "Exemplars": [
       "Berkshire"
     ],
+    "IcuCollationRules": null,
+    "SystemCollationLocale": null,
     "Type": 1,
     "Order": 0.7988004089201804,
     "EntityId": "0fccfaaa-1e55-f6fb-966f-5c1e6caedf45"
   },
+  {
+    "$type": "CreateWritingSystemChange",
+    "WsId": "prl",
+    "Name": "Investment Account",
+    "Abbreviation": "JBOD",
+    "Font": "SCSI",
+    "Exemplars": [
+      "Games \u0026 Industrial"
+    ],
+    "IcuCollationRules": "parsing",
+    "SystemCollationLocale": "interactive",
+    "Type": 1,
+    "Order": 0.10446673530044903,
+    "EntityId": "f6928dbf-09ad-c0e8-a12a-23616641b071"
+  },
   {
     "$type": "AddComplexFormTypeChange",
     "ComplexFormType": {

backend/FwLite/LcmCrdt.Tests/Changes/ChangeDeserializationRegressionData.legacy.verified.txt

@@ -625,5 +625,65 @@
       ],
       "EntityId": "f27a032b-e0f6-f389-b466-959bdb17086d"
     }
+  },
+  {
+    "Input": {
+      "$type": "CreateWritingSystemChange",
+      "WsId": "zms",
+      "Name": "quantify",
+      "Abbreviation": "Ergonomic",
+      "Font": "software",
+      "Exemplars": [
+        "Berkshire"
+      ],
+      "Type": 1,
+      "Order": 0.7988004089201804,
+      "EntityId": "0fccfaaa-1e55-f6fb-966f-5c1e6caedf45"
+    },
+    "Output": {
+      "$type": "CreateWritingSystemChange",
+      "WsId": "zms",
+      "Name": "quantify",
+      "Abbreviation": "Ergonomic",
+      "Font": "software",
+      "Exemplars": [
+        "Berkshire"
+      ],
+      "IcuCollationRules": null,
+      "SystemCollationLocale": null,
+      "Type": 1,
+      "Order": 0.7988004089201804,
+      "EntityId": "0fccfaaa-1e55-f6fb-966f-5c1e6caedf45"
+    }
+  },
+  {
+    "Input": {
+      "$type": "CreateWritingSystemChange",
+      "WsId": "zms",
+      "Name": "quantify",
+      "Abbreviation": "Ergonomic",
+      "Font": "software",
+      "Exemplars": [
+        "Berkshire"
+      ],
+      "Type": 1,
+      "Order": 0.7988004089201804,
+      "EntityId": "0fccfaaa-1e55-f6fb-966f-5c1e6caedf45"
+    },
+    "Output": {
+      "$type": "CreateWritingSystemChange",
+      "WsId": "zms",
+      "Name": "quantify",
+      "Abbreviation": "Ergonomic",
+      "Font": "software",
+      "Exemplars": [
+        "Berkshire"
+      ],
+      "IcuCollationRules": null,
+      "SystemCollationLocale": null,
+      "Type": 1,
+      "Order": 0.7988004089201804,
+      "EntityId": "0fccfaaa-1e55-f6fb-966f-5c1e6caedf45"
+    }
   }
 ]

backend/FwLite/LcmCrdt.Tests/Culture/WritingSystemCollatorProviderTests.cs

@@ -0,0 +1,89 @@
+using LcmCrdt.Culture;
+using Microsoft.Extensions.Caching.Memory;
+using Microsoft.Extensions.DependencyInjection;
+using MiniLcm.Culture;
+using MiniLcm.Models;
+using SIL.WritingSystems;
+
+namespace LcmCrdt.Tests.Culture;
+
+public class WritingSystemCollatorProviderTests
+{
+    private static IWritingSystemCollatorProvider CreateProvider()
+    {
+        var services = new ServiceCollection();
+        services.AddLogging();
+        services.AddMemoryCache();
+        services.AddSingleton<IMiniLcmCultureProvider, LcmCrdtCultureProvider>();
+        services.AddSingleton<IWritingSystemCollatorProvider, WritingSystemCollatorProvider>();
+        return services.BuildServiceProvider().GetRequiredService<IWritingSystemCollatorProvider>();
+    }
+
+    private static WritingSystem BaseWs() => new()
+    {
+        Id = Guid.NewGuid(),
+        WsId = "en",
+        Name = "English",
+        Abbreviation = "En",
+        Font = "Arial",
+        Type = WritingSystemType.Vernacular,
+    };
+
+    [Fact]
+    public void GetCollator_UsesIcu4NLocaleCollator_WhenLocaleSet()
+    {
+        var provider = CreateProvider();
+        var collator = provider.GetCollator(BaseWs() with { SystemCollationLocale = "de" });
+        collator.Should().BeOfType<Icu4NLocaleCollator>();
+    }
+
+    [Fact]
+    public void GetCollator_UsesIcu4NRulesCollator_WhenRulesSet()
+    {
+        var provider = CreateProvider();
+        var collator = provider.GetCollator(BaseWs() with { IcuCollationRules = "&a < b" });
+        collator.Should().BeOfType<Icu4NRulesCollator>();
+    }
+
+    [Fact]
+    public void GetCollator_UsesLegacyFallback_WhenCollationUnset()
+    {
+        var provider = CreateProvider();
+        var collator = provider.GetCollator(BaseWs());
+        collator.Should().NotBeOfType<Icu4NLocaleCollator>();
+        collator.Should().NotBeOfType<Icu4NRulesCollator>();
+    }
+
+    [Fact]
+    public void Icu4NRulesCollator_SortsByCustomRules()
+    {
+        var provider = CreateProvider();
+        var collator = provider.GetCollator(BaseWs() with { IcuCollationRules = "&z < a" });
+        collator.Compare("z", "a").Should().BeLessThan(0);
+    }
+
+    [Fact]
+    public void Icu4NRulesCollator_ReversesAAndB()
+    {
+        var provider = CreateProvider();
+        var collator = provider.GetCollator(BaseWs() with { IcuCollationRules = "&b < a &B < A" });
+        collator.Compare("Banane", "Apfel").Should().BeLessThan(0);
+    }
+
+    [Fact]
+    public void LegacyCollator_PrefersLowercaseOnCaseInsensitiveTie()
+    {
+        var provider = CreateProvider();
+        var collator = provider.GetCollator(BaseWs());
+        collator.Compare("Ab", "ab").Should().BeGreaterThan(0);
+    }
+
+    [Fact]
+    public void GetCollator_FallsBackToLegacy_WhenRulesInvalid()
+    {
+        var provider = CreateProvider();
+        var collator = provider.GetCollator(BaseWs() with { IcuCollationRules = "not valid icu rules <<<<" });
+        collator.Should().NotBeOfType<Icu4NRulesCollator>();
+        collator.Should().BeOfType<LegacyCompareInfoCollator>();
+    }
+}