From a38e336161c2bc26d333d01e00f35a754874deab Mon Sep 17 00:00:00 2001 From: Tim Haasdyk Date: Tue, 7 Jul 2026 10:06:18 +0200 Subject: [PATCH 1/2] Bind SQLite timestamp parameters as the exact stored text linq2db renders DateTime/DateTimeOffset parameters truncated to milliseconds while Microsoft.Data.Sqlite stores 7-digit TEXT and SQLite's strftime rounds to the nearest millisecond, so timestamp comparisons could misclassify rows around the parameter value. In lexbox this made WhereAfter via ToLinqToDB() classify the target commit as after itself and delete it during snapshot regeneration. Binding parameters as the exact stored text makes both sides of a comparison normalize identically. linq2db comparisons remain millisecond-grained (strftime '%f' wraps both sides), so exact same-millisecond ordering still requires EF. Co-Authored-By: Claude Fable 5 --- src/SIL.Harmony.Linq2db/Linq2dbKernel.cs | 20 ++++++- src/SIL.Harmony.Tests/DbContextTests.cs | 67 ++++++++++++++++++++++-- 2 files changed, 81 insertions(+), 6 deletions(-) diff --git a/src/SIL.Harmony.Linq2db/Linq2dbKernel.cs b/src/SIL.Harmony.Linq2db/Linq2dbKernel.cs index a4d9f8f..c6a9d31 100644 --- a/src/SIL.Harmony.Linq2db/Linq2dbKernel.cs +++ b/src/SIL.Harmony.Linq2db/Linq2dbKernel.cs @@ -1,4 +1,5 @@ -using SIL.Harmony.Core; +using System.Globalization; +using SIL.Harmony.Core; using LinqToDB.EntityFrameworkCore; using LinqToDB.Extensions.Logging; using LinqToDB.Mapping; @@ -33,6 +34,23 @@ public static DbContextOptionsBuilder UseLinqToDbCrdt(this DbContextOptionsBuild dt => new DateTimeOffset(dt.Ticks, TimeSpan.Zero)) .Build(); + //bind timestamp parameters as the exact TEXT Microsoft.Data.Sqlite writes for DateTime columns + //("yyyy-MM-dd HH:mm:ss.FFFFFFF"): linq2db's own rendering truncates to milliseconds while + //SQLite's strftime rounds, so comparisons against stored values can misclassify rows + //(e.g. WhereAfter matching the target commit itself). Even with identical text, linq2db wraps + //SQLite timestamp comparisons in strftime('...%f'), which is millisecond-grained — use EF for + //comparisons that must distinguish same-millisecond commits. + mappingSchema.SetConvertExpression((DateTime dt) => new LinqToDB.Data.DataParameter + { + Value = dt.ToString("yyyy-MM-dd HH:mm:ss.FFFFFFF", CultureInfo.InvariantCulture), + DataType = LinqToDB.DataType.Text + }); + mappingSchema.SetConvertExpression((DateTimeOffset dto) => new LinqToDB.Data.DataParameter + { + Value = dto.UtcDateTime.ToString("yyyy-MM-dd HH:mm:ss.FFFFFFF", CultureInfo.InvariantCulture), + DataType = LinqToDB.DataType.Text + }); + var loggerFactory = provider.GetService(); if (loggerFactory is not null) optionsBuilder.AddCustomOptions(dataOptions => dataOptions.UseLoggerFactory(loggerFactory)); diff --git a/src/SIL.Harmony.Tests/DbContextTests.cs b/src/SIL.Harmony.Tests/DbContextTests.cs index 3b7677d..9e00483 100644 --- a/src/SIL.Harmony.Tests/DbContextTests.cs +++ b/src/SIL.Harmony.Tests/DbContextTests.cs @@ -68,6 +68,67 @@ await DbContext.Set().ToLinqToDBTable().AsValueInsertable() [InlineData(TimeSpan.TicksPerMicrosecond)] [InlineData(1)] public async Task CanFilterCommitsByDateTime(double scale) + { + var baseDateTime = await SeedCommitsAtTickScale(scale); + var commits = await DbContext.Commits + .Where(c => c.HybridDateTime.DateTime > baseDateTime.Add(new TimeSpan((long)(25 * scale)))) + .OrderBy(c => c.HybridDateTime.DateTime) + .ToArrayAsyncEF(TestContext.Current.CancellationToken); + commits.Should().HaveCount(24); + } + + //no sub-millisecond scales here: linq2db wraps SQLite timestamp comparisons in + //strftime('%Y-%m-%d %H:%M:%f', ...), which normalizes both sides to milliseconds + [Theory] + [InlineData(TimeSpan.TicksPerHour)] + [InlineData(TimeSpan.TicksPerMinute)] + [InlineData(TimeSpan.TicksPerSecond)] + [InlineData(TimeSpan.TicksPerMillisecond)] + public async Task CanFilterCommitsByDateTimeViaLinq2db(double scale) + { + var baseDateTime = await SeedCommitsAtTickScale(scale); + var cutoff = baseDateTime.Add(new TimeSpan((long)(25 * scale))); + var commits = await DbContext.Commits + .ToLinqToDB() + .Where(c => c.HybridDateTime.DateTime > cutoff) + .OrderBy(c => c.HybridDateTime.DateTime) + .ToArrayAsyncLinqToDB(TestContext.Current.CancellationToken); + commits.Should().HaveCount(24); + } + + //commits sharing a millisecond compare equal under linq2db (see above), so WhereAfter must + //never see the target itself as "after": the parameter has to render byte-identical to the + //stored text or strftime can normalize the two differently and delete the target commit + [Fact] + public async Task WhereAfterViaLinq2dbExcludesTheTargetCommit() + { + //sub-millisecond part must be >= 0.5ms: SQLite's strftime rounds to the nearest millisecond + //while a truncating parameter rendering lands one millisecond lower, which is the disagreement + var baseDateTime = new DateTimeOffset(2000, 1, 1, 1, 11, 11, TimeSpan.Zero).Add(TimeSpan.FromTicks(6000)); + for (int i = 0; i < 50; i++) + { + DbContext.Add(new Commit + { + ClientId = Guid.NewGuid(), + HybridDateTime = new HybridDateTime(baseDateTime.Add(TimeSpan.FromTicks(i)), 0) + }); + } + + await DbContext.SaveChangesAsync(TestContext.Current.CancellationToken); + var commits = await DbContext.Commits.ToArrayAsyncEF(TestContext.Current.CancellationToken); + foreach (var target in commits) + { + var after = await DbContext.Commits + .WhereAfter(target) + .ToLinqToDB() + .Select(c => c.Id) + .ToArrayAsyncLinqToDB(TestContext.Current.CancellationToken); + after.Should().NotContain(target.Id, + $"WhereAfter via linq2db should never include the target commit itself (target at {target.HybridDateTime.DateTime:O})"); + } + } + + private async Task SeedCommitsAtTickScale(double scale) { var baseDateTime = new DateTimeOffset(2000, 1, 1, 1, 11, 11, TimeSpan.Zero); for (int i = 0; i < 50; i++) @@ -81,10 +142,6 @@ public async Task CanFilterCommitsByDateTime(double scale) } await DbContext.SaveChangesAsync(TestContext.Current.CancellationToken); - var commits = await DbContext.Commits - .Where(c => c.HybridDateTime.DateTime > baseDateTime.Add(new TimeSpan((long)(25 * scale)))) - .OrderBy(c => c.HybridDateTime.DateTime) - .ToArrayAsyncEF(TestContext.Current.CancellationToken); - commits.Should().HaveCount(24); + return baseDateTime; } } From dcd968a694a4d0c37e0c3dc2ca4877ad8c736e6b Mon Sep 17 00:00:00 2001 From: Tim Haasdyk Date: Tue, 7 Jul 2026 11:15:50 +0200 Subject: [PATCH 2/2] Characterize the millisecond limit of linq2db timestamp comparisons An active test alarms when a linq2db upgrade changes the translation, unlike a skipped failing test, which never runs. Co-Authored-By: Claude Fable 5 --- src/SIL.Harmony.Tests/DbContextTests.cs | 31 +++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/src/SIL.Harmony.Tests/DbContextTests.cs b/src/SIL.Harmony.Tests/DbContextTests.cs index 9e00483..2cbee52 100644 --- a/src/SIL.Harmony.Tests/DbContextTests.cs +++ b/src/SIL.Harmony.Tests/DbContextTests.cs @@ -128,6 +128,37 @@ public async Task WhereAfterViaLinq2dbExcludesTheTargetCommit() } } + //characterizes the limitation documented in Linq2dbKernel rather than a desired behavior: + //if the linq2db side starts seeing the later commit, its SQLite translation has changed — + //revisit whether timestamp comparisons still need to stay in EF + [Fact] + public async Task Linq2dbCannotOrderCommitsWithinTheSameMillisecond() + { + var baseDateTime = new DateTimeOffset(2000, 1, 1, 1, 11, 11, TimeSpan.Zero); + var earlier = new Commit(Guid.NewGuid()) + { + ClientId = Guid.NewGuid(), + HybridDateTime = new HybridDateTime(baseDateTime.Add(TimeSpan.FromTicks(100)), 0) + }; + var later = new Commit(Guid.NewGuid()) + { + ClientId = Guid.NewGuid(), + HybridDateTime = new HybridDateTime(baseDateTime.Add(TimeSpan.FromTicks(200)), 0) + }; + DbContext.AddRange(earlier, later); + await DbContext.SaveChangesAsync(TestContext.Current.CancellationToken); + + var linq2dbCount = await DbContext.Commits.ToLinqToDB() + .Where(c => c.HybridDateTime.DateTime > earlier.HybridDateTime.DateTime) + .CountAsyncLinqToDB(TestContext.Current.CancellationToken); + linq2dbCount.Should().Be(0, "linq2db compares SQLite timestamps at millisecond precision (strftime '%f')"); + + var efCount = await DbContext.Commits + .Where(c => c.HybridDateTime.DateTime > earlier.HybridDateTime.DateTime) + .CountAsyncEF(TestContext.Current.CancellationToken); + efCount.Should().Be(1, "EF compares the stored text at full precision"); + } + private async Task SeedCommitsAtTickScale(double scale) { var baseDateTime = new DateTimeOffset(2000, 1, 1, 1, 11, 11, TimeSpan.Zero);