feat: enhance normalization for legacy TableData scripts to support multi-line INSERT statements and update related tests

zacateras · zacateras · commit 849cdfb41123 · 2026-04-09T17:54:18.000+02:00
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -9,7 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/).
 
 ### Fixed
 - Trailing semicolon differences on `INSERT` statement lines in data scripts are now suppressed during comparison normalization; scripts emitted with and without statement terminators compare as compatible (#47).
-- Legacy `TableData` scripts now compare as compatible when they differ from canonical output only by `SET IDENTITY_INSERT` semicolons or top-level `N'...'` string literal prefixes.
+- Legacy `TableData` scripts now compare as compatible when they differ from canonical output only by `SET IDENTITY_INSERT` semicolons or top-level `N'...'` string literal prefixes, including inside multi-line `INSERT ... VALUES (...)` statements.
 - Whitespace-only separator lines now compare as compatible with empty blank lines during `status` and `diff`.
 - Preserve reference banner-comment formatting and module-declaration identifier quoting during programmable-object compatibility reconciliation.
 - Preserve compatible computed-column arithmetic grouping parentheses during table compatibility reconciliation.
diff --git a/specs/04-scripting.md b/specs/04-scripting.md
@@ -779,7 +779,7 @@ When compatibility reference files are available, `sqlct` MAY apply reconciliati
 - Whitespace-only lines MUST be normalized to empty lines during comparison so that blank separators differing only by spaces or tabs compare as compatible.
 - Trailing semicolons on `INSERT` statement lines MUST be stripped by comparison normalization so that scripts emitted with and without statement terminators compare as compatible.
 - For `TableData`, trailing semicolons on `SET IDENTITY_INSERT` lines MUST also be stripped by comparison normalization.
-- For `TableData`, comparison normalization MUST treat legacy top-level `N'...'` string literals inside `INSERT ... VALUES (...)` as compatible with canonical `'...'` literals; canonical script generation remains governed by Section 8.26.
+- For `TableData`, comparison normalization MUST treat legacy top-level `N'...'` string literals inside single-line or multi-line `INSERT ... VALUES (...)` statements as compatible with canonical `'...'` literals; canonical script generation remains governed by Section 8.26.
 
 ## 11. Error and Unsupported Behavior
 - Missing SQL object metadata for requested object MUST fail with an error.
diff --git a/src/SqlChangeTracker/Sync/SyncCommandService.cs b/src/SqlChangeTracker/Sync/SyncCommandService.cs
@@ -1792,25 +1792,28 @@ internal static string NormalizeForComparison(string script, string? objectType)
         }
 
         var isTableData = string.Equals(objectType, TableDataObjectType, StringComparison.OrdinalIgnoreCase);
-        if (!normalized.Contains("INSERT ", StringComparison.OrdinalIgnoreCase) &&
-            (!isTableData || !normalized.Contains("SET IDENTITY_INSERT ", StringComparison.OrdinalIgnoreCase)))
+        var joined = string.Join("\n", lines);
+        if (isTableData)
         {
-            return string.Join("\n", lines);
+            return !joined.Contains("INSERT ", StringComparison.OrdinalIgnoreCase) &&
+                   !joined.Contains("SET IDENTITY_INSERT ", StringComparison.OrdinalIgnoreCase)
+                ? joined
+                : NormalizeLegacyTableDataScript(joined);
+        }
+
+        if (!joined.Contains("INSERT ", StringComparison.OrdinalIgnoreCase))
+        {
+            return joined;
         }
 
         for (var i = 0; i < lines.Length; i++)
         {
             var line = lines[i];
-            if (line.EndsWith(';') && (LineStartsWithInsert(line) || (isTableData && LineStartsWithIdentityInsert(line))))
+            if (line.EndsWith(';') && LineStartsWithInsert(line))
             {
                 line = line[..^1];
             }
 
-            if (isTableData && LineStartsWithInsert(line))
-            {
-                line = NormalizeLegacyTableDataInsertLine(line);
-            }
-
             lines[i] = line;
         }
 
@@ -1837,7 +1840,165 @@ private static bool LineStartsWithIdentityInsert(string line)
                line.AsSpan(pos, prefix.Length).Equals(prefix, StringComparison.OrdinalIgnoreCase);
     }
 
-    private static string NormalizeLegacyTableDataInsertLine(string line)
+    private static string NormalizeLegacyTableDataScript(string script)
+    {
+        var builder = new StringBuilder(script.Length);
+        var position = 0;
+        while (position < script.Length)
+        {
+            var lineEnd = script.IndexOf('\n', position);
+            var segmentEnd = lineEnd >= 0 ? lineEnd : script.Length;
+            var line = script.Substring(position, segmentEnd - position);
+
+            if (LineStartsWithIdentityInsert(line))
+            {
+                builder.Append(StripTrailingSemicolon(line));
+                if (lineEnd >= 0)
+                {
+                    builder.Append('\n');
+                    position = lineEnd + 1;
+                }
+                else
+                {
+                    position = script.Length;
+                }
+
+                continue;
+            }
+
+            if (LineStartsWithInsert(line))
+            {
+                var insertRange = TryFindInsertValuesStatementRange(script, position);
+                if (insertRange.HasValue)
+                {
+                    var statement = script.Substring(
+                        position,
+                        insertRange.Value.StatementEndExclusive - position);
+                    builder.Append(NormalizeLegacyTableDataInsertStatement(statement));
+                    position = insertRange.Value.ConsumedEndExclusive;
+                    continue;
+                }
+            }
+
+            builder.Append(line);
+            if (lineEnd >= 0)
+            {
+                builder.Append('\n');
+                position = lineEnd + 1;
+            }
+            else
+            {
+                position = script.Length;
+            }
+        }
+
+        return builder.ToString();
+    }
+
+    private static string StripTrailingSemicolon(string line)
+        => line.EndsWith(';') ? line[..^1] : line;
+
+    private static (int StatementEndExclusive, int ConsumedEndExclusive)? TryFindInsertValuesStatementRange(
+        string script,
+        int start)
+    {
+        var valuesKeywordIndex = script.IndexOf("VALUES", start, StringComparison.OrdinalIgnoreCase);
+        if (valuesKeywordIndex < 0)
+        {
+            return null;
+        }
+
+        var valuesOpenParenIndex = script.IndexOf('(', valuesKeywordIndex);
+        if (valuesOpenParenIndex < 0)
+        {
+            return null;
+        }
+
+        var inSingleQuotedString = false;
+        var inBracketedIdentifier = false;
+        var parenDepth = 0;
+        for (var i = valuesOpenParenIndex; i < script.Length; i++)
+        {
+            var ch = script[i];
+            if (inSingleQuotedString)
+            {
+                if (ch == '\'')
+                {
+                    if (i + 1 < script.Length && script[i + 1] == '\'')
+                    {
+                        i++;
+                    }
+                    else
+                    {
+                        inSingleQuotedString = false;
+                    }
+                }
+
+                continue;
+            }
+
+            if (inBracketedIdentifier)
+            {
+                if (ch == ']')
+                {
+                    inBracketedIdentifier = false;
+                }
+
+                continue;
+            }
+
+            if (ch == '[')
+            {
+                inBracketedIdentifier = true;
+                continue;
+            }
+
+            if (ch == '\'')
+            {
+                inSingleQuotedString = true;
+                continue;
+            }
+
+            if (ch == '(')
+            {
+                parenDepth++;
+                continue;
+            }
+
+            if (ch != ')')
+            {
+                continue;
+            }
+
+            parenDepth--;
+            if (parenDepth != 0)
+            {
+                continue;
+            }
+
+            var statementEndExclusive = i + 1;
+            var consumedEndExclusive = statementEndExclusive;
+            while (consumedEndExclusive < script.Length && script[consumedEndExclusive] is ' ' or '\t')
+            {
+                consumedEndExclusive++;
+            }
+
+            if (consumedEndExclusive < script.Length && script[consumedEndExclusive] == ';')
+            {
+                consumedEndExclusive++;
+            }
+            else
+            {
+                consumedEndExclusive = statementEndExclusive;
+            }
+
+            return (statementEndExclusive, consumedEndExclusive);
+        }
+
+        return null;
+    }
+
+    private static string NormalizeLegacyTableDataInsertStatement(string line)
     {
         var valuesKeywordIndex = line.IndexOf("VALUES", StringComparison.OrdinalIgnoreCase);
         if (valuesKeywordIndex < 0)
@@ -1938,7 +2099,7 @@ private static bool IsTopLevelInsertStringPrefixBoundary(string line, int values
         for (var i = prefixIndex - 1; i > valuesOpenParenIndex; i--)
         {
             var ch = line[i];
-            if (ch is ' ' or '\t')
+            if (char.IsWhiteSpace(ch))
             {
                 continue;
             }
diff --git a/tests/SqlChangeTracker.Tests/Sync/SyncCommandServiceTests.cs b/tests/SqlChangeTracker.Tests/Sync/SyncCommandServiceTests.cs
@@ -841,6 +841,45 @@ public void BuildUnifiedDiff_TableData_SuppressesLegacyIdentityInsertAndUnicodeL
         Assert.Empty(diff);
     }
 
+    [Fact]
+    public void NormalizeForComparison_TableData_NormalizesLegacyPrefixesInMultilineInsertValues()
+    {
+        var canonical = SyncCommandService.NormalizeForComparison(
+            "INSERT INTO [dbo].[Variable] ([Description], [GroupCode], [Name], [Flag]) VALUES ('Line 1\n" +
+            "Line 2\n" +
+            "0 - empty', 'ScoreApl', 'EmploymentBasis', 0);",
+            SyncCommandService.TableDataObjectType);
+        var legacy = SyncCommandService.NormalizeForComparison(
+            "INSERT INTO [dbo].[Variable] ([Description], [GroupCode], [Name], [Flag]) VALUES ('Line 1\n" +
+            "Line 2\n" +
+            "0 - empty', N'ScoreApl', N'EmploymentBasis', 0)",
+            SyncCommandService.TableDataObjectType);
+
+        Assert.Equal(canonical, legacy);
+    }
+
+    [Fact]
+    public void BuildUnifiedDiff_TableData_SuppressesLegacyPrefixesInMultilineInsertValues()
+    {
+        var source =
+            "INSERT INTO [dbo].[Variable] ([Description], [GroupCode], [Name], [Flag]) VALUES ('Line 1\n" +
+            "Line 2\n" +
+            "0 - empty', 'ScoreApl', 'EmploymentBasis', 0);";
+        var target =
+            "INSERT INTO [dbo].[Variable] ([Description], [GroupCode], [Name], [Flag]) VALUES ('Line 1\n" +
+            "Line 2\n" +
+            "0 - empty', N'ScoreApl', N'EmploymentBasis', 0)";
+
+        var diff = SyncCommandService.BuildUnifiedDiff(
+            SyncCommandService.TableDataObjectType,
+            "db",
+            "folder",
+            source,
+            target);
+
+        Assert.Empty(diff);
+    }
+
     [Fact]
     public void NormalizeForComparison_DoesNotNormalizeUnicodeLiteralPrefixesOutsideTableData()
     {
