From 4694fb0eb5c70d537edacbb056132dfd1d3ab15f Mon Sep 17 00:00:00 2001 From: Leila Lali Date: Wed, 17 Sep 2025 15:33:39 -0700 Subject: [PATCH 1/2] Fixing a bug with regex_Like function as boolean exp --- .github/BUG_FIXING_GUIDE.md | 11 ++ .github/PARSER_PREDICATE_RECOGNITION_FIX.md | 154 ++++++++++++++++++ .github/copilot-instructions.md | 2 + .../Parser/TSql/TSql80ParserBaseInternal.cs | 9 + .../Baselines170/RegexpLikeTests170.sql | 8 +- .../BooleanExpressionTests.sql | 6 + Test/SqlDom/Only170SyntaxTests.cs | 2 +- .../TestScripts/BooleanExpressionTests.sql | 5 + .../SqlDom/TestScripts/RegexpLikeTests170.sql | 6 +- 9 files changed, 200 insertions(+), 3 deletions(-) create mode 100644 .github/PARSER_PREDICATE_RECOGNITION_FIX.md diff --git a/.github/BUG_FIXING_GUIDE.md b/.github/BUG_FIXING_GUIDE.md index f3a3d0f..a1e929d 100644 --- a/.github/BUG_FIXING_GUIDE.md +++ b/.github/BUG_FIXING_GUIDE.md @@ -40,3 +40,14 @@ The process of fixing a bug, especially one that involves adding new syntax, fol * **d. Re-run the Tests**: Run the same test command again. This time, the tests should pass, confirming that the generated script matches the new baseline. By following these steps, you can ensure that new syntax is correctly parsed, represented in the AST, generated back into a script, and fully validated by the testing framework. + +## Special Case: Parser Predicate Recognition Issues + +If you encounter a bug where: +- An identifier-based predicate (like `REGEXP_LIKE`) works without parentheses: `WHERE REGEXP_LIKE('a', 'pattern')` ✅ +- But fails with parentheses: `WHERE (REGEXP_LIKE('a', 'pattern'))` ❌ +- The error is a syntax error near the closing parenthesis or semicolon + +This is likely a **parser predicate recognition issue**. The grammar and AST are correct, but the `IsNextRuleBooleanParenthesis()` function doesn't recognize the identifier-based predicate. + +**Solution**: Follow the [Parser Predicate Recognition Fix Guide](PARSER_PREDICATE_RECOGNITION_FIX.md) instead of the standard grammar modification workflow. diff --git a/.github/PARSER_PREDICATE_RECOGNITION_FIX.md b/.github/PARSER_PREDICATE_RECOGNITION_FIX.md new file mode 100644 index 0000000..acd96d6 --- /dev/null +++ b/.github/PARSER_PREDICATE_RECOGNITION_FIX.md @@ -0,0 +1,154 @@ +# Parser Predicate Recognition Bug Fix Guide + +This guide documents the specific pattern for fixing bugs where identifier-based predicates (like `REGEXP_LIKE`) are not properly recognized when wrapped in parentheses in boolean expressions. + +## Problem Description + +**Symptom**: Parentheses around identifier-based boolean predicates cause syntax errors. +- Example: `SELECT 1 WHERE (REGEXP_LIKE('a', 'pattern'))` fails to parse +- Works: `SELECT 1 WHERE REGEXP_LIKE('a', 'pattern')` (without parentheses) + +**Root Cause**: The `IsNextRuleBooleanParenthesis()` function in `TSql80ParserBaseInternal.cs` only recognizes: +- Keyword-based predicates (tokens): `LIKE`, `BETWEEN`, `CONTAINS`, `EXISTS`, etc. +- One identifier-based predicate: `IIF` +- But doesn't recognize newer identifier-based predicates like `REGEXP_LIKE` + +## Understanding the Fix + +### The `IsNextRuleBooleanParenthesis()` Function + +This function determines whether parentheses contain a boolean expression vs. a scalar expression. It scans forward from a `LeftParenthesis` token looking for boolean operators or predicates. + +**Location**: `SqlScriptDom/Parser/TSql/TSql80ParserBaseInternal.cs` + +**Key Logic**: +```csharp +case TSql80ParserInternal.Identifier: + // if identifier is IIF + if(NextTokenMatches(CodeGenerationSupporter.IIf)) + { + ++insideIIf; + } + // ADD NEW IDENTIFIER-BASED PREDICATES HERE + break; +``` + +### The Solution Pattern + +For identifier-based boolean predicates, add detection logic in the `Identifier` case: + +```csharp +case TSql80ParserInternal.Identifier: + // if identifier is IIF + if(NextTokenMatches(CodeGenerationSupporter.IIf)) + { + ++insideIIf; + } + // if identifier is REGEXP_LIKE + else if(NextTokenMatches(CodeGenerationSupporter.RegexpLike)) + { + if (caseDepth == 0 && topmostSelect == 0 && insideIIf == 0) + { + matches = true; + loop = false; + } + } + break; +``` + +## Step-by-Step Fix Process + +### 1. Reproduce the Issue +Create a test case to confirm the bug: +```sql +SELECT 1 WHERE (REGEXP_LIKE('a', 'pattern')); -- Should fail without fix +``` + +### 2. Identify the Predicate Constant +Find the predicate identifier in `CodeGenerationSupporter`: +```csharp +// In CodeGenerationSupporter.cs +public const string RegexpLike = "REGEXP_LIKE"; +``` + +### 3. Apply the Fix +Modify `TSql80ParserBaseInternal.cs` in the `IsNextRuleBooleanParenthesis()` method: + +**File**: `SqlScriptDom/Parser/TSql/TSql80ParserBaseInternal.cs` +**Method**: `IsNextRuleBooleanParenthesis()` +**Location**: Around line 808, in the `case TSql80ParserInternal.Identifier:` block + +Add the predicate detection logic following the pattern shown above. + +### 4. Update Test Cases +Add test cases covering the parentheses scenario: + +**Test Script**: `Test/SqlDom/TestScripts/RegexpLikeTests170.sql` +```sql +SELECT 1 WHERE (REGEXP_LIKE('a', '%pattern%')); +``` + +**Baseline**: `Test/SqlDom/Baselines170/RegexpLikeTests170.sql` +```sql +SELECT 1 +WHERE (REGEXP_LIKE ('a', '%pattern%')); +``` + +**Test Configuration**: Update error counts in `Only170SyntaxTests.cs` if the new test cases affect older parser versions. + +### 5. Build and Verify +```bash +# Build the parser +dotnet build SqlScriptDom/Microsoft.SqlServer.TransactSql.ScriptDom.csproj -c Debug + +# Run the specific test +dotnet test Test/SqlDom/UTSqlScriptDom.csproj --filter "FullyQualifiedName~SqlStudio.Tests.UTSqlScriptDom.SqlDomTests.TSql170SyntaxIn170ParserTest" -c Debug +``` + +## When to Apply This Pattern + +This fix pattern applies when: + +1. **Identifier-based predicates**: The predicate is defined as an identifier (not a keyword token) +2. **Boolean context**: The predicate returns a boolean value for use in WHERE clauses, CHECK constraints, etc. +3. **Parentheses fail**: The predicate works without parentheses but fails with parentheses +4. **Already implemented**: The predicate grammar and AST are already correctly implemented + +## Common Predicates That May Need This Fix + +- `REGEXP_LIKE` (✅ Fixed) +- Future identifier-based boolean functions +- Custom function predicates that return boolean values + +## Related Files Modified + +This type of fix typically involves: + +1. **Core Parser Logic**: + - `SqlScriptDom/Parser/TSql/TSql80ParserBaseInternal.cs` - Main fix + +2. **Test Infrastructure**: + - `Test/SqlDom/TestScripts/[TestName].sql` - Input test cases + - `Test/SqlDom/Baselines[Version]/[TestName].sql` - Expected output + - `Test/SqlDom/Only[Version]SyntaxTests.cs` - Test configuration + +3. **Potentially Affected**: + - `Test/SqlDom/TestScripts/BooleanExpressionTests.sql` - May need additional test cases + - `Test/SqlDom/BaselinesCommon/BooleanExpressionTests.sql` - Corresponding baselines + +## Verification Checklist + +- [ ] Parentheses syntax parses without errors +- [ ] Non-parentheses syntax still works +- [ ] Test suite passes for target SQL version +- [ ] Older SQL versions have appropriate error counts +- [ ] Related boolean expression tests still pass + +## Notes and Gotchas + +- **IIF Special Handling**: `IIF` has special logic (`++insideIIf`) because it's not a simple boolean predicate +- **Context Conditions**: The fix includes conditions (`caseDepth == 0 && topmostSelect == 0 && insideIIf == 0`) to ensure proper parsing context +- **Token vs Identifier**: Keyword predicates are handled as tokens, identifier predicates need special detection +- **Cross-Version Impact**: Adding test cases may increase error counts for older SQL Server parsers + +This pattern ensures that identifier-based boolean predicates work consistently with parentheses, maintaining parser compatibility across different syntactic contexts. \ No newline at end of file diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index 9347f5a..0d40204 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -45,6 +45,8 @@ ScriptDom is a library for parsing and generating T-SQL scripts. It is primarily ## Bug Fixing and Baseline Generation For a practical guide on fixing bugs, including the detailed workflow for generating test baselines, see the [Bug Fixing Guide](BUG_FIXING_GUIDE.md). +For specific parser predicate recognition issues (when identifier-based predicates like `REGEXP_LIKE` don't work with parentheses), see the [Parser Predicate Recognition Fix Guide](PARSER_PREDICATE_RECOGNITION_FIX.md). + ## Editing generated outputs, debugging generation - Never edit generated files permanently (they live under `obj/...`/CsGenIntermediateOutputPath). Instead change: - `.g` grammar files diff --git a/SqlScriptDom/Parser/TSql/TSql80ParserBaseInternal.cs b/SqlScriptDom/Parser/TSql/TSql80ParserBaseInternal.cs index 7415c4a..7c59643 100644 --- a/SqlScriptDom/Parser/TSql/TSql80ParserBaseInternal.cs +++ b/SqlScriptDom/Parser/TSql/TSql80ParserBaseInternal.cs @@ -808,6 +808,15 @@ protected bool IsNextRuleBooleanParenthesis() { ++insideIIf; } + // if identifier is REGEXP_LIKE + else if(NextTokenMatches(CodeGenerationSupporter.RegexpLike)) + { + if (caseDepth == 0 && topmostSelect == 0 && insideIIf == 0) + { + matches = true; + loop = false; + } + } break; case TSql80ParserInternal.LeftParenthesis: ++openParens; diff --git a/Test/SqlDom/Baselines170/RegexpLikeTests170.sql b/Test/SqlDom/Baselines170/RegexpLikeTests170.sql index 2faab93..8fb24cc 100644 --- a/Test/SqlDom/Baselines170/RegexpLikeTests170.sql +++ b/Test/SqlDom/Baselines170/RegexpLikeTests170.sql @@ -30,4 +30,10 @@ SELECT CASE WHEN NOT REGEXP_LIKE ('abc', '^a', NULL) THEN 1 ELSE 0 END AS is_mat SELECT CASE WHEN REGEXP_LIKE (NULL, '^a', 'c') THEN 1 ELSE 0 END AS is_match; -SELECT IIF (NOT REGEXP_LIKE ('abc', NULL), 1, 0) AS is_match; \ No newline at end of file +SELECT IIF (NOT REGEXP_LIKE ('abc', NULL), 1, 0) AS is_match; + +SELECT 1 +WHERE REGEXP_LIKE ('a', '^a'); + +SELECT 1 +WHERE (REGEXP_LIKE ('a', '%pattern%')); \ No newline at end of file diff --git a/Test/SqlDom/BaselinesCommon/BooleanExpressionTests.sql b/Test/SqlDom/BaselinesCommon/BooleanExpressionTests.sql index e9b639d..ff3958c 100644 --- a/Test/SqlDom/BaselinesCommon/BooleanExpressionTests.sql +++ b/Test/SqlDom/BaselinesCommon/BooleanExpressionTests.sql @@ -138,6 +138,12 @@ FROM Production.Document WHERE CONTAINS ((t1.c1), @a); + +GO +SELECT Title +FROM Production.Document +WHERE (CONTAINS ((t1.c1), @a)); + GO SELECT Title FROM Production.Document diff --git a/Test/SqlDom/Only170SyntaxTests.cs b/Test/SqlDom/Only170SyntaxTests.cs index 45cd6d8..d93dbc9 100644 --- a/Test/SqlDom/Only170SyntaxTests.cs +++ b/Test/SqlDom/Only170SyntaxTests.cs @@ -27,7 +27,7 @@ public partial class SqlDomTests new ParserTest170("VectorFunctionTests170.sql", nErrors80: 1, nErrors90: 1, nErrors100: 2, nErrors110: 2, nErrors120: 2, nErrors130: 2, nErrors140: 2, nErrors150: 2, nErrors160: 2), new ParserTest170("SecurityStatementExternalModelTests170.sql", nErrors80: 2, nErrors90: 17, nErrors100: 17, nErrors110: 17, nErrors120: 17, nErrors130: 17, nErrors140: 17, nErrors150: 17, nErrors160: 17), new ParserTest170("CreateEventSessionNotLikePredicate.sql", nErrors80: 2, nErrors90: 1, nErrors100: 1, nErrors110: 1, nErrors120: 1, nErrors130: 0, nErrors140: 0, nErrors150: 0, nErrors160: 0), - new ParserTest170("RegexpLikeTests170.sql", nErrors80: 13, nErrors90: 13, nErrors100: 13, nErrors110: 15, nErrors120: 15, nErrors130: 15, nErrors140: 15, nErrors150: 15, nErrors160: 15) + new ParserTest170("RegexpLikeTests170.sql", nErrors80: 15, nErrors90: 15, nErrors100: 15, nErrors110: 17, nErrors120: 17, nErrors130: 17, nErrors140: 17, nErrors150: 17, nErrors160: 17) }; private static readonly ParserTest[] SqlAzure170_TestInfos = diff --git a/Test/SqlDom/TestScripts/BooleanExpressionTests.sql b/Test/SqlDom/TestScripts/BooleanExpressionTests.sql index 175259b..dc19cb9 100644 --- a/Test/SqlDom/TestScripts/BooleanExpressionTests.sql +++ b/Test/SqlDom/TestScripts/BooleanExpressionTests.sql @@ -128,6 +128,11 @@ FROM Production.Document WHERE Contains ( t1.c1, @a); GO +SELECT Title +FROM Production.Document +WHERE (Contains ( t1.c1, @a)); +GO + SELECT Title FROM Production.Document WHERE Freetext ( t2.*, N'abc'); diff --git a/Test/SqlDom/TestScripts/RegexpLikeTests170.sql b/Test/SqlDom/TestScripts/RegexpLikeTests170.sql index 2faab93..3933a04 100644 --- a/Test/SqlDom/TestScripts/RegexpLikeTests170.sql +++ b/Test/SqlDom/TestScripts/RegexpLikeTests170.sql @@ -30,4 +30,8 @@ SELECT CASE WHEN NOT REGEXP_LIKE ('abc', '^a', NULL) THEN 1 ELSE 0 END AS is_mat SELECT CASE WHEN REGEXP_LIKE (NULL, '^a', 'c') THEN 1 ELSE 0 END AS is_match; -SELECT IIF (NOT REGEXP_LIKE ('abc', NULL), 1, 0) AS is_match; \ No newline at end of file +SELECT IIF (NOT REGEXP_LIKE ('abc', NULL), 1, 0) AS is_match; + +SELECT 1 WHERE REGEXP_LIKE('a', '^a'); + +SELECT 1 WHERE (REGEXP_LIKE('a', '%pattern%')); \ No newline at end of file From a54b4245de24758693c32db05a8184b59b04da3c Mon Sep 17 00:00:00 2001 From: Leila Lali Date: Thu, 18 Sep 2025 14:55:01 -0700 Subject: [PATCH 2/2] Adding more test --- Test/SqlDom/Baselines170/RegexpLikeTests170.sql | 2 ++ Test/SqlDom/Only170SyntaxTests.cs | 2 +- Test/SqlDom/TestScripts/RegexpLikeTests170.sql | 3 +++ 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/Test/SqlDom/Baselines170/RegexpLikeTests170.sql b/Test/SqlDom/Baselines170/RegexpLikeTests170.sql index 8fb24cc..47b91bf 100644 --- a/Test/SqlDom/Baselines170/RegexpLikeTests170.sql +++ b/Test/SqlDom/Baselines170/RegexpLikeTests170.sql @@ -8,6 +8,8 @@ SELECT IIF (REGEXP_LIKE ('abc', '^a'), 1, 0) AS is_match; SELECT IIF (NOT REGEXP_LIKE ('abc', '^a'), 1, 0) AS is_match; +SELECT (IIF (REGEXP_LIKE ('abc', '^a'), 'Match', 'No Match')) AS result; + SELECT CASE WHEN REGEXP_LIKE ('abc', '^a') THEN 1 ELSE 0 END AS is_match; SELECT CASE WHEN NOT REGEXP_LIKE ('abc', '^a') THEN 1 ELSE 0 END AS is_match; diff --git a/Test/SqlDom/Only170SyntaxTests.cs b/Test/SqlDom/Only170SyntaxTests.cs index d93dbc9..579a8c9 100644 --- a/Test/SqlDom/Only170SyntaxTests.cs +++ b/Test/SqlDom/Only170SyntaxTests.cs @@ -27,7 +27,7 @@ public partial class SqlDomTests new ParserTest170("VectorFunctionTests170.sql", nErrors80: 1, nErrors90: 1, nErrors100: 2, nErrors110: 2, nErrors120: 2, nErrors130: 2, nErrors140: 2, nErrors150: 2, nErrors160: 2), new ParserTest170("SecurityStatementExternalModelTests170.sql", nErrors80: 2, nErrors90: 17, nErrors100: 17, nErrors110: 17, nErrors120: 17, nErrors130: 17, nErrors140: 17, nErrors150: 17, nErrors160: 17), new ParserTest170("CreateEventSessionNotLikePredicate.sql", nErrors80: 2, nErrors90: 1, nErrors100: 1, nErrors110: 1, nErrors120: 1, nErrors130: 0, nErrors140: 0, nErrors150: 0, nErrors160: 0), - new ParserTest170("RegexpLikeTests170.sql", nErrors80: 15, nErrors90: 15, nErrors100: 15, nErrors110: 17, nErrors120: 17, nErrors130: 17, nErrors140: 17, nErrors150: 17, nErrors160: 17) + new ParserTest170("RegexpLikeTests170.sql", nErrors80: 15, nErrors90: 15, nErrors100: 15, nErrors110: 18, nErrors120: 18, nErrors130: 18, nErrors140: 18, nErrors150: 18, nErrors160: 18) }; private static readonly ParserTest[] SqlAzure170_TestInfos = diff --git a/Test/SqlDom/TestScripts/RegexpLikeTests170.sql b/Test/SqlDom/TestScripts/RegexpLikeTests170.sql index 3933a04..989cc6a 100644 --- a/Test/SqlDom/TestScripts/RegexpLikeTests170.sql +++ b/Test/SqlDom/TestScripts/RegexpLikeTests170.sql @@ -8,6 +8,9 @@ SELECT IIF (REGEXP_LIKE ('abc', '^a'), 1, 0) AS is_match; SELECT IIF (NOT REGEXP_LIKE ('abc', '^a'), 1, 0) AS is_match; +-- Test REGEXP_LIKE inside IIF with parentheses (should be scalar parentheses) +SELECT (IIF (REGEXP_LIKE ('abc', '^a'), 'Match', 'No Match')) AS result; + SELECT CASE WHEN REGEXP_LIKE ('abc', '^a') THEN 1 ELSE 0 END AS is_match; SELECT CASE WHEN NOT REGEXP_LIKE ('abc', '^a') THEN 1 ELSE 0 END AS is_match;