From ec8be62ef936032fc9605d5a6d6d0b9b571628d1 Mon Sep 17 00:00:00 2001 From: C Shanmukha Reddy Date: Tue, 15 Jul 2025 07:21:58 +0000 Subject: [PATCH 01/12] Merged PR 1735207: Adding regex functions tests # Pull Request Template for ScriptDom ## Description This PR adds tests for regex intrinsic functions. Functional Spec - https://microsoft.sharepoint.com/:w:/t/Ad-Astra/EZAk15L-vkJHqsaCybly0QIB0eBUdv02BBaHUAl48aY_Pw?e=NPRj0w ## Code Change - [x] The [Common checklist](https://msdata.visualstudio.com/SQLToolsAndLibraries/_git/Common?path=/Templates/PR%20Checklist%20for%20SQLToolsAndLibraries.md&version=GBmain&_a=preview) has been reviewed and followed - [x] Code changes are accompanied by appropriate unit tests - [ ] Identified and included SMEs needed to review code changes - [x] Follow the [steps](https://msdata.visualstudio.com/SQLToolsAndLibraries/_wiki/wikis/SQLToolsAndLibraries.wiki/33838/Adding-or-Extending-TSql-support-in-Sql-Dom?anchor=make-the-changes-in) here to make changes in the code ## Testing - [x] Follow the [steps](https://msdata.visualstudio.com/SQLToolsAndLibraries/_wiki/wikis/SQLToolsAndLibraries.wiki/33838/Adding-or-Extending-TSql-support-in-Sql-Dom?anchor=to-extend-the-tests-do-the-following%3A) here to add new tests for your feature ## Documentation - [ ] Update relevant documentation in the [wiki](https://dev.azure.com/msdata/SQLToolsAndLibraries/_wiki/wikis/SQLToolsAndLibraries.wiki) and the README.md file ## Additional Information None Adding regex functions tests Related work items: #4005966 --- Test/SqlDom/Baselines170/RegexpTests170.sql | 22 +++++++++++++++++ Test/SqlDom/Only170SyntaxTests.cs | 3 ++- Test/SqlDom/TestScripts/RegexpTests170.sql | 26 +++++++++++++++++++++ 3 files changed, 50 insertions(+), 1 deletion(-) create mode 100644 Test/SqlDom/Baselines170/RegexpTests170.sql create mode 100644 Test/SqlDom/TestScripts/RegexpTests170.sql diff --git a/Test/SqlDom/Baselines170/RegexpTests170.sql b/Test/SqlDom/Baselines170/RegexpTests170.sql new file mode 100644 index 0000000..e13c3da --- /dev/null +++ b/Test/SqlDom/Baselines170/RegexpTests170.sql @@ -0,0 +1,22 @@ +SELECT REGEXP_COUNT('hello', 'he(l+)o'); +SELECT REGEXP_COUNT('hello', 'he(l+)o', 1); +SELECT REGEXP_COUNT('hello', 'he(l+)o', 1, 'i'); + +SELECT REGEXP_INSTR('hello', 'he(l+)o'); +SELECT REGEXP_INSTR('hello', 'he(l+)o', 1); +SELECT REGEXP_INSTR('hello', 'he(l+)o', 1, 1); +SELECT REGEXP_INSTR('hello', 'he(l+)o', 1, 1, 0); +SELECT REGEXP_INSTR('hello', 'he(l+)o', 1, 1, 0, 'i'); +SELECT REGEXP_INSTR('hello', 'he(l+)o', 1, 1, 0, 'c', 1); + +SELECT REGEXP_REPLACE('hello', 'he(l+)o'); +SELECT REGEXP_REPLACE('hello', 'he(l+)o', 'hi\1o'); +SELECT REGEXP_REPLACE('hello', 'he(l+)o', 'hi\1o', 1); +SELECT REGEXP_REPLACE('hello', 'he(l+)o', 'hi\1o', 1, 1); +SELECT REGEXP_REPLACE('hello', 'he(l+)o', 'hi\1o', 1, 1, 'm'); + +SELECT REGEXP_SUBSTR('hello', 'he(l+)o'); +SELECT REGEXP_SUBSTR('hello', 'he(l+)o', 1); +SELECT REGEXP_SUBSTR('hello', 'he(l+)o', 1, 1); +SELECT REGEXP_SUBSTR('hello', 'he(l+)o', 1, 1, 's'); +SELECT REGEXP_SUBSTR('hello', 'he(l+)o', 1, 1, 'c', 1); \ No newline at end of file diff --git a/Test/SqlDom/Only170SyntaxTests.cs b/Test/SqlDom/Only170SyntaxTests.cs index 05ade64..80ec622 100644 --- a/Test/SqlDom/Only170SyntaxTests.cs +++ b/Test/SqlDom/Only170SyntaxTests.cs @@ -13,7 +13,8 @@ public partial class SqlDomTests new ParserTest170("JsonIndexTests170.sql", nErrors80: 2, nErrors90: 8, nErrors100: 8, nErrors110: 8, nErrors120: 8, nErrors130: 8, nErrors140: 8, nErrors150: 8, nErrors160: 8), new ParserTest170("VectorIndexTests170.sql", nErrors80: 2, nErrors90: 12, nErrors100: 12, nErrors110: 12, nErrors120: 12, nErrors130: 12, nErrors140: 12, nErrors150: 12, nErrors160: 12), new ParserTest170("AlterDatabaseManualCutoverTests170.sql", nErrors80: 4, nErrors90: 4, nErrors100: 4, nErrors110: 4, nErrors120: 4, nErrors130: 4, nErrors140: 4, nErrors150: 4, nErrors160: 4), - new ParserTest170("CreateColumnStoreIndexTests170.sql", nErrors80: 3, nErrors90: 3, nErrors100: 3, nErrors110: 3, nErrors120: 3, nErrors130: 0, nErrors140: 0, nErrors150: 0, nErrors160: 0) + new ParserTest170("CreateColumnStoreIndexTests170.sql", nErrors80: 3, nErrors90: 3, nErrors100: 3, nErrors110: 3, nErrors120: 3, nErrors130: 0, nErrors140: 0, nErrors150: 0, nErrors160: 0), + new ParserTest170("RegexpTests170.sql", nErrors80: 0, nErrors90: 0, nErrors100: 0, nErrors110: 0, nErrors120: 0, nErrors130: 0, nErrors140: 0, nErrors150: 0, nErrors160: 0) }; private static readonly ParserTest[] SqlAzure170_TestInfos = diff --git a/Test/SqlDom/TestScripts/RegexpTests170.sql b/Test/SqlDom/TestScripts/RegexpTests170.sql new file mode 100644 index 0000000..160857a --- /dev/null +++ b/Test/SqlDom/TestScripts/RegexpTests170.sql @@ -0,0 +1,26 @@ +-- REGEXP_COUNT(text, pattern [, start [, flags ]]) +SELECT REGEXP_COUNT('hello', 'he(l+)o'); +SELECT REGEXP_COUNT('hello', 'he(l+)o', 1); +SELECT REGEXP_COUNT('hello', 'he(l+)o', 1, 'i'); + +-- REGEXP_INSTR(text, pattern [, start [, occurrence [, return_option [, flags [, group ]]]]]) +SELECT REGEXP_INSTR('hello', 'he(l+)o'); +SELECT REGEXP_INSTR('hello', 'he(l+)o', 1); +SELECT REGEXP_INSTR('hello', 'he(l+)o', 1, 1); +SELECT REGEXP_INSTR('hello', 'he(l+)o', 1, 1, 0); +SELECT REGEXP_INSTR('hello', 'he(l+)o', 1, 1, 0, 'i'); +SELECT REGEXP_INSTR('hello', 'he(l+)o', 1, 1, 0, 'c', 1); + +-- REGEXP_REPLACE(text, pattern [, replacement [, start [, occurrence [, flags ]]]]) +SELECT REGEXP_REPLACE('hello', 'he(l+)o'); +SELECT REGEXP_REPLACE('hello', 'he(l+)o', 'hi\1o'); +SELECT REGEXP_REPLACE('hello', 'he(l+)o', 'hi\1o', 1); +SELECT REGEXP_REPLACE('hello', 'he(l+)o', 'hi\1o', 1, 1); +SELECT REGEXP_REPLACE('hello', 'he(l+)o', 'hi\1o', 1, 1, 'm'); + +-- REGEXP_SUBSTR(text, pattern [, start [, occurrence [, flags [, group ]]]]) +SELECT REGEXP_SUBSTR('hello', 'he(l+)o'); +SELECT REGEXP_SUBSTR('hello', 'he(l+)o', 1); +SELECT REGEXP_SUBSTR('hello', 'he(l+)o', 1, 1); +SELECT REGEXP_SUBSTR('hello', 'he(l+)o', 1, 1, 's'); +SELECT REGEXP_SUBSTR('hello', 'he(l+)o', 1, 1, 'c', 1); \ No newline at end of file From 1704a7fc98e052e6786f2f8ce365851cbc851842 Mon Sep 17 00:00:00 2001 From: Sicong Liu Date: Thu, 17 Jul 2025 20:23:57 +0000 Subject: [PATCH 02/12] Merged PR 1718694: [ai_generate_chunks] Add syntax to ScriptDom # Pull Request Template for ScriptDom ## Description # Add syntax and script generation support for AI_GENERATE_CHUNKS table-valued function ## Description This PR introduces parser and script generator support for the new built-in table-valued function (TVF) `AI_GENERATE_CHUNKS`. `AI_GENERATE_CHUNKS` enables chunking of input text based on a specified strategy. It accepts required and optional parameters and supports aliasing for cross apply scenarios. ### Syntax supported ```sql SELECT * FROM AI_GENERATE_CHUNKS ( source = 'some text', chunk_type = fixed, chunk_size = 5, overlap = 2, enable_chunk_set_id = 1 ); ``` ### Parameters | Parameter | Required | Type | Description | |------------------------|----------|-------------------|--------------------------------------------------------------------| | `source` | Yes | `ScalarExpression`| Input text to be chunked | | `chunk_type` | Yes | `Identifier` | Type of chunking strategy. Currently only `fixed` is supported | | `chunk_size` | Yes | `ScalarExpression`| Number of tokens per chunk | | `overlap` | No | `ScalarExpression`| Optional token overlap between adjacent chunks | | `enable_chunk_set_id` | No | `ScalarExpression`| Optional flag. If set, adds a `chunk_set_id` column to the result | ### Summary of Changes - Added support for the `ai_generate_chunks` table-valued function to the SQL parser. - Introduced new AST class `AiGenerateChunksTableReference` with members: - `Source` - `ChunkType` - `ChunkSize` - `Overlap` - `EnableChunkSetId` - Added grammar rule for parsing `ai_generate_chunks`. - Implemented corresponding visitor logic for script generation (`SqlScriptGeneratorVisitor`). - Added unit tests for: - Positive cases covering all combinations of optional parameters. - Negative cases for missing/invalid parameter order, unexpected tokens, or malformed expressions. ### Example Use Cases This function is designed to support downstream AI workflows where long documents need to be split into manageable token chunks for embedding generation or other ML tasks. ## Code Change - [X] The [Common checklist](https://msdata.visualstudio.com/SQLToolsAndLibraries/_git/Common?path=/Templates/PR%20Checklist%20for%20SQLToolsAndLibraries.md&version=GBmain&_a=preview) has been reviewed and followed - [X] Code changes are accompanied by appropriate unit tests - [X] Identified and included SMEs needed to review code changes - [X] Follow the [steps](https://msdata.visualstudio.com/SQLToolsAndLibraries/_wiki/wikis/SQLToolsAndLibraries.wiki/33838/Adding-or-Extending-TSql-support-in-Sql-Dom?anchor=make-the-changes-in) here to make changes in the code ## Testing - [X] Follow the [steps](https://msdata.vi... --- SqlScriptDom/Parser/TSql/Ast.xml | 11 ++ .../Parser/TSql/CodeGenerationSupporter.cs | 6 + SqlScriptDom/Parser/TSql/TSql170.g | 106 +++++++++++++++- ...tor.AIGenerateFixedChunksTableReference.cs | 31 +++++ ...rVisitor.AiGenerateChunksTableReference.cs | 51 ++++++++ .../Baselines170/AiGenerateChunksTests170.sql | 101 +++++++++++++++ Test/SqlDom/Only170SyntaxTests.cs | 3 +- Test/SqlDom/ParserErrorsTests.cs | 116 +++++++++++++++++- .../TestScripts/AiGenerateChunksTests170.sql | 101 +++++++++++++++ 9 files changed, 523 insertions(+), 3 deletions(-) create mode 100644 SqlScriptDom/ScriptDom/SqlServer/ScriptGenerator/SqlScriptGeneratorVisitor.AIGenerateFixedChunksTableReference.cs create mode 100644 SqlScriptDom/ScriptDom/SqlServer/ScriptGenerator/SqlScriptGeneratorVisitor.AiGenerateChunksTableReference.cs create mode 100644 Test/SqlDom/Baselines170/AiGenerateChunksTests170.sql create mode 100644 Test/SqlDom/TestScripts/AiGenerateChunksTests170.sql diff --git a/SqlScriptDom/Parser/TSql/Ast.xml b/SqlScriptDom/Parser/TSql/Ast.xml index c8ebb38..49c7bdb 100644 --- a/SqlScriptDom/Parser/TSql/Ast.xml +++ b/SqlScriptDom/Parser/TSql/Ast.xml @@ -4713,4 +4713,15 @@ + + + + + + + + + + + diff --git a/SqlScriptDom/Parser/TSql/CodeGenerationSupporter.cs b/SqlScriptDom/Parser/TSql/CodeGenerationSupporter.cs index 4deaa86..cd727c7 100644 --- a/SqlScriptDom/Parser/TSql/CodeGenerationSupporter.cs +++ b/SqlScriptDom/Parser/TSql/CodeGenerationSupporter.cs @@ -99,6 +99,7 @@ internal static class CodeGenerationSupporter internal const string Affinity = "AFFINITY"; internal const string After = "AFTER"; internal const string Aggregate = "AGGREGATE"; + internal const string AiGenerateChunks = "AI_GENERATE_CHUNKS"; internal const string Algorithm = "ALGORITHM"; internal const string AlterColumn = "ALTERCOLUMN"; internal const string All = "ALL"; @@ -198,6 +199,8 @@ internal static class CodeGenerationSupporter internal const string CheckPolicy = "CHECK_POLICY"; internal const string Checksum = "CHECKSUM"; internal const string ChecksumAgg = "CHECKSUM_AGG"; + internal const string ChunkSize = "CHUNK_SIZE"; + internal const string ChunkType = "CHUNK_TYPE"; internal const string ModularSum = "MODULAR_SUM"; internal const string Classifier = "CLASSIFIER"; internal const string Classification = "CLASSIFICATION"; @@ -334,6 +337,7 @@ internal static class CodeGenerationSupporter internal const string Enable = "ENABLE"; internal const string Enabled = "ENABLED"; internal const string EnableBroker = "ENABLE_BROKER"; + internal const string EnableChunkSetId = "ENABLE_CHUNK_SET_ID"; internal const string EnclaveComputations = "ENCLAVE_COMPUTATIONS"; internal const string Encoding = "ENCODING"; internal const string Encrypted = "ENCRYPTED"; @@ -415,6 +419,7 @@ internal static class CodeGenerationSupporter internal const string FieldQuote = "FIELDQUOTE"; internal const string FipsFlagger = "FIPS_FLAGGER"; internal const string First = "FIRST"; + internal const string Fixed = "FIXED"; internal const string FlushIntervalSeconds = "FLUSH_INTERVAL_SECONDS"; internal const string FlushIntervalSecondsAlt = "DATA_FLUSH_INTERVAL_SECONDS"; internal const string Fn = "FN"; @@ -761,6 +766,7 @@ internal static class CodeGenerationSupporter internal const string Model = "MODEL"; internal const string RunTime = "RUNTIME"; internal const string Onnx = "ONNX"; + internal const string Overlap = "OVERLAP"; internal const string Process = "PROCESS"; internal const string PropertySetGuid = "PROPERTY_SET_GUID"; diff --git a/SqlScriptDom/Parser/TSql/TSql170.g b/SqlScriptDom/Parser/TSql/TSql170.g index 1ef4339..e43eda1 100644 --- a/SqlScriptDom/Parser/TSql/TSql170.g +++ b/SqlScriptDom/Parser/TSql/TSql170.g @@ -19122,6 +19122,8 @@ selectTableReferenceElementWithoutJoinParenthesis[SubDmlFlags subDmlFlags] retur : {NextTokenMatches(CodeGenerationSupporter.ChangeTable)}? vResult=changeTableTableReference + | {NextTokenMatches(CodeGenerationSupporter.AiGenerateChunks)}? + vResult = aiGenerateChunksTableReference | vResult=builtInFunctionTableReference | {NextIdentifierMatchesOneOf(new string[] {CodeGenerationSupporter.StringSplit, CodeGenerationSupporter.GenerateSeries, CodeGenerationSupporter.RegexpMatches, CodeGenerationSupporter.RegexpSplitToTable})}? vResult=globalFunctionTableReference @@ -19140,6 +19142,98 @@ selectTableReferenceElementWithoutJoinParenthesis[SubDmlFlags subDmlFlags] retur | vResult=schemaObjectOrFunctionTableReference ; +aiGenerateChunksTableReference returns [AIGenerateChunksTableReference vResult = null] +{ + ScalarExpression vSource; + Identifier vChunkType; +} + : + {NextTokenMatches(CodeGenerationSupporter.AiGenerateChunks)}? + tFunc:Identifier + { + Match(tFunc, CodeGenerationSupporter.AiGenerateChunks); + } + LeftParenthesis + tSourceToken:Identifier + { + Match(tSourceToken, CodeGenerationSupporter.Source); + } + EqualsSign + vSource = expression + Comma + tChunkTypeToken:Identifier + { + Match(tChunkTypeToken, CodeGenerationSupporter.ChunkType); + } + EqualsSign + vChunkType = identifier + Comma + vResult = aiGenerateFixedChunksTableReference[vSource, vChunkType] + tRParen:RightParenthesis + { + if (vResult != null) + { + UpdateTokenInfo(vResult, tFunc); + UpdateTokenInfo(vResult, tRParen); + } + } + simpleTableReferenceAliasOpt[vResult] + ; + +aiGenerateFixedChunksTableReference [ScalarExpression vSource, Identifier vChunkType] + returns [AIGenerateFixedChunksTableReference vResult = FragmentFactory.CreateFragment()] +{ + Identifier vChunkSizeParam; + Identifier vOverlapParam = null; + Identifier vEnableChunkSetIdParam = null; + + ScalarExpression vChunkSize = null; + ScalarExpression vOverlap = null; + ScalarExpression vEnableChunkSetId = null; +} + : + { + Match(vChunkType, CodeGenerationSupporter.Fixed); + vResult.Source = vSource; + vResult.ChunkType = vChunkType; + } + vChunkSizeParam = identifier + { + Match(vChunkSizeParam, CodeGenerationSupporter.ChunkSize); + } + EqualsSign + vChunkSize = expression + { + vResult.ChunkSize = vChunkSize; + } + + ( + Comma + vOverlapParam = identifier + { + Match(vOverlapParam, CodeGenerationSupporter.Overlap); + } + EqualsSign + vOverlap = expression + { + vResult.Overlap = vOverlap; + } + )? + + ( + Comma + vEnableChunkSetIdParam = identifier + { + Match(vEnableChunkSetIdParam, CodeGenerationSupporter.EnableChunkSetId); + } + EqualsSign + vEnableChunkSetId = expression + { + vResult.EnableChunkSetId = vEnableChunkSetId; + } + )? + ; + predictTableReference[SubDmlFlags subDmlFlags] returns [PredictTableReference vResult] : {NextTokenMatches(CodeGenerationSupporter.Predict)}? @@ -20264,13 +20358,23 @@ schemaObjectTableDmlTarget [bool indexHintAllowed] returns [NamedTableReference )? ; -schemaObjectOrFunctionTableReference returns [TableReference vResult] +schemaObjectOrFunctionTableReference returns [TableReference vResult = null] { SchemaObjectName vSchemaObjectName; } : vSchemaObjectName=schemaObjectFourPartName ( + { + vSchemaObjectName.BaseIdentifier != null && + vSchemaObjectName.BaseIdentifier.Value.Equals(CodeGenerationSupporter.AiGenerateChunks, StringComparison.OrdinalIgnoreCase) && + vSchemaObjectName.BaseIdentifier.QuoteType == QuoteType.NotQuoted && + LT(2).getText().Equals(CodeGenerationSupporter.Source, StringComparison.OrdinalIgnoreCase) + }? + { + vResult = aiGenerateChunksTableReference(); + } + | {IsTableReference(false)}? vResult=schemaObjectTableReference[vSchemaObjectName] | vResult=schemaObjectFunctionTableReference[vSchemaObjectName] diff --git a/SqlScriptDom/ScriptDom/SqlServer/ScriptGenerator/SqlScriptGeneratorVisitor.AIGenerateFixedChunksTableReference.cs b/SqlScriptDom/ScriptDom/SqlServer/ScriptGenerator/SqlScriptGeneratorVisitor.AIGenerateFixedChunksTableReference.cs new file mode 100644 index 0000000..318cee0 --- /dev/null +++ b/SqlScriptDom/ScriptDom/SqlServer/ScriptGenerator/SqlScriptGeneratorVisitor.AIGenerateFixedChunksTableReference.cs @@ -0,0 +1,31 @@ +//------------------------------------------------------------------------------ +// +// Copyright (c) Microsoft Corporation. All rights reserved. +// +//------------------------------------------------------------------------------ +using System.Collections.Generic; +using Microsoft.SqlServer.TransactSql.ScriptDom; + +namespace Microsoft.SqlServer.TransactSql.ScriptDom.ScriptGenerator +{ + partial class SqlScriptGeneratorVisitor + { + public override void ExplicitVisit(AIGenerateFixedChunksTableReference node) + { + List<(string, TSqlFragment)> specificParameters = new List<(string, TSqlFragment)>(); + specificParameters.Add((CodeGenerationSupporter.ChunkSize, node.ChunkSize)); + + if (node.Overlap != null) + { + specificParameters.Add((CodeGenerationSupporter.Overlap, node.Overlap)); + } + + if (node.EnableChunkSetId != null) + { + specificParameters.Add((CodeGenerationSupporter.EnableChunkSetId, node.EnableChunkSetId)); + } + + GenerateAIChunkTableReference(node, specificParameters); + } + } +} diff --git a/SqlScriptDom/ScriptDom/SqlServer/ScriptGenerator/SqlScriptGeneratorVisitor.AiGenerateChunksTableReference.cs b/SqlScriptDom/ScriptDom/SqlServer/ScriptGenerator/SqlScriptGeneratorVisitor.AiGenerateChunksTableReference.cs new file mode 100644 index 0000000..eefa100 --- /dev/null +++ b/SqlScriptDom/ScriptDom/SqlServer/ScriptGenerator/SqlScriptGeneratorVisitor.AiGenerateChunksTableReference.cs @@ -0,0 +1,51 @@ +//------------------------------------------------------------------------------ +// +// Copyright (c) Microsoft Corporation. All rights reserved. +// +//------------------------------------------------------------------------------ +using System.Collections.Generic; +using Microsoft.SqlServer.TransactSql.ScriptDom; + +namespace Microsoft.SqlServer.TransactSql.ScriptDom.ScriptGenerator +{ + partial class SqlScriptGeneratorVisitor + { + public void GenerateNameValuePairs(IList<(string Name, TSqlFragment Value)> pairs) + { + for (int index = 0; index < pairs.Count; index++) + { + (string name, TSqlFragment value) = pairs[index]; + + if (index > 0) + { + GenerateSymbol(TSqlTokenType.Comma); + GenerateSpace(); + } + + GenerateIdentifierWithoutCasing(name); + GenerateSpace(); + GenerateSymbol(TSqlTokenType.EqualsSign); + GenerateSpace(); + GenerateFragmentIfNotNull(value); + } + } + + public void GenerateAIChunkTableReference(AIGenerateChunksTableReference node, IList<(string, TSqlFragment)> additionalParameters) + { + GenerateIdentifierWithoutCasing(CodeGenerationSupporter.AiGenerateChunks); + GenerateSpaceAndSymbol(TSqlTokenType.LeftParenthesis); + + // Include common parameters: source, chunk_type + List<(string, TSqlFragment)> parameters = new List<(string, TSqlFragment)>(); + parameters.Add((CodeGenerationSupporter.Source, node.Source)); + parameters.Add((CodeGenerationSupporter.ChunkType, node.ChunkType)); + parameters.AddRange(additionalParameters); + + GenerateNameValuePairs(parameters); + + GenerateSymbol(TSqlTokenType.RightParenthesis); + + GenerateSpaceAndAlias(node.Alias); + } + } +} diff --git a/Test/SqlDom/Baselines170/AiGenerateChunksTests170.sql b/Test/SqlDom/Baselines170/AiGenerateChunksTests170.sql new file mode 100644 index 0000000..8374acc --- /dev/null +++ b/Test/SqlDom/Baselines170/AiGenerateChunksTests170.sql @@ -0,0 +1,101 @@ +SELECT * +FROM AI_GENERATE_CHUNKS (SOURCE = 'some text', CHUNK_TYPE = fixed, CHUNK_SIZE = 5); + +SELECT * +FROM AI_GENERATE_CHUNKS (SOURCE = N'some text', CHUNK_TYPE = fixed, CHUNK_SIZE = 5); + +SELECT * +FROM AI_GENERATE_CHUNKS (SOURCE = 'some text', CHUNK_TYPE = fixed, CHUNK_SIZE = 5, OVERLAP = 50); + +SELECT * +FROM AI_GENERATE_CHUNKS (SOURCE = 'some text', CHUNK_TYPE = fixed, CHUNK_SIZE = 5, OVERLAP = NULL, ENABLE_CHUNK_SET_ID = 50); + +SELECT * +FROM AI_GENERATE_CHUNKS (SOURCE = 'some text', CHUNK_TYPE = fixed, CHUNK_SIZE = 5, OVERLAP = 10, ENABLE_CHUNK_SET_ID = 1); + +SELECT * +FROM AI_GENERATE_CHUNKS (SOURCE = @SOURCE, CHUNK_TYPE = fixed, CHUNK_SIZE = @CHUNK_SIZE); + +SELECT * +FROM AI_GENERATE_CHUNKS (SOURCE = @SOURCE, CHUNK_TYPE = fixed, CHUNK_SIZE = @CHUNK_SIZE, OVERLAP = @OVERLAP, ENABLE_CHUNK_SET_ID = @ENABLE_CHUNK_SET_ID); + +SELECT * +FROM AI_GENERATE_CHUNKS (SOURCE = NULL, CHUNK_TYPE = fixed, CHUNK_SIZE = 5); + +SELECT * +FROM t1 CROSS APPLY AI_GENERATE_CHUNKS (SOURCE = t1.c1, CHUNK_TYPE = fixed, CHUNK_SIZE = 10); + +SELECT * +FROM t1 CROSS APPLY AI_GENERATE_CHUNKS (SOURCE = 'some text', CHUNK_TYPE = fixed, CHUNK_SIZE = t1.c1); + +SELECT * +FROM t1 CROSS APPLY AI_GENERATE_CHUNKS (SOURCE = 'some text', CHUNK_TYPE = fixed, CHUNK_SIZE = 10, OVERLAP = t1.c1); + +SELECT * +FROM t1 CROSS APPLY AI_GENERATE_CHUNKS (SOURCE = 'some text', CHUNK_TYPE = fixed, CHUNK_SIZE = 10, OVERLAP = 5, ENABLE_CHUNK_SET_ID = t1.c1); + +SELECT * +FROM AI_GENERATE_CHUNKS (SOURCE = N'some text', CHUNK_TYPE = fixed, CHUNK_SIZE = NULL); + +SELECT * +FROM AI_GENERATE_CHUNKS (SOURCE = N'some text', CHUNK_TYPE = fixed, CHUNK_SIZE = 10, OVERLAP = NULL); + +SELECT * +FROM AI_GENERATE_CHUNKS (SOURCE = N'some text', CHUNK_TYPE = fixed, CHUNK_SIZE = 10, OVERLAP = 5, ENABLE_CHUNK_SET_ID = NULL); + +CREATE TABLE t2 ( + Id INT IDENTITY PRIMARY KEY, + Text NVARCHAR (MAX) +); +GO + +INSERT INTO t2 (Text) +VALUES (N'This is the first text.'), +(N'Second sample text.'), +(N'Third example text.'); +GO + +CREATE VIEW v_GeneratedChunksFromTable +AS +SELECT t2.Id, + chunks.* +FROM t2 CROSS APPLY AI_GENERATE_CHUNKS (SOURCE = t2.Text, CHUNK_TYPE = fixed, CHUNK_SIZE = 5) AS chunks; +GO + +CREATE PROCEDURE usp_GenerateChunks +AS +BEGIN + SELECT * + FROM AI_GENERATE_CHUNKS (SOURCE = N'This is some sample text that will be chunked.', CHUNK_TYPE = fixed, CHUNK_SIZE = 5, OVERLAP = 2); +END +GO + +CREATE FUNCTION dbo.AI_GENERATE_CHUNKS +(@input NVARCHAR (MAX)) +RETURNS NVARCHAR (MAX) +AS +BEGIN + RETURN CONCAT('Chunked: ', @input); +END +GO + +CREATE TABLE dbo.AI_GENERATE_CHUNKS ( + id INT PRIMARY KEY, + data NVARCHAR (MAX) +); + +SELECT SOURCE +FROM userTable CROSS APPLY dbo.AI_GENERATE_CHUNKS(target); + +SELECT SOURCE +FROM userTable CROSS APPLY dbo.[AI_GENERATE_CHUNKS](SOURCE); + + +SELECT SOURCE +FROM userTable CROSS APPLY dbo.AI_GENERATE_CHUNKS([SOURCE]); + +SELECT SOURCE +FROM userTable CROSS APPLY dbo.[AI_GENERATE_CHUNKS]([SOURCE]); + +SELECT * +FROM dbo.AI_GENERATE_CHUNKS(3); \ No newline at end of file diff --git a/Test/SqlDom/Only170SyntaxTests.cs b/Test/SqlDom/Only170SyntaxTests.cs index 80ec622..b63b2b0 100644 --- a/Test/SqlDom/Only170SyntaxTests.cs +++ b/Test/SqlDom/Only170SyntaxTests.cs @@ -14,7 +14,8 @@ public partial class SqlDomTests new ParserTest170("VectorIndexTests170.sql", nErrors80: 2, nErrors90: 12, nErrors100: 12, nErrors110: 12, nErrors120: 12, nErrors130: 12, nErrors140: 12, nErrors150: 12, nErrors160: 12), new ParserTest170("AlterDatabaseManualCutoverTests170.sql", nErrors80: 4, nErrors90: 4, nErrors100: 4, nErrors110: 4, nErrors120: 4, nErrors130: 4, nErrors140: 4, nErrors150: 4, nErrors160: 4), new ParserTest170("CreateColumnStoreIndexTests170.sql", nErrors80: 3, nErrors90: 3, nErrors100: 3, nErrors110: 3, nErrors120: 3, nErrors130: 0, nErrors140: 0, nErrors150: 0, nErrors160: 0), - new ParserTest170("RegexpTests170.sql", nErrors80: 0, nErrors90: 0, nErrors100: 0, nErrors110: 0, nErrors120: 0, nErrors130: 0, nErrors140: 0, nErrors150: 0, nErrors160: 0) + new ParserTest170("RegexpTests170.sql", nErrors80: 0, nErrors90: 0, nErrors100: 0, nErrors110: 0, nErrors120: 0, nErrors130: 0, nErrors140: 0, nErrors150: 0, nErrors160: 0), + new ParserTest170("AiGenerateChunksTests170.sql", nErrors80: 21, nErrors90: 18, nErrors100: 17, nErrors110: 17, nErrors120: 17, nErrors130: 17, nErrors140: 17, nErrors150: 17, nErrors160: 17) }; private static readonly ParserTest[] SqlAzure170_TestInfos = diff --git a/Test/SqlDom/ParserErrorsTests.cs b/Test/SqlDom/ParserErrorsTests.cs index 23d2be1..d075563 100644 --- a/Test/SqlDom/ParserErrorsTests.cs +++ b/Test/SqlDom/ParserErrorsTests.cs @@ -6864,7 +6864,7 @@ FROM OPENROWSET ('a', 'b', [ab cd]) WITH a;"; ParserTestUtils.ErrorTest160(invalidWithClause3Syntax, new ParserErrorInfo(invalidWithClause3Syntax.IndexOf(@"a;"), "SQL46010", "a")); } - + /// /// Negative tests for Scalar Functions in Fabric DW. /// @@ -7031,5 +7031,119 @@ public void VectorIndexNegativeTests() ParserTestUtils.ErrorTest170("CREATE VECTOR INDEX IX_Test ON dbo.Documents (VectorData) WITH", new ParserErrorInfo(58, "SQL46010", "WITH")); } + + /// + /// Negative tests for AI_GENERATE_CHUNKS syntax + /// + [TestMethod] + [Priority(0)] + [SqlStudioTestCategory(Category.UnitTest)] + public void GenerateChunksNegativeTest170() + { + // Missing required parameters + ParserTestUtils.ErrorTest170( + "SELECT * FROM AI_GENERATE_CHUNKS(source = 'text')", + new ParserErrorInfo(48, "SQL46010", ")")); + + // Missing CHUNK_SIZE + ParserTestUtils.ErrorTest170( + "SELECT * FROM AI_GENERATE_CHUNKS(source = 'text', CHUNK_TYPE = fixed)", + new ParserErrorInfo(68, "SQL46010", ")")); + + // Invalid order: CHUNK_SIZE before CHUNK_TYPE + ParserTestUtils.ErrorTest170( + "SELECT * FROM AI_GENERATE_CHUNKS(source = 'text', CHUNK_SIZE = 5, CHUNK_TYPE = fixed)", + new ParserErrorInfo(50, "SQL46005", "CHUNK_TYPE", "CHUNK_SIZE")); + + // Invalid order: enable_chunk_set_id before overlap + ParserTestUtils.ErrorTest170( + "SELECT * FROM AI_GENERATE_CHUNKS(source = 'text', CHUNK_TYPE = fixed, CHUNK_SIZE = 5, enable_chunk_set_id = 1, overlap = 10)", + new ParserErrorInfo(86, "SQL46010", "enable_chunk_set_id")); + + // Invalid order: enable_chunk_set_id before CHUNK_SIZE + ParserTestUtils.ErrorTest170( + "SELECT * FROM AI_GENERATE_CHUNKS(source = 'text', CHUNK_TYPE = fixed, enable_chunk_set_id = 1, CHUNK_SIZE = 5, overlap = 10)", + new ParserErrorInfo(70, "SQL46010", "enable_chunk_set_id")); + + // Invalid value: CHUNK_TYPE = 'fixed' (should be keyword, not string) + ParserTestUtils.ErrorTest170( + "SELECT * FROM AI_GENERATE_CHUNKS(source = 'text', CHUNK_TYPE = 'fixed', CHUNK_SIZE = 5)", + new ParserErrorInfo(63, "SQL46010", "'fixed'")); + + // Invalid expression: CHUNK_TYPE = @CHUNK_TYPE (should not be variable) + ParserTestUtils.ErrorTest170( + "SELECT * FROM AI_GENERATE_CHUNKS(source = 'text', CHUNK_TYPE = @CHUNK_TYPE, CHUNK_SIZE = 5)", + new ParserErrorInfo(63, "SQL46010", "@CHUNK_TYPE")); + + // Invalid parameter: CHUNK_TYPE = t1.c1 (should not be column reference) + ParserTestUtils.ErrorTest170( + "SELECT * FROM t1 CROSS APPLY AI_GENERATE_CHUNKS(source = 'text', CHUNK_TYPE = t1.c1, CHUNK_SIZE = 5)", + new ParserErrorInfo(80, "SQL46010", ".")); + + // Missing value after equals + ParserTestUtils.ErrorTest170( + "SELECT * FROM AI_GENERATE_CHUNKS(source = , CHUNK_TYPE = fixed, CHUNK_SIZE = 5)", + new ParserErrorInfo(42, "SQL46010", ",")); + + // Missing value for CHUNK_SIZE + ParserTestUtils.ErrorTest170( + "SELECT * FROM AI_GENERATE_CHUNKS(source = 'text', CHUNK_TYPE = fixed, CHUNK_SIZE = )", + new ParserErrorInfo(83, "SQL46010", ")")); + + // Unknown parameter + ParserTestUtils.ErrorTest170( + "SELECT * FROM AI_GENERATE_CHUNKS(source = 'text', CHUNK_TYPE = fixed, CHUNK_SIZE = 5, invalid_param = 123)", + new ParserErrorInfo(86, "SQL46010", "invalid_param")); + + // Extra comma at end + ParserTestUtils.ErrorTest170( + "SELECT * FROM AI_GENERATE_CHUNKS(source = 'text', CHUNK_TYPE = fixed, CHUNK_SIZE = 5,)", + new ParserErrorInfo(85, "SQL46010", ")")); + + // Too many parameters + ParserTestUtils.ErrorTest170( + "SELECT * FROM AI_GENERATE_CHUNKS(source = 'text', CHUNK_TYPE = fixed, CHUNK_SIZE = 5, overlap = 1, enable_chunk_set_id = 2, extra = 3)", + new ParserErrorInfo(122, "SQL46010", ",")); + + // Function call with constant input, not keyword params + ParserTestUtils.ErrorTest170( + "SELECT * FROM ai_generate_chunks(3)", + new ParserErrorInfo(33, "SQL46010", "3")); + + // Missing paramter "source" + ParserTestUtils.ErrorTest170( + "SELECT source, target FROM userTable cross apply ai_generate_chunks(target)", + new ParserErrorInfo(68, "SQL46005", "SOURCE", "target")); + + // Misuse of parameter "source" + ParserTestUtils.ErrorTest170( + "SELECT source, target FROM userTable cross apply ai_generate_chunks(source)", + new ParserErrorInfo(74, "SQL46010", ")")); + + // Misuse of bracketed function name + ParserTestUtils.ErrorTest170( + "SELECT * FROM [ai_generate_chunks](source = 'something to chunk', chunk_type = fixed, chunk_size = 5)", + new ParserErrorInfo(42, "SQL46010", "=")); + + // Misuse of 2-part identifier + ParserTestUtils.ErrorTest170( + "SELECT * FROM dbo.ai_generate_chunks(source = 'something to chunk', chunk_type = fixed, chunk_size = 5)", + new ParserErrorInfo(36, "SQL46010", "(")); + + // 2-part identifier misuse in CROSS APPLY + ParserTestUtils.ErrorTest170( + "SELECT * FROM source CROSS APPLY dbo.ai_generate_chunks(source.c1)", + new ParserErrorInfo(55, "SQL46010", "(")); + + // 2-part identifier misuse in CROSS APPLY + ParserTestUtils.ErrorTest170( + "SELECT source, target FROM userTable cross apply dbo.ai_generate_chunks(source)", + new ParserErrorInfo(71, "SQL46010", "(")); + + // Invalid CHUNK_TYPE + ParserTestUtils.ErrorTest170( + "SELECT * FROM AI_GENERATE_CHUNKS (source = 'some text', chunk_type = other, chunk_size = 5)", + new ParserErrorInfo(69, "SQL46010", "other")); + } } } diff --git a/Test/SqlDom/TestScripts/AiGenerateChunksTests170.sql b/Test/SqlDom/TestScripts/AiGenerateChunksTests170.sql new file mode 100644 index 0000000..8374acc --- /dev/null +++ b/Test/SqlDom/TestScripts/AiGenerateChunksTests170.sql @@ -0,0 +1,101 @@ +SELECT * +FROM AI_GENERATE_CHUNKS (SOURCE = 'some text', CHUNK_TYPE = fixed, CHUNK_SIZE = 5); + +SELECT * +FROM AI_GENERATE_CHUNKS (SOURCE = N'some text', CHUNK_TYPE = fixed, CHUNK_SIZE = 5); + +SELECT * +FROM AI_GENERATE_CHUNKS (SOURCE = 'some text', CHUNK_TYPE = fixed, CHUNK_SIZE = 5, OVERLAP = 50); + +SELECT * +FROM AI_GENERATE_CHUNKS (SOURCE = 'some text', CHUNK_TYPE = fixed, CHUNK_SIZE = 5, OVERLAP = NULL, ENABLE_CHUNK_SET_ID = 50); + +SELECT * +FROM AI_GENERATE_CHUNKS (SOURCE = 'some text', CHUNK_TYPE = fixed, CHUNK_SIZE = 5, OVERLAP = 10, ENABLE_CHUNK_SET_ID = 1); + +SELECT * +FROM AI_GENERATE_CHUNKS (SOURCE = @SOURCE, CHUNK_TYPE = fixed, CHUNK_SIZE = @CHUNK_SIZE); + +SELECT * +FROM AI_GENERATE_CHUNKS (SOURCE = @SOURCE, CHUNK_TYPE = fixed, CHUNK_SIZE = @CHUNK_SIZE, OVERLAP = @OVERLAP, ENABLE_CHUNK_SET_ID = @ENABLE_CHUNK_SET_ID); + +SELECT * +FROM AI_GENERATE_CHUNKS (SOURCE = NULL, CHUNK_TYPE = fixed, CHUNK_SIZE = 5); + +SELECT * +FROM t1 CROSS APPLY AI_GENERATE_CHUNKS (SOURCE = t1.c1, CHUNK_TYPE = fixed, CHUNK_SIZE = 10); + +SELECT * +FROM t1 CROSS APPLY AI_GENERATE_CHUNKS (SOURCE = 'some text', CHUNK_TYPE = fixed, CHUNK_SIZE = t1.c1); + +SELECT * +FROM t1 CROSS APPLY AI_GENERATE_CHUNKS (SOURCE = 'some text', CHUNK_TYPE = fixed, CHUNK_SIZE = 10, OVERLAP = t1.c1); + +SELECT * +FROM t1 CROSS APPLY AI_GENERATE_CHUNKS (SOURCE = 'some text', CHUNK_TYPE = fixed, CHUNK_SIZE = 10, OVERLAP = 5, ENABLE_CHUNK_SET_ID = t1.c1); + +SELECT * +FROM AI_GENERATE_CHUNKS (SOURCE = N'some text', CHUNK_TYPE = fixed, CHUNK_SIZE = NULL); + +SELECT * +FROM AI_GENERATE_CHUNKS (SOURCE = N'some text', CHUNK_TYPE = fixed, CHUNK_SIZE = 10, OVERLAP = NULL); + +SELECT * +FROM AI_GENERATE_CHUNKS (SOURCE = N'some text', CHUNK_TYPE = fixed, CHUNK_SIZE = 10, OVERLAP = 5, ENABLE_CHUNK_SET_ID = NULL); + +CREATE TABLE t2 ( + Id INT IDENTITY PRIMARY KEY, + Text NVARCHAR (MAX) +); +GO + +INSERT INTO t2 (Text) +VALUES (N'This is the first text.'), +(N'Second sample text.'), +(N'Third example text.'); +GO + +CREATE VIEW v_GeneratedChunksFromTable +AS +SELECT t2.Id, + chunks.* +FROM t2 CROSS APPLY AI_GENERATE_CHUNKS (SOURCE = t2.Text, CHUNK_TYPE = fixed, CHUNK_SIZE = 5) AS chunks; +GO + +CREATE PROCEDURE usp_GenerateChunks +AS +BEGIN + SELECT * + FROM AI_GENERATE_CHUNKS (SOURCE = N'This is some sample text that will be chunked.', CHUNK_TYPE = fixed, CHUNK_SIZE = 5, OVERLAP = 2); +END +GO + +CREATE FUNCTION dbo.AI_GENERATE_CHUNKS +(@input NVARCHAR (MAX)) +RETURNS NVARCHAR (MAX) +AS +BEGIN + RETURN CONCAT('Chunked: ', @input); +END +GO + +CREATE TABLE dbo.AI_GENERATE_CHUNKS ( + id INT PRIMARY KEY, + data NVARCHAR (MAX) +); + +SELECT SOURCE +FROM userTable CROSS APPLY dbo.AI_GENERATE_CHUNKS(target); + +SELECT SOURCE +FROM userTable CROSS APPLY dbo.[AI_GENERATE_CHUNKS](SOURCE); + + +SELECT SOURCE +FROM userTable CROSS APPLY dbo.AI_GENERATE_CHUNKS([SOURCE]); + +SELECT SOURCE +FROM userTable CROSS APPLY dbo.[AI_GENERATE_CHUNKS]([SOURCE]); + +SELECT * +FROM dbo.AI_GENERATE_CHUNKS(3); \ No newline at end of file From 9a11bdf9b2d9a20d2bc743f719a8ccec56cb4855 Mon Sep 17 00:00:00 2001 From: MerlinBot Date: Thu, 17 Jul 2025 23:50:50 +0000 Subject: [PATCH 03/12] Update .NET SDK to latest patch version --- global.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/global.json b/global.json index 457d429..3fc0ba6 100644 --- a/global.json +++ b/global.json @@ -1,6 +1,6 @@ { "sdk": { - "version": "8.0.117", + "version": "8.0.412", "rollForward": "latestMajor" }, "msbuild-sdks": { From d4b4affc0669d4663be2b8eff54d8cfbae7bab27 Mon Sep 17 00:00:00 2001 From: Shiv Prashant Sood Date: Mon, 28 Jul 2025 12:29:18 +0000 Subject: [PATCH 04/12] Merged PR 1724944: [MSJSON] added support for json_objectagg and Returning JSON option for json_object, json_array # Pull Request Template for ScriptDom ## Description adds support to scriptdom for new functions and options in SQLServer. Specifically support for - Json_objectagg with options NULL ON NULL/ABSENT ON NULL and RETURNING JSON. - Support for RETURNING JSON option for json_object, json_array. Added relevant test for all options. ## Code Change - [ ] The [Common checklist](https://msdata.visualstudio.com/SQLToolsAndLibraries/_git/Common?path=/Templates/PR%20Checklist%20for%20SQLToolsAndLibraries.md&version=GBmain&_a=preview) has been reviewed and followed - [X] Code changes are accompanied by appropriate unit tests - [X] Identified and included SMEs needed to review code changes - [ ] Follow the [steps](https://msdata.visualstudio.com/SQLToolsAndLibraries/_wiki/wikis/SQLToolsAndLibraries.wiki/33838/Adding-or-Extending-TSql-support-in-Sql-Dom?anchor=make-the-changes-in) here to make changes in the code ## Testing - [X] Follow the [steps](https://msdata.visualstudio.com/SQLToolsAndLibraries/_wiki/wikis/SQLToolsAndLibraries.wiki/33838/Adding-or-Extending-TSql-support-in-Sql-Dom?anchor=to-extend-the-tests-do-the-following%3A) here to add new tests for your feature Added Test. --- SqlScriptDom/Parser/TSql/Ast.xml | 1 + .../Parser/TSql/CodeGenerationSupporter.cs | 2 + SqlScriptDom/Parser/TSql/TSql170.g | 70 +++++++++++- .../SqlScriptGeneratorVisitor.FunctionCall.cs | 26 +++++ .../Baselines170/JsonFunctionTests170.sql | 101 ++++++++++++++++++ Test/SqlDom/Only170SyntaxTests.cs | 3 +- Test/SqlDom/ParserErrorsTests.cs | 45 ++++++++ .../TestScripts/JsonFunctionTests170.sql | 98 +++++++++++++++++ 8 files changed, 344 insertions(+), 2 deletions(-) create mode 100644 Test/SqlDom/Baselines170/JsonFunctionTests170.sql create mode 100644 Test/SqlDom/TestScripts/JsonFunctionTests170.sql diff --git a/SqlScriptDom/Parser/TSql/Ast.xml b/SqlScriptDom/Parser/TSql/Ast.xml index 49c7bdb..058571f 100644 --- a/SqlScriptDom/Parser/TSql/Ast.xml +++ b/SqlScriptDom/Parser/TSql/Ast.xml @@ -642,6 +642,7 @@ + diff --git a/SqlScriptDom/Parser/TSql/CodeGenerationSupporter.cs b/SqlScriptDom/Parser/TSql/CodeGenerationSupporter.cs index cd727c7..bd051e6 100644 --- a/SqlScriptDom/Parser/TSql/CodeGenerationSupporter.cs +++ b/SqlScriptDom/Parser/TSql/CodeGenerationSupporter.cs @@ -519,6 +519,7 @@ internal static class CodeGenerationSupporter internal const string Json = "JSON"; internal const string JsonArray = "JSON_ARRAY"; internal const string JsonObject = "JSON_OBJECT"; + internal const string JsonObjectAgg = "JSON_OBJECTAGG"; internal const string Keep = "KEEP"; internal const string KeepDefaults = "KEEPDEFAULTS"; internal const string KeepFixed = "KEEPFIXED"; @@ -848,6 +849,7 @@ internal static class CodeGenerationSupporter internal const string RetentionDays = "RETENTION_DAYS"; internal const string RetentionPeriod = "RETENTION_PERIOD"; internal const string Returns = "RETURNS"; + internal const string Returning = "RETURNING"; internal const string RequestMaxCpuTimeSec = "REQUEST_MAX_CPU_TIME_SEC"; internal const string RequestMaxMemoryGrantPercent = "REQUEST_MAX_MEMORY_GRANT_PERCENT"; internal const string RequestMemoryGrantTimeoutSec = "REQUEST_MEMORY_GRANT_TIMEOUT_SEC"; diff --git a/SqlScriptDom/Parser/TSql/TSql170.g b/SqlScriptDom/Parser/TSql/TSql170.g index e43eda1..db65649 100644 --- a/SqlScriptDom/Parser/TSql/TSql170.g +++ b/SqlScriptDom/Parser/TSql/TSql170.g @@ -32066,6 +32066,36 @@ expressionList [TSqlFragment vParent, IList expressions] | /* empty */ ) + ( + jsonReturningClause[vParent] + | + /* empty */ + ) + ; + + jsonObjectAggExpressionList [FunctionCall vParent] + { + JsonKeyValue vExpression; + } + : + ( + vExpression=jsonKeyValueExpression + { + AddAndUpdateTokenInfo(vParent, vParent.JsonParameters, vExpression); + } + | + /* empty */ + ) + ( + jsonNullClauseFunction[vParent] + | + /* empty */ + ) + ( + jsonReturningClause[vParent] + | + /* empty */ + ) ; jsonNullClauseFunction [FunctionCall vParent] @@ -32092,7 +32122,23 @@ expressionList [TSqlFragment vParent, IList expressions] } ) ; - + +jsonReturningClause [FunctionCall vParent] +{ + Identifier vJson; +} +: + tReturning:Identifier tJson:Identifier + { + Match(tReturning, CodeGenerationSupporter.Returning); + Match(tJson, CodeGenerationSupporter.Json); + UpdateTokenInfo(vParent,tJson); + vJson = FragmentFactory.CreateFragment(); + AddAndUpdateTokenInfo(vParent, vParent.ReturnType, vJson); + vJson.SetUnquotedIdentifier(tJson.getText()); + } +; + jsonKeyValueExpression returns [JsonKeyValue vResult = FragmentFactory.CreateFragment()] { ScalarExpression vKey; @@ -32388,6 +32434,9 @@ builtInFunctionCall returns [FunctionCall vResult = FragmentFactory.CreateFragme | {(vResult.FunctionName != null && vResult.FunctionName.Value.ToUpper(CultureInfo.InvariantCulture) == CodeGenerationSupporter.JsonObject)}? jsonObjectBuiltInFunctionCall[vResult] + | + {(vResult.FunctionName != null && vResult.FunctionName.Value.ToUpper(CultureInfo.InvariantCulture) == CodeGenerationSupporter.JsonObjectAgg)}? + jsonObjectAggBuiltInFunctionCall[vResult] | {(vResult.FunctionName != null && vResult.FunctionName.Value.ToUpper(CultureInfo.InvariantCulture) == CodeGenerationSupporter.Trim) && (NextTokenMatches(CodeGenerationSupporter.Leading) | NextTokenMatches(CodeGenerationSupporter.Trailing) | NextTokenMatches(CodeGenerationSupporter.Both))}? @@ -32421,6 +32470,11 @@ jsonArrayBuiltInFunctionCall [FunctionCall vParent] | /* empty */ ) + ( + jsonReturningClause[vParent] + | + /* empty */ + ) tRParen:RightParenthesis { UpdateTokenInfo(vParent, tRParen); @@ -32443,6 +32497,20 @@ jsonObjectBuiltInFunctionCall [FunctionCall vParent] } ; +jsonObjectAggBuiltInFunctionCall [FunctionCall vParent] +{ +} + : ( + jsonObjectAggExpressionList[vParent] + | + /* empty */ + ) + tRParen:RightParenthesis + { + UpdateTokenInfo(vParent, tRParen); + } + ; + regularBuiltInFunctionCall [FunctionCall vParent] { ColumnReferenceExpression vColumn; diff --git a/SqlScriptDom/ScriptDom/SqlServer/ScriptGenerator/SqlScriptGeneratorVisitor.FunctionCall.cs b/SqlScriptDom/ScriptDom/SqlServer/ScriptGenerator/SqlScriptGeneratorVisitor.FunctionCall.cs index daf325d..f4e50f9 100644 --- a/SqlScriptDom/ScriptDom/SqlServer/ScriptGenerator/SqlScriptGeneratorVisitor.FunctionCall.cs +++ b/SqlScriptDom/ScriptDom/SqlServer/ScriptGenerator/SqlScriptGeneratorVisitor.FunctionCall.cs @@ -55,6 +55,20 @@ public override void ExplicitVisit(FunctionCall node) if (node.JsonParameters?.Count > 0 && node.AbsentOrNullOnNull?.Count > 0) //If there are values and null on null or absent on null present then generate space in between them GenerateSpace(); GenerateNullOnNullOrAbsentOnNull(node?.AbsentOrNullOnNull); + if (node.JsonParameters?.Count > 0 && node.ReturnType?.Count > 0) //If there are values and null on null or absent on null present then generate space in between them + GenerateSpace(); + GenerateReturnType(node?.ReturnType); + GenerateSymbol(TSqlTokenType.RightParenthesis); + } + else if (node.FunctionName.Value.ToUpper(CultureInfo.InvariantCulture) == CodeGenerationSupporter.JsonObjectAgg) + { + GenerateCommaSeparatedList(node.JsonParameters); + if (node.JsonParameters?.Count > 0 && node.AbsentOrNullOnNull?.Count > 0) //If there are values and null on null or absent on null present then generate space in between them + GenerateSpace(); + GenerateNullOnNullOrAbsentOnNull(node?.AbsentOrNullOnNull); + if (node.JsonParameters?.Count > 0 && node.ReturnType?.Count > 0) //If there are values and null on null or absent on null present then generate space in between them + GenerateSpace(); + GenerateReturnType(node?.ReturnType); GenerateSymbol(TSqlTokenType.RightParenthesis); } else if (node.FunctionName.Value.ToUpper(CultureInfo.InvariantCulture) == CodeGenerationSupporter.JsonArray) @@ -63,6 +77,9 @@ public override void ExplicitVisit(FunctionCall node) if (node.Parameters?.Count > 0 && node?.AbsentOrNullOnNull?.Count > 0) //If there are values and null on null or absent on null present then generate space in between them GenerateSpace(); GenerateNullOnNullOrAbsentOnNull(node?.AbsentOrNullOnNull); + if (node.ReturnType?.Count > 0) //If there are values and null on null or absent on null present then generate space in between them + GenerateSpace(); + GenerateReturnType(node?.ReturnType); GenerateSymbol(TSqlTokenType.RightParenthesis); } else @@ -117,5 +134,14 @@ private void GenerateNullOnNullOrAbsentOnNull(IList list) GenerateKeyword(TSqlTokenType.Null); } } + private void GenerateReturnType(IList list) + { + if (list?.Count > 0 && list[0].Value?.ToUpper(CultureInfo.InvariantCulture) == CodeGenerationSupporter.Json) + { + GenerateIdentifier("RETURNING"); + GenerateSpace(); + GenerateSpaceSeparatedList(list); + } + } } } diff --git a/Test/SqlDom/Baselines170/JsonFunctionTests170.sql b/Test/SqlDom/Baselines170/JsonFunctionTests170.sql new file mode 100644 index 0000000..29adc38 --- /dev/null +++ b/Test/SqlDom/Baselines170/JsonFunctionTests170.sql @@ -0,0 +1,101 @@ +SELECT id, + json_col +FROM tab1 +WHERE ISJSON(json_col) = 1; + +SELECT id, + json_col +FROM tab1 +WHERE ISJSON(json_col, SCALAR) = 1; + +SELECT ISJSON('true', VALUE); + +DECLARE @jsonInfo AS NVARCHAR (MAX); + +SET @jsonInfo = N'{"info":{"address":[{"town":"Paris"},{"town":"London"}]}}'; + +SELECT JSON_PATH_EXISTS(@jsonInfo, '$.info.address'); + +SELECT JSON_OBJECT('name':'value'); + +SELECT JSON_OBJECT('name':'value', 'type':1 NULL ON NULL); + +SELECT JSON_OBJECT(NULL ON NULL); + +SELECT JSON_OBJECT('name':'value', 'type':NULL ABSENT ON NULL); + +SELECT JSON_OBJECT('name':'value', 'type':JSON_OBJECT('type_id':1, 'name':'a')); + +SELECT JSON_OBJECT(); + +SELECT JSON_OBJECT('name':'value', 'type':1); + +SELECT JSON_OBJECT('name':'value', 'type':JSON_ARRAY(1, 2)); + +DECLARE @id_key AS NVARCHAR (10) = N'id', @id_value AS NVARCHAR (64) = NEWID(); + +SELECT JSON_OBJECT('user_name':USER_NAME(), @id_key:@id_value, 'sid':(SELECT @@SPID)); + +SELECT s.session_id, + JSON_OBJECT('security_id':s.security_id, 'login':s.login_name, 'status':s.status) AS info +FROM sys.dm_exec_sessions AS s +WHERE s.is_user_process = 1; + +SELECT JSON_OBJECT('name':'b' RETURNING JSON); + +SELECT JSON_OBJECT('name':'b' NULL ON NULL RETURNING JSON); + +SELECT JSON_OBJECT('name':'b' ABSENT ON NULL RETURNING JSON); + +SELECT JSON_ARRAY('a', JSON_OBJECT('name':'value', 'type':1 NULL ON NULL) NULL ON NULL); + +SELECT JSON_ARRAY(); + +SELECT JSON_ARRAY('name'); + +SELECT JSON_ARRAY('a', 1, 'b', 2); + +SELECT JSON_ARRAY('a', 1, NULL, 2 NULL ON NULL); + +SELECT JSON_ARRAY('a', 1, NULL, 2 ABSENT ON NULL); + +SELECT JSON_ARRAY(NULL ON NULL); + +SELECT JSON_ARRAY(ABSENT ON NULL); + +DECLARE @id_value AS NVARCHAR (64) = NEWID(); + +SELECT JSON_ARRAY(1, @id_value, (SELECT @@SPID)); + +SELECT s.session_id, + JSON_ARRAY(s.host_name, s.program_name, s.client_interface_name) +FROM sys.dm_exec_sessions AS s +WHERE s.is_user_process = 1; + +SELECT JSON_ARRAY('a', 1, NULL, 2 RETURNING JSON); + +SELECT JSON_ARRAY('a', 1, NULL, 2 NULL ON NULL RETURNING JSON); + +SELECT JSON_ARRAY('a', 1, NULL, 2 ABSENT ON NULL RETURNING JSON); + +SELECT JSON_OBJECTAGG('name':'value'); + +SELECT JSON_OBJECTAGG('name':'value' NULL ON NULL); + +SELECT JSON_OBJECTAGG(NULL ON NULL); + +SELECT JSON_OBJECTAGG('name':NULL ABSENT ON NULL); + +SELECT JSON_OBJECTAGG('name':JSON_OBJECT('type_id':1, 'name':'a')); + +SELECT JSON_OBJECTAGG(); + +SELECT JSON_OBJECTAGG('name':1); + +SELECT JSON_OBJECTAGG('name':JSON_ARRAY(1, 2)); +SELECT JSON_OBJECTAGG('name':'b' NULL ON NULL RETURNING JSON); + +SELECT JSON_OBJECTAGG('name':'b' ABSENT ON NULL RETURNING JSON); + +SELECT JSON_OBJECTAGG('name':'b' RETURNING JSON); + diff --git a/Test/SqlDom/Only170SyntaxTests.cs b/Test/SqlDom/Only170SyntaxTests.cs index b63b2b0..7391e5b 100644 --- a/Test/SqlDom/Only170SyntaxTests.cs +++ b/Test/SqlDom/Only170SyntaxTests.cs @@ -15,7 +15,8 @@ public partial class SqlDomTests new ParserTest170("AlterDatabaseManualCutoverTests170.sql", nErrors80: 4, nErrors90: 4, nErrors100: 4, nErrors110: 4, nErrors120: 4, nErrors130: 4, nErrors140: 4, nErrors150: 4, nErrors160: 4), new ParserTest170("CreateColumnStoreIndexTests170.sql", nErrors80: 3, nErrors90: 3, nErrors100: 3, nErrors110: 3, nErrors120: 3, nErrors130: 0, nErrors140: 0, nErrors150: 0, nErrors160: 0), new ParserTest170("RegexpTests170.sql", nErrors80: 0, nErrors90: 0, nErrors100: 0, nErrors110: 0, nErrors120: 0, nErrors130: 0, nErrors140: 0, nErrors150: 0, nErrors160: 0), - new ParserTest170("AiGenerateChunksTests170.sql", nErrors80: 21, nErrors90: 18, nErrors100: 17, nErrors110: 17, nErrors120: 17, nErrors130: 17, nErrors140: 17, nErrors150: 17, nErrors160: 17) + new ParserTest170("AiGenerateChunksTests170.sql", nErrors80: 21, nErrors90: 18, nErrors100: 17, nErrors110: 17, nErrors120: 17, nErrors130: 17, nErrors140: 17, nErrors150: 17, nErrors160: 17), + new ParserTest170("JsonFunctionTests170.sql", nErrors80: 9, nErrors90: 8, nErrors100: 30, nErrors110: 30, nErrors120: 30, nErrors130: 30, nErrors140: 30, nErrors150: 30, nErrors160: 30), }; private static readonly ParserTest[] SqlAzure170_TestInfos = diff --git a/Test/SqlDom/ParserErrorsTests.cs b/Test/SqlDom/ParserErrorsTests.cs index d075563..90df4e7 100644 --- a/Test/SqlDom/ParserErrorsTests.cs +++ b/Test/SqlDom/ParserErrorsTests.cs @@ -503,6 +503,51 @@ public void JsonArraySyntaxNegativeTest() new ParserErrorInfo(48, "SQL46010", "ON")); } + /// + /// Negative tests for JSON_OBJECTAGG syntax in functions + /// + [TestMethod] + [Priority(0)] + [SqlStudioTestCategory(Category.UnitTest)] + public void JsonObjectAGGSyntaxNegativeTest() + { + // Incorrect key value parameter number + ParserTestUtils.ErrorTest170("SELECT JSON_OBJECTAGG('name':'value', 'type':1)", + new ParserErrorInfo(36, "SQL46010", ",")); + + // No closing quotation mark + ParserTestUtils.ErrorTest170("SELECT JSON_OBJECTAGG('name':'value)", + new ParserErrorInfo(29, "SQL46030", "'value)")); + + // Incorrect placing of colon + ParserTestUtils.ErrorTest170("SELECT JSON_OBJECTAGG('name':'value''type':1)", + new ParserErrorInfo(42, "SQL46010", ":")); + + // cannot use expression without colon + ParserTestUtils.ErrorTest170("SELECT JSON_OBJECTAGG('name')", + new ParserErrorInfo(22, "SQL46010", "'name'")); + + // cannot use expression without colon + ParserTestUtils.ErrorTest170("SELECT JSON_OBJECTAGG('name' ABSENT ON NULL)", + new ParserErrorInfo(29, "SQL46010", "ABSENT")); + + // Cannot use empty value + ParserTestUtils.ErrorTest170("SELECT JSON_OBJECTAGG('name':)", + new ParserErrorInfo(29, "SQL46010", ")")); + + // Cannot use incomplete absent on null clause cases + ParserTestUtils.ErrorTest170("SELECT JSON_OBJECTAGG('name':NULL ABSENT ON)", + new ParserErrorInfo(43, "SQL46010", ")")); + + // Cannot use Incomplete RETURNING clause + ParserTestUtils.ErrorTest170("SELECT JSON_OBJECTAGG('name':NULL RETURNING ON)", + new ParserErrorInfo(46, "SQL46010", ")")); + + // Cannot use anything other than JSON in RETURNING clause + ParserTestUtils.ErrorTest170("SELECT JSON_OBJECTAGG('name':NULL RETURNING INT)", + new ParserErrorInfo(44, "SQL46005", "JSON", "INT")); + } + /// /// Negative tests for Data Masking Alter Column syntax. /// diff --git a/Test/SqlDom/TestScripts/JsonFunctionTests170.sql b/Test/SqlDom/TestScripts/JsonFunctionTests170.sql new file mode 100644 index 0000000..d15a477 --- /dev/null +++ b/Test/SqlDom/TestScripts/JsonFunctionTests170.sql @@ -0,0 +1,98 @@ +-- ISJSON +SELECT id, json_col +FROM tab1 +WHERE ISJSON(json_col) = 1 + +SELECT id, json_col +FROM tab1 +WHERE ISJSON(json_col, SCALAR) = 1 + +SELECT ISJSON('true', VALUE) + +-- JSON_PATH_EXISTS +DECLARE @jsonInfo NVARCHAR(MAX); +SET @jsonInfo=N'{"info":{"address":[{"town":"Paris"},{"town":"London"}]}}'; +SELECT JSON_PATH_EXISTS(@jsonInfo,'$.info.address'); + +SELECT JSON_OBJECT('name':'value'); + +SELECT JSON_OBJECT('name':'value', 'type':1 NULL ON NULL); + +SELECT JSON_OBJECT(NULL ON NULL); + +SELECT JSON_OBJECT('name':'value', 'type':NULL ABSENT ON NULL); + +SELECT JSON_OBJECT('name':'value', 'type':JSON_OBJECT('type_id':1, 'name':'a')); + +SELECT JSON_OBJECT(); + +SELECT JSON_OBJECT('name':'value', 'type':1); + +SELECT JSON_OBJECT('name':'value', 'type':JSON_ARRAY(1, 2)); + +DECLARE @id_key AS NVARCHAR (10) = N'id', @id_value AS NVARCHAR (64) = NEWID(); + +SELECT JSON_OBJECT('user_name':USER_NAME(), @id_key:@id_value, 'sid':(SELECT @@SPID)); + +SELECT s.session_id, + JSON_OBJECT('security_id':s.security_id, 'login':s.login_name, 'status':s.status) AS info +FROM sys.dm_exec_sessions AS s +WHERE s.is_user_process = 1; + +SELECT JSON_OBJECT('name':'b' RETURNING JSON); + +SELECT JSON_OBJECT('name':'b' NULL ON NULL RETURNING JSON); + +SELECT JSON_OBJECT('name':'b' ABSENT ON NULL RETURNING JSON); + +SELECT JSON_ARRAY('a', JSON_OBJECT('name':'value', 'type':1 NULL ON NULL) NULL ON NULL); + +SELECT JSON_ARRAY(); + +SELECT JSON_ARRAY('name'); + +SELECT JSON_ARRAY('a', 1, 'b', 2); + +SELECT JSON_ARRAY('a', 1, NULL, 2 NULL ON NULL); + +SELECT JSON_ARRAY('a', 1, NULL, 2 ABSENT ON NULL); + +SELECT JSON_ARRAY(NULL ON NULL); + +SELECT JSON_ARRAY(ABSENT ON NULL); + +DECLARE @id_value AS NVARCHAR (64) = NEWID(); + +SELECT JSON_ARRAY(1, @id_value, (SELECT @@SPID)); + +SELECT s.session_id, + JSON_ARRAY(s.host_name, s.program_name, s.client_interface_name) +FROM sys.dm_exec_sessions AS s +WHERE s.is_user_process = 1; + +SELECT JSON_ARRAY('a', 1, NULL, 2 RETURNING JSON); + +SELECT JSON_ARRAY('a', 1, NULL, 2 NULL ON NULL RETURNING JSON); + +SELECT JSON_ARRAY('a', 1, NULL, 2 ABSENT ON NULL RETURNING JSON); + +SELECT JSON_OBJECTAGG('name':'value'); + +SELECT JSON_OBJECTAGG('name':'value' NULL ON NULL); + +SELECT JSON_OBJECTAGG(NULL ON NULL); + +SELECT JSON_OBJECTAGG('name':NULL ABSENT ON NULL); + +SELECT JSON_OBJECTAGG('name':JSON_OBJECT('type_id':1, 'name':'a')); + +SELECT JSON_OBJECTAGG(); + +SELECT JSON_OBJECTAGG('name':1); + +SELECT JSON_OBJECTAGG('name':JSON_ARRAY(1, 2)); +SELECT JSON_OBJECTAGG('name':'b' NULL ON NULL RETURNING JSON); + +SELECT JSON_OBJECTAGG('name':'b' ABSENT ON NULL RETURNING JSON); + +SELECT JSON_OBJECTAGG('name':'b' RETURNING JSON); From 05e099332c1525b7adc3f616311d2e2b7809e17b Mon Sep 17 00:00:00 2001 From: Sicong Liu Date: Wed, 30 Jul 2025 18:30:48 +0000 Subject: [PATCH 05/12] Merged PR 1724727: [ai_generate_embeddings] Add syntax to ScriptDom # Pull Request Template for ScriptDom ## Description This PR adds support for the new `AI_GENERATE_EMBEDDINGS` T-SQL built-in function syntax: ```sql AI_GENERATE_EMBEDDINGS('input text' USE MODEL model_name [PARAMETERS (...optional JSON...)]) ``` Input expression: required string or expression representing the text to embed. USE MODEL model_name: required keyword clause specifying the embedding model. PARAMETERS ( ... ): optional block allowing additional JSON parameters to customize embedding behavior. ### Examples ```sql -- Basic usage with required input and model SELECT AI_GENERATE_EMBEDDINGS('Hello world' USE MODEL MyEmbeddingModel); -- Usage with optional PARAMETERS block SELECT AI_GENERATE_EMBEDDINGS('Hello world' USE MODEL MyEmbeddingModel PARAMETERS (TRY_CONVERT(JSON, N'{"param":"value"}'))); ``` ### Features - Adds `AiGenerateEmbeddingsFunctionCall` AST node. - Grammar recognizes required `USE MODEL` clause and optional `PARAMETERS` block. - Script generator visitor outputs syntax matching expected SQL Server formatting, with or without optional PARAMETERS. - Comprehensive unit tests cover: - Missing required keywords or values (e.g., missing USE, missing MODEL). - Misplaced keywords (e.g., PARAMETERS before input). - Invalid syntax like missing parentheses or incomplete expressions. - Extra commas or unknown parameters. - Negative tests aligned with expected SQL46010 and SQL46029 parser errors. - Supports round-tripping: parsed AST can be pretty-printed back to valid SQL matching the original input. ### Compatibility - Valid only in SQL Server with compatibility level 170 or higher. ## Code Change - [X] The [Common checklist](https://msdata.visualstudio.com/SQLToolsAndLibraries/_git/Common?path=/Templates/PR%20Checklist%20for%20SQLToolsAndLibraries.md&version=GBmain&_a=preview) has been reviewed and followed - [X] Code changes are accompanied by appropriate unit tests - [X] Identified and included SMEs needed to review code changes - [X] Follow the [steps](https://msdata.visualstudio.com/SQLToolsAndLibraries/_wiki/wikis/SQLToolsAndLibraries.wiki/33838/Adding-or-Extending-TSql-support-in-Sql-Dom?anchor=make-the-changes-in) here to make changes in the code ## Testing - [X] Follow the [steps](https://msdata.visualstudio.com/SQLToolsAndLibraries/_wiki/wikis/SQLToolsAndLibraries.wiki/33838/Adding-or-Extending-TSql-support-in-Sql-Dom?anchor=to-extend-the-tests-do-the-following%3A) here to add new tests for your feature ## Documentation - [ ] Update relevant documentation in the [wiki](https://dev.azure.com/msdata/SQLToolsAndLibraries/_wiki/wikis/SQLToolsAndLibraries.wiki) and the README.md file ---- #### AI description (iteration 1) #### PR Classification New feature implementation to add syntax support for the AI_GENERATE_EMBEDDINGS function in ScriptDom. #### PR Summary This PR introduces a new SQL function, AI_GENERATE_EMBEDDINGS, which allows users to send input text to external AI models and retrieve an em... --- SqlScriptDom/Parser/TSql/Ast.xml | 6 ++ .../Parser/TSql/CodeGenerationSupporter.cs | 2 + SqlScriptDom/Parser/TSql/TSql170.g | 56 ++++++++++++++++ ...torVisitor.AiGenerateEmbeddingsFunction.cs | 48 ++++++++++++++ .../AiGenerateEmbeddingsTests170.sql | 52 +++++++++++++++ Test/SqlDom/Only170SyntaxTests.cs | 1 + Test/SqlDom/ParserErrorsTests.cs | 65 +++++++++++++++++++ .../AiGenerateEmbeddingsTests170.sql | 52 +++++++++++++++ 8 files changed, 282 insertions(+) create mode 100644 SqlScriptDom/ScriptDom/SqlServer/ScriptGenerator/SqlScriptGeneratorVisitor.AiGenerateEmbeddingsFunction.cs create mode 100644 Test/SqlDom/Baselines170/AiGenerateEmbeddingsTests170.sql create mode 100644 Test/SqlDom/TestScripts/AiGenerateEmbeddingsTests170.sql diff --git a/SqlScriptDom/Parser/TSql/Ast.xml b/SqlScriptDom/Parser/TSql/Ast.xml index 058571f..b37cd1a 100644 --- a/SqlScriptDom/Parser/TSql/Ast.xml +++ b/SqlScriptDom/Parser/TSql/Ast.xml @@ -4725,4 +4725,10 @@ + + + + + + diff --git a/SqlScriptDom/Parser/TSql/CodeGenerationSupporter.cs b/SqlScriptDom/Parser/TSql/CodeGenerationSupporter.cs index bd051e6..cd11042 100644 --- a/SqlScriptDom/Parser/TSql/CodeGenerationSupporter.cs +++ b/SqlScriptDom/Parser/TSql/CodeGenerationSupporter.cs @@ -100,6 +100,7 @@ internal static class CodeGenerationSupporter internal const string After = "AFTER"; internal const string Aggregate = "AGGREGATE"; internal const string AiGenerateChunks = "AI_GENERATE_CHUNKS"; + internal const string AIGenerateEmbeddings = "AI_GENERATE_EMBEDDINGS"; internal const string Algorithm = "ALGORITHM"; internal const string AlterColumn = "ALTERCOLUMN"; internal const string All = "ALL"; @@ -1067,6 +1068,7 @@ internal static class CodeGenerationSupporter internal const string Unpivot = "UNPIVOT"; internal const string UpdLock = "UPDLOCK"; internal const string Url = "URL"; + internal const string Use = "USE"; internal const string Used = "USED"; internal const string UseIdentity = "USE_IDENTITY"; internal const string UseTypeDefault = "USE_TYPE_DEFAULT"; diff --git a/SqlScriptDom/Parser/TSql/TSql170.g b/SqlScriptDom/Parser/TSql/TSql170.g index db65649..3730c92 100644 --- a/SqlScriptDom/Parser/TSql/TSql170.g +++ b/SqlScriptDom/Parser/TSql/TSql170.g @@ -31733,6 +31733,9 @@ expressionPrimary [ExpressionFlags expressionFlags] returns [PrimaryExpression v | {NextTokenMatches(CodeGenerationSupporter.IIf) && (LA(2) == LeftParenthesis)}? vResult=iIfCall + | + {NextTokenMatches(CodeGenerationSupporter.AIGenerateEmbeddings) && LA(2) == LeftParenthesis}? + vResult = aiGenerateEmbeddingsFunctionCall | (Identifier LeftParenthesis)=> vResult=builtInFunctionCall @@ -31767,6 +31770,58 @@ expressionPrimary [ExpressionFlags expressionFlags] returns [PrimaryExpression v collationOpt[vResult] ; +aiGenerateEmbeddingsFunctionCall + returns [AIGenerateEmbeddingsFunctionCall vResult = this.FragmentFactory.CreateFragment()] +{ + ScalarExpression vInput; + SchemaObjectName vModelName; + ScalarExpression vParams = null; +} + : + tFunc:Identifier LeftParenthesis + { + Match(tFunc, CodeGenerationSupporter.AIGenerateEmbeddings); + UpdateTokenInfo(vResult, tFunc); + } + vInput=expression + { + vResult.Input = vInput; + } + + tUse:Use // ← your reserved keyword + { + UpdateTokenInfo(vResult, tUse); + } + + tModel:Identifier + { + Match(tModel, CodeGenerationSupporter.Model); + } + + vModelName=schemaObjectThreePartName + { + vResult.ModelName = vModelName; + } + + ( + tParams:Identifier + { + Match(tParams, CodeGenerationSupporter.Parameters); + } + LeftParenthesis + vParams=expression + RightParenthesis + { + vResult.OptionalParameters = vParams; + } + )? + + tRParen:RightParenthesis + { + UpdateTokenInfo(vResult, tRParen); + } + ; + parenthesisDisambiguatorForExpressions [ExpressionFlags expressionFlags] returns [PrimaryExpression vResult] : @@ -33916,6 +33971,7 @@ securityStatementPermission returns [Identifier vResult = this.FragmentFactory.C } : tId:Identifier + | AiGenerateEmbeddings | Add | All | Alter diff --git a/SqlScriptDom/ScriptDom/SqlServer/ScriptGenerator/SqlScriptGeneratorVisitor.AiGenerateEmbeddingsFunction.cs b/SqlScriptDom/ScriptDom/SqlServer/ScriptGenerator/SqlScriptGeneratorVisitor.AiGenerateEmbeddingsFunction.cs new file mode 100644 index 0000000..0efdc60 --- /dev/null +++ b/SqlScriptDom/ScriptDom/SqlServer/ScriptGenerator/SqlScriptGeneratorVisitor.AiGenerateEmbeddingsFunction.cs @@ -0,0 +1,48 @@ +//------------------------------------------------------------------------------ +// +// Copyright (c) Microsoft Corporation. All rights reserved. +// +//------------------------------------------------------------------------------ + +using Microsoft.SqlServer.TransactSql.ScriptDom; + +namespace Microsoft.SqlServer.TransactSql.ScriptDom.ScriptGenerator +{ + partial class SqlScriptGeneratorVisitor + { + public override void ExplicitVisit(AIGenerateEmbeddingsFunctionCall node) + { + if (node.Input == null || node.ModelName == null) + { + return; + } + + // Emit function name without extra space before '(' + GenerateIdentifierWithoutCasing(CodeGenerationSupporter.AIGenerateEmbeddings); + GenerateSymbol(TSqlTokenType.LeftParenthesis); + + // Emit input expression + GenerateFragmentIfNotNull(node.Input); + + // Emit " USE MODEL" with explicit spaces + GenerateSpace(); + GenerateKeyword(TSqlTokenType.Use); + GenerateSpace(); + GenerateIdentifierWithoutCasing(CodeGenerationSupporter.Model); + GenerateSpaceAndFragmentIfNotNull(node.ModelName); + + // Emit optional PARAMETERS block + if (node.OptionalParameters != null) + { + GenerateSpace(); + GenerateIdentifierWithoutCasing(CodeGenerationSupporter.Parameters); + GenerateSpaceAndSymbol(TSqlTokenType.LeftParenthesis); + GenerateFragmentIfNotNull(node.OptionalParameters); + GenerateSymbol(TSqlTokenType.RightParenthesis); + } + + // Emit closing parenthesis + GenerateSymbol(TSqlTokenType.RightParenthesis); + } + } +} diff --git a/Test/SqlDom/Baselines170/AiGenerateEmbeddingsTests170.sql b/Test/SqlDom/Baselines170/AiGenerateEmbeddingsTests170.sql new file mode 100644 index 0000000..72493c7 --- /dev/null +++ b/Test/SqlDom/Baselines170/AiGenerateEmbeddingsTests170.sql @@ -0,0 +1,52 @@ +SELECT AI_GENERATE_EMBEDDINGS('My Default Input Text' USE MODEL MyDefaultModel); +SELECT AI_GENERATE_EMBEDDINGS(N'My Default Input Text' USE MODEL MyDefaultModel); +SELECT AI_GENERATE_EMBEDDINGS('My Default Input Text' USE MODEL MyDefaultModel PARAMETERS (TRY_CONVERT (JSON, N'{}'))); +SELECT AI_GENERATE_EMBEDDINGS('My Default Input Text' USE MODEL dbo.MyDefaultModel); +SELECT AI_GENERATE_EMBEDDINGS('My Default Input Text' USE MODEL MyDatabase.dbo.MyDefaultModel); +GO + +CREATE FUNCTION dbo.AI_GENERATE_EMBEDDINGS +(@input NVARCHAR (MAX)) +RETURNS NVARCHAR (MAX) +AS +BEGIN + RETURN CONCAT('Embed: ', @input); +END +GO + +CREATE TABLE dbo.AI_GENERATE_EMBEDDINGS ( + id INT PRIMARY KEY, + data NVARCHAR (MAX) +); +GO + +CREATE VIEW dbo.ai_generate_embeddings +AS +SELECT 'View result' AS result; +GO + +CREATE FUNCTION dbo.MyEmbeddingFunction +( ) +RETURNS TABLE +AS +RETURN + SELECT AI_GENERATE_EMBEDDINGS('Function Input' USE MODEL MyDefaultModel) AS EmbeddingResult +GO + +CREATE VIEW dbo.MyEmbeddingView +AS +SELECT AI_GENERATE_EMBEDDINGS(N'View Input' USE MODEL dbo.MyDefaultModel) AS EmbeddingResult; +GO + +CREATE TABLE dbo.SimpleEmbeddingTable ( + InputText NVARCHAR (MAX), + Embedding AS CONVERT (NVARCHAR (MAX), AI_GENERATE_EMBEDDINGS(InputText USE MODEL MyDefaultModel)) +); +GO + +CREATE PROCEDURE dbo.GetEmbedding +@InputText NVARCHAR (MAX) +AS +BEGIN + SELECT CONVERT (NVARCHAR (MAX), AI_GENERATE_EMBEDDINGS(@InputText USE MODEL MyDefaultModel)) AS Embedding; +END diff --git a/Test/SqlDom/Only170SyntaxTests.cs b/Test/SqlDom/Only170SyntaxTests.cs index 7391e5b..1dc8fc4 100644 --- a/Test/SqlDom/Only170SyntaxTests.cs +++ b/Test/SqlDom/Only170SyntaxTests.cs @@ -17,6 +17,7 @@ public partial class SqlDomTests new ParserTest170("RegexpTests170.sql", nErrors80: 0, nErrors90: 0, nErrors100: 0, nErrors110: 0, nErrors120: 0, nErrors130: 0, nErrors140: 0, nErrors150: 0, nErrors160: 0), new ParserTest170("AiGenerateChunksTests170.sql", nErrors80: 21, nErrors90: 18, nErrors100: 17, nErrors110: 17, nErrors120: 17, nErrors130: 17, nErrors140: 17, nErrors150: 17, nErrors160: 17), new ParserTest170("JsonFunctionTests170.sql", nErrors80: 9, nErrors90: 8, nErrors100: 30, nErrors110: 30, nErrors120: 30, nErrors130: 30, nErrors140: 30, nErrors150: 30, nErrors160: 30), + new ParserTest170("AiGenerateEmbeddingsTests170.sql", nErrors80: 12, nErrors90: 9, nErrors100: 9, nErrors110: 9, nErrors120: 9, nErrors130: 9, nErrors140: 9, nErrors150: 9, nErrors160: 9) }; private static readonly ParserTest[] SqlAzure170_TestInfos = diff --git a/Test/SqlDom/ParserErrorsTests.cs b/Test/SqlDom/ParserErrorsTests.cs index 90df4e7..05fcc97 100644 --- a/Test/SqlDom/ParserErrorsTests.cs +++ b/Test/SqlDom/ParserErrorsTests.cs @@ -7190,5 +7190,70 @@ public void GenerateChunksNegativeTest170() "SELECT * FROM AI_GENERATE_CHUNKS (source = 'some text', chunk_type = other, chunk_size = 5)", new ParserErrorInfo(69, "SQL46010", "other")); } + + + /// + /// Negative tests for AI_GENERATE_EMBEDDINGS syntax + /// + [TestMethod] + [Priority(0)] + [SqlStudioTestCategory(Category.UnitTest)] + public void GenerateEmbeddingsNegativeTest170() + { + // Missing required USE MODEL clause + ParserTestUtils.ErrorTest170( + "SELECT AI_GENERATE_EMBEDDINGS('My Default Input Text')", + new ParserErrorInfo(53, "SQL46010", ")")); + + // Missing model name after USE MODEL + ParserTestUtils.ErrorTest170( + "SELECT AI_GENERATE_EMBEDDINGS('My Default Input Text' USE MODEL)", + new ParserErrorInfo(63, "SQL46010", ")")); + + // Missing USE keyword before MODEL + ParserTestUtils.ErrorTest170( + "SELECT AI_GENERATE_EMBEDDINGS('My Default Input Text' MODEL MyModel)", + new ParserErrorInfo(54, "SQL46010", "MODEL")); + + // USE keyword misplaced before input expression + ParserTestUtils.ErrorTest170( + "SELECT AI_GENERATE_EMBEDDINGS(USE MODEL MyModel 'My Default Input Text')", + new ParserErrorInfo(30, "SQL46010", "USE")); + + // PARAMETERS specified without USE MODEL + ParserTestUtils.ErrorTest170( + "SELECT AI_GENERATE_EMBEDDINGS('My Default Input Text' PARAMETERS (TRY_CONVERT(JSON, N'{}')))", + new ParserErrorInfo(54, "SQL46010", "PARAMETERS")); + + // PARAMETERS missing parentheses + ParserTestUtils.ErrorTest170( + "SELECT AI_GENERATE_EMBEDDINGS('My Default Input Text' USE MODEL MyModel PARAMETERS TRY_CONVERT(JSON, N'{}'))", + new ParserErrorInfo(83, "SQL46010", "TRY_CONVERT")); + + // Invalid expression inside PARAMETERS (missing closing parenthesis) + ParserTestUtils.ErrorTest170( + "SELECT AI_GENERATE_EMBEDDINGS('My Default Input Text' USE MODEL MyModel PARAMETERS (TRY_CONVERT(JSON, N'{}')", + new ParserErrorInfo(108, "SQL46029", "EOF")); + + // Extra comma at end inside PARAMETERS + ParserTestUtils.ErrorTest170( + "SELECT AI_GENERATE_EMBEDDINGS('My Default Input Text' USE MODEL MyModel PARAMETERS (TRY_CONVERT(JSON, N'{}'),))", + new ParserErrorInfo(108, "SQL46010", ",")); + + // PARAMETERS misplaced before input + ParserTestUtils.ErrorTest170( + "SELECT AI_GENERATE_EMBEDDINGS(PARAMETERS (TRY_CONVERT(JSON, N'{}') 'My Default Input Text'))", + new ParserErrorInfo(67, "SQL46010", "'My Default Input Text'")); + + // Missing MODEL keyword after USE + ParserTestUtils.ErrorTest170( + "SELECT AI_GENERATE_EMBEDDINGS('My Default Input Text' USE MyModel)", + new ParserErrorInfo(58, "SQL46005", "MODEL", "MyModel")); + + // NULL model name after USE MODEL + ParserTestUtils.ErrorTest170( + "SELECT AI_GENERATE_EMBEDDINGS('My Default Input Text' USE MODEL NULL)", + new ParserErrorInfo(64, "SQL46010", "NULL")); + } } } diff --git a/Test/SqlDom/TestScripts/AiGenerateEmbeddingsTests170.sql b/Test/SqlDom/TestScripts/AiGenerateEmbeddingsTests170.sql new file mode 100644 index 0000000..72493c7 --- /dev/null +++ b/Test/SqlDom/TestScripts/AiGenerateEmbeddingsTests170.sql @@ -0,0 +1,52 @@ +SELECT AI_GENERATE_EMBEDDINGS('My Default Input Text' USE MODEL MyDefaultModel); +SELECT AI_GENERATE_EMBEDDINGS(N'My Default Input Text' USE MODEL MyDefaultModel); +SELECT AI_GENERATE_EMBEDDINGS('My Default Input Text' USE MODEL MyDefaultModel PARAMETERS (TRY_CONVERT (JSON, N'{}'))); +SELECT AI_GENERATE_EMBEDDINGS('My Default Input Text' USE MODEL dbo.MyDefaultModel); +SELECT AI_GENERATE_EMBEDDINGS('My Default Input Text' USE MODEL MyDatabase.dbo.MyDefaultModel); +GO + +CREATE FUNCTION dbo.AI_GENERATE_EMBEDDINGS +(@input NVARCHAR (MAX)) +RETURNS NVARCHAR (MAX) +AS +BEGIN + RETURN CONCAT('Embed: ', @input); +END +GO + +CREATE TABLE dbo.AI_GENERATE_EMBEDDINGS ( + id INT PRIMARY KEY, + data NVARCHAR (MAX) +); +GO + +CREATE VIEW dbo.ai_generate_embeddings +AS +SELECT 'View result' AS result; +GO + +CREATE FUNCTION dbo.MyEmbeddingFunction +( ) +RETURNS TABLE +AS +RETURN + SELECT AI_GENERATE_EMBEDDINGS('Function Input' USE MODEL MyDefaultModel) AS EmbeddingResult +GO + +CREATE VIEW dbo.MyEmbeddingView +AS +SELECT AI_GENERATE_EMBEDDINGS(N'View Input' USE MODEL dbo.MyDefaultModel) AS EmbeddingResult; +GO + +CREATE TABLE dbo.SimpleEmbeddingTable ( + InputText NVARCHAR (MAX), + Embedding AS CONVERT (NVARCHAR (MAX), AI_GENERATE_EMBEDDINGS(InputText USE MODEL MyDefaultModel)) +); +GO + +CREATE PROCEDURE dbo.GetEmbedding +@InputText NVARCHAR (MAX) +AS +BEGIN + SELECT CONVERT (NVARCHAR (MAX), AI_GENERATE_EMBEDDINGS(@InputText USE MODEL MyDefaultModel)) AS Embedding; +END From e8e9082db1c06b810a274a6c69a6229f213accac Mon Sep 17 00:00:00 2001 From: Urvashi Raj Date: Mon, 4 Aug 2025 07:12:44 +0000 Subject: [PATCH 06/12] Merged PR 1752033: [modle_management]ScriptDom changes for external model support MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit # Pull Request Template for ScriptDom ## Description Introduces syntax support for Create, Alter and Drop operation for External Model DDL CREATE CREATE EXTERNAL MODEL AUTHORIZATION owner WITH ( LOCATION = 'URL or FILE PATH', API_FORMAT = ‘OpenAI, Azure OpenAI, etc’, MODEL_TYPE = , MODEL = ‘text-embedding-ada-002,etc’, CREDENTIAL = , PARAMETERS = ‘{ “valid”:”JSON”}’ } | **Column Name** | **Data Type** | **Notes** | |-----------------------|---------------------|-----------| | `model_name` | `SYSNAME` | Name of the external model object | | `owner_name` | `SYSNAME` | Database principal that owns the external model object. Defaults to connected user if not specified. | | `location` | `NVARCHAR(4000)` | Location of the endpoint. Can be a URL (`https://...`) or file path (`mountpoint:\\...`). | | `api_format` | `NVARCHAR(100)` | Format of the API the endpoint accepts/produces. P0: Azure OpenAI, OpenAI, Ollama; P1: Local, Nvidia Triton, Databricks; P2: TBD | | `model_type` | `SYSNAME` | Model type for the data plane. P0: EMBEDDINGS; Post GA: CHAT, MODERATE, TRANSLATE, SPEECH. Only supported types are accepted. | | `model` | `NVARCHAR(100)` | Name of the embedding model used by the endpoint. | | `local_runtime_path` | `NVARCHAR(4000)` | Path to local model runtime (e.g., ONNX runtime). Required for local models. | | `credential` | `SYSNAME` | Name of the database scoped credential. Not used for ONNX Runtime. | | `parameters` | `JSON` | Default JSON parameters to be appended to the embeddings payload. | ## Code Change - [X] The [Common checklist](https://msdata.visualstudio.com/SQLToolsAndLibraries/_git/Common?path=/Templates/PR%20Checklist%20for%20SQLToolsAndLibraries.md&version=GBmain&_a=preview) has been reviewed and followed - [X] Code changes are accompanied by appropriate unit tests - [X] Identified and included SMEs needed to review code changes - [X] Follow the [steps](https://msdata.visualstudio.com/SQLToolsAndLibraries/_wiki/wikis/SQLToolsAndLibraries.wiki/33838/Adding-or-Extending-TSql-support-in-Sql-Dom?anchor=make-the-changes-in) here to make changes in the code ## Testing - [X] Follow the [steps](https://msdata.visualstudio.com/SQLToolsAndLibraries/_wiki/wikis/SQLToolsAndLibraries.wiki/33838/Adding-or-Extending-TSql-support-in-Sql-Dom?anchor=to-extend-the-tests-do-the-following%3A) here to add new tests for your feature ## Documentation - [X] Update relevant documentation in the [wiki](https://dev.azure.com/msdata/SQLToolsAndLibraries/_wiki/wikis/SQLToolsAndLibraries.wiki) and the README.md file ## Additional Information Please provide any additional information that might be helpful for the reviewers ScriptDom channges ---- #### AI description (iteration 1) #### PR Classific... --- SqlScriptDom/Parser/TSql/Ast.xml | 27 +- .../Parser/TSql/CodeGenerationSupporter.cs | 5 + .../Parser/TSql/ExternalModelTypeOption.cs | 26 + SqlScriptDom/Parser/TSql/TSql170.g | 626 ++++++++++++------ ...ptGenerator.AlterExternalModelStatement.cs | 121 ++++ ...iptGenerator.DropExternalModelStatement.cs | 23 + ...torVisitor.CreateExternalModelStatement.cs | 122 ++++ .../AlterExternalModelStatementTests170.sql | 27 + .../CreateExternalModelStatementTests170.sql | 33 + .../DropExternalModelStatementTests170.sql | 1 + Test/SqlDom/Only170SyntaxTests.cs | 5 +- Test/SqlDom/ParserErrorsTests.cs | 104 ++- .../AlterExternalModelStatementTests170.sql | 27 + .../CreateExternalModelStatementTests170.sql | 33 + .../DropExternalModelStatementTests170.sql | 1 + 15 files changed, 988 insertions(+), 193 deletions(-) create mode 100644 SqlScriptDom/Parser/TSql/ExternalModelTypeOption.cs create mode 100644 SqlScriptDom/ScriptDom/SqlServer/ScriptGenerator/SqlScriptGenerator.AlterExternalModelStatement.cs create mode 100644 SqlScriptDom/ScriptDom/SqlServer/ScriptGenerator/SqlScriptGenerator.DropExternalModelStatement.cs create mode 100644 SqlScriptDom/ScriptDom/SqlServer/ScriptGenerator/SqlScriptGeneratorVisitor.CreateExternalModelStatement.cs create mode 100644 Test/SqlDom/Baselines170/AlterExternalModelStatementTests170.sql create mode 100644 Test/SqlDom/Baselines170/CreateExternalModelStatementTests170.sql create mode 100644 Test/SqlDom/Baselines170/DropExternalModelStatementTests170.sql create mode 100644 Test/SqlDom/TestScripts/AlterExternalModelStatementTests170.sql create mode 100644 Test/SqlDom/TestScripts/CreateExternalModelStatementTests170.sql create mode 100644 Test/SqlDom/TestScripts/DropExternalModelStatementTests170.sql diff --git a/SqlScriptDom/Parser/TSql/Ast.xml b/SqlScriptDom/Parser/TSql/Ast.xml index b37cd1a..942ac65 100644 --- a/SqlScriptDom/Parser/TSql/Ast.xml +++ b/SqlScriptDom/Parser/TSql/Ast.xml @@ -403,8 +403,8 @@ Summary="Actual JSON for clause options. First one is always present (JSON mode)."/> - - + + @@ -1347,6 +1347,27 @@ + + + + + + + + + + + + + + + + + + + + + @@ -3559,7 +3580,7 @@ - + diff --git a/SqlScriptDom/Parser/TSql/CodeGenerationSupporter.cs b/SqlScriptDom/Parser/TSql/CodeGenerationSupporter.cs index cd11042..2e6edfb 100644 --- a/SqlScriptDom/Parser/TSql/CodeGenerationSupporter.cs +++ b/SqlScriptDom/Parser/TSql/CodeGenerationSupporter.cs @@ -120,6 +120,7 @@ internal static class CodeGenerationSupporter internal const string Always = "ALWAYS"; internal const string Anonymous = "ANONYMOUS"; internal const string AnsiNullDefault = "ANSI_NULL_DEFAULT"; + internal const string ApiFormat = "API_FORMAT"; internal const string Application = "APPLICATION"; internal const string ApplicationLog = "APPLICATION_LOG"; internal const string Apply = "APPLY"; @@ -333,6 +334,7 @@ internal static class CodeGenerationSupporter internal const string Edition = "EDITION"; internal const string ElasticPool = "ELASTIC_POOL"; internal const string Elements = "ELEMENTS"; + internal const string Embeddings = "EMBEDDINGS"; internal const string Emergency = "EMERGENCY"; internal const string Empty = "EMPTY"; internal const string Enable = "ENABLE"; @@ -562,6 +564,7 @@ internal static class CodeGenerationSupporter internal const string LoadHistory = "LOADHISTORY"; internal const string LobCompaction = "LOB_COMPACTION"; internal const string Local = "LOCAL"; + internal const string LocalRuntimePath = "LOCAL_RUNTIME_PATH"; internal const string Location = "LOCATION"; internal const string LocationUserDB = "USER_DB"; internal const string LocalServiceName = "LOCAL_SERVICE_NAME"; @@ -635,6 +638,8 @@ internal static class CodeGenerationSupporter internal const string Mirror = "MIRROR"; internal const string Mixed = "MIXED"; internal const string MixedPageAllocation = "MIXED_PAGE_ALLOCATION"; + internal const string ModelType = "MODEL_TYPE"; + internal const string ModelName = "MODEL"; internal const string Modify = "MODIFY"; internal const string Money = "MONEY"; internal const string Move = "MOVE"; diff --git a/SqlScriptDom/Parser/TSql/ExternalModelTypeOption.cs b/SqlScriptDom/Parser/TSql/ExternalModelTypeOption.cs new file mode 100644 index 0000000..858784e --- /dev/null +++ b/SqlScriptDom/Parser/TSql/ExternalModelTypeOption.cs @@ -0,0 +1,26 @@ +//------------------------------------------------------------------------------ +// +// Copyright (c) Microsoft Corporation. All rights reserved. +// +//------------------------------------------------------------------------------ +using System; + +namespace Microsoft.SqlServer.TransactSql.ScriptDom +{ +#pragma warning disable 1591 + + /// + /// The enumeration specifies the external model type + /// Currently, we support EMBEDDINGS only. + /// + public enum ExternalModelTypeOption + { + /// + /// MODEL_TYPE = EMBEDDINGS + /// + EMBEDDINGS = 0, + + } + +#pragma warning restore 1591 +} diff --git a/SqlScriptDom/Parser/TSql/TSql170.g b/SqlScriptDom/Parser/TSql/TSql170.g index 3730c92..1276434 100644 --- a/SqlScriptDom/Parser/TSql/TSql170.g +++ b/SqlScriptDom/Parser/TSql/TSql170.g @@ -906,7 +906,7 @@ create2005Statements returns [TSqlStatement vResult = null] vResult=createEventStatement // NOTIFICATION or SESSION | {NextTokenMatches(CodeGenerationSupporter.External)}? - vResult=createExternalStatements // EXTERNAL DATA SOURCE, FILE FORMAT, STREAM, TABLE, RESOURCE POOL, LIBRARY, LANGUAGE + vResult=createExternalStatements // EXTERNAL DATA SOURCE, FILE FORMAT, STREAM, TABLE, RESOURCE POOL, LIBRARY, LANGUAGE, MODEL | {NextTokenMatches(CodeGenerationSupporter.Fulltext)}? vResult=createFulltextStatement // Index or CATALOG @@ -4299,8 +4299,8 @@ alterDatabaseEncryptionKey [IToken tAlter] returns [AlterDatabaseEncryptionKeySt addSensitivityClassificationStatement returns [AddSensitivityClassificationStatement vResult = this.FragmentFactory.CreateFragment()] { ColumnReferenceExpression vColumn; - SensitivityClassificationOption vOption; - long encounteredOptions = 0; + SensitivityClassificationOption vOption; + long encounteredOptions = 0; } : tSensitivity:Identifier tClassification:Identifier To { @@ -4309,24 +4309,24 @@ addSensitivityClassificationStatement returns [AddSensitivityClassificationState } (vColumn = column { - CheckTableNameExistsForColumn(vColumn, true); + CheckTableNameExistsForColumn(vColumn, true); AddAndUpdateTokenInfo(vResult, vResult.Columns, vColumn); } (Comma vColumn = column { - CheckTableNameExistsForColumn(vColumn, true); + CheckTableNameExistsForColumn(vColumn, true); AddAndUpdateTokenInfo(vResult, vResult.Columns, vColumn); } )* ) - With LeftParenthesis vOption = sensitivityClassificationOption + With LeftParenthesis vOption = sensitivityClassificationOption { - CheckOptionDuplication(ref encounteredOptions, (int)vOption.Type, vOption); + CheckOptionDuplication(ref encounteredOptions, (int)vOption.Type, vOption); AddAndUpdateTokenInfo(vResult, vResult.Options, vOption); } (Comma vOption = sensitivityClassificationOption { - CheckOptionDuplication(ref encounteredOptions, (int)vOption.Type, vOption); + CheckOptionDuplication(ref encounteredOptions, (int)vOption.Type, vOption); AddAndUpdateTokenInfo(vResult, vResult.Options, vOption); } )* @@ -4357,10 +4357,10 @@ sensitivityClassificationOption returns [SensitivityClassificationOption vResult break; } - vResult = FragmentFactory.CreateFragment(); + vResult = FragmentFactory.CreateFragment(); vResult.Value = vSensitivityValue; - vResult.Type = optionType; + vResult.Type = optionType; UpdateTokenInfo(vResult, tOption); } @@ -4409,17 +4409,17 @@ dropSensitivityClassificationStatement returns [DropSensitivityClassificationSta } (vColumn = column { - CheckTableNameExistsForColumn(vColumn, true); + CheckTableNameExistsForColumn(vColumn, true); AddAndUpdateTokenInfo(vResult, vResult.Columns, vColumn); } (Comma vColumn = column { - CheckTableNameExistsForColumn(vColumn, true); + CheckTableNameExistsForColumn(vColumn, true); AddAndUpdateTokenInfo(vResult, vResult.Columns, vColumn); } )* ) - ; + ; ////////////////////////////////////////////////////////////////////// // Create Database @@ -6266,7 +6266,7 @@ simpleBulkInsertOptionWithValue returns [LiteralBulkInsertOption vResult = Fragm | iValue = identifier { vResult.OptionKind = BulkInsertStringOptionsHelper.Instance.ParseOption(tOption, SqlVersionFlags.TSql150); - UpdateTokenInfo(vResult, tOption); + UpdateTokenInfo(vResult, tOption); if (vResult.OptionKind == BulkInsertOptionKind.HeaderRow) if(!TryMatch(iValue, CodeGenerationSupporter.True)) Match(iValue, CodeGenerationSupporter.False); @@ -8947,6 +8947,9 @@ createExternalStatements returns [TSqlStatement vResult = null] | {NextTokenMatches(CodeGenerationSupporter.Stream)}? vResult = createExternalStreamStatement + | + {NextTokenMatches(CodeGenerationSupporter.Model)}? + vResult = createExternalModelStatement ) ; @@ -8964,6 +8967,9 @@ alterExternalStatements returns [TSqlStatement vResult = null] | {NextTokenMatches(CodeGenerationSupporter.Language)}? vResult = alterExternalLanguageStatement + | + {NextTokenMatches(CodeGenerationSupporter.Model)}? + vResult = alterExternalModelStatement ) ; @@ -14768,9 +14774,9 @@ dropStatements returns [TSqlStatement vResult] | {NextTokenMatches(CodeGenerationSupporter.Column)}? vResult = dropColumnStatements | {NextTokenMatches(CodeGenerationSupporter.External)}? - vResult = dropExternalStatement // EXTERNAL DATA SOURCE, FILE FORMAT, TABLE or RESOURCE POOL - | {NextTokenMatches(CodeGenerationSupporter.Sensitivity)}? - vResult = dropSensitivityClassificationStatement + vResult = dropExternalStatement // EXTERNAL DATA SOURCE, FILE FORMAT, TABLE , MODEL or RESOURCE POOL + | {NextTokenMatches(CodeGenerationSupporter.Sensitivity)}? + vResult = dropSensitivityClassificationStatement | vResult = dropServerStatements | vResult = dropUserStatement ) @@ -15662,6 +15668,9 @@ dropExternalStatement returns [TSqlStatement vResult = null] | {NextTokenMatches(CodeGenerationSupporter.Resource)}? vResult = dropExternalResourcePoolStatement + | + {NextTokenMatches(CodeGenerationSupporter.Model)}? + vResult = dropExternalModelStatement ) ; @@ -16952,13 +16961,13 @@ createColumnStoreIndexStatement [IToken tUnique, bool? isClustered] returns [Cre ( identifierColumnList[vResult, vResult.OrderedColumns] { - foreach (var col in vResult.OrderedColumns) - { - if (PseudoColumnHelper.IsGraphPseudoColumn(col)) - { + foreach (var col in vResult.OrderedColumns) + { + if (PseudoColumnHelper.IsGraphPseudoColumn(col)) + { ThrowIncorrectSyntaxErrorException(col); } - } + } } ) )? @@ -17184,9 +17193,9 @@ filterExpressionPrimary returns [BooleanExpression vResult] vExpression = filterColumn ( Is - ( - vResult = filterNullPredicate[vExpression] - | + ( + vResult = filterNullPredicate[vExpression] + | ( Not { @@ -17199,7 +17208,7 @@ filterExpressionPrimary returns [BooleanExpression vResult] | vResult = filterDistinctPredicate[vExpression, vNotDefined] ) - ) + ) | vResult = filterComparisonPredicate[vExpression] | vResult = filterInPredicate[vExpression] ) @@ -17252,9 +17261,9 @@ filterDistinctPredicate[ScalarExpression vColumn, bool vNotDefined] returns [Dis ScalarExpression vExpression; } : - vExpression = expression + vExpression = expression { - vResult.FirstExpression = vColumn; + vResult.FirstExpression = vColumn; vResult.SecondExpression = vExpression; vResult.IsNot = vNotDefined; } @@ -17262,9 +17271,9 @@ filterDistinctPredicate[ScalarExpression vColumn, bool vNotDefined] returns [Dis filterNullPredicateFromDistinctPredicate[ScalarExpression vColumn, bool vNotDefined] returns [BooleanIsNullExpression vResult = this.FragmentFactory.CreateFragment()] : - tNull:Null + tNull:Null { - vResult.IsNot = vNotDefined; + vResult.IsNot = vNotDefined; vResult.Expression = vColumn; UpdateTokenInfo(vResult,tNull); } @@ -19235,120 +19244,120 @@ aiGenerateFixedChunksTableReference [ScalarExpression vSource, Identifier vChunk ; predictTableReference[SubDmlFlags subDmlFlags] returns [PredictTableReference vResult] - : - {NextTokenMatches(CodeGenerationSupporter.Predict)}? - tPredict:Identifier LeftParenthesis vResult = predictParams[subDmlFlags, ExpressionFlags.None] tRParen:RightParenthesis predictWithClauseOpt[vResult] simpleTableReferenceAliasOpt[vResult] - { - Match(tPredict, CodeGenerationSupporter.Predict); - UpdateTokenInfo(vResult, tPredict); - UpdateTokenInfo(vResult, tRParen); - } - ; + : + {NextTokenMatches(CodeGenerationSupporter.Predict)}? + tPredict:Identifier LeftParenthesis vResult = predictParams[subDmlFlags, ExpressionFlags.None] tRParen:RightParenthesis predictWithClauseOpt[vResult] simpleTableReferenceAliasOpt[vResult] + { + Match(tPredict, CodeGenerationSupporter.Predict); + UpdateTokenInfo(vResult, tPredict); + UpdateTokenInfo(vResult, tRParen); + } + ; predictParams[SubDmlFlags subDmlFlags, ExpressionFlags expressionFlags] returns [PredictTableReference vResult = FragmentFactory.CreateFragment()] { - ScalarExpression vModelVariable; - ScalarSubquery vModelSubquery; - TableReferenceWithAlias vDataSource; - Identifier vRuntime; -} - : - ( - tModelVariable:Identifier EqualsSign vModelVariable = expression - { - Match(tModelVariable, CodeGenerationSupporter.Model); - vResult.ModelVariable = vModelVariable; - UpdateTokenInfo(vResult, tModelVariable); - } - | tModelSubquery:Identifier EqualsSign vModelSubquery = subquery[SubDmlFlags.SelectNotForInsert, expressionFlags] - { - vResult.ModelSubquery = vModelSubquery; - } - ) - Comma tData:Identifier EqualsSign vDataSource = mergeTarget[false] - { - vResult.DataSource = vDataSource; - } - ( - Comma tRunTime:Identifier EqualsSign vRuntime = identifier - { - vResult.RunTime = vRuntime; - } - )? - ; + ScalarExpression vModelVariable; + ScalarSubquery vModelSubquery; + TableReferenceWithAlias vDataSource; + Identifier vRuntime; +} + : + ( + tModelVariable:Identifier EqualsSign vModelVariable = expression + { + Match(tModelVariable, CodeGenerationSupporter.Model); + vResult.ModelVariable = vModelVariable; + UpdateTokenInfo(vResult, tModelVariable); + } + | tModelSubquery:Identifier EqualsSign vModelSubquery = subquery[SubDmlFlags.SelectNotForInsert, expressionFlags] + { + vResult.ModelSubquery = vModelSubquery; + } + ) + Comma tData:Identifier EqualsSign vDataSource = mergeTarget[false] + { + vResult.DataSource = vDataSource; + } + ( + Comma tRunTime:Identifier EqualsSign vRuntime = identifier + { + vResult.RunTime = vRuntime; + } + )? + ; predictWithClauseOpt [PredictTableReference vParent] - : (With) => - ( - (With LeftParenthesis predictSchemaItemList[vParent] tRParen:RightParenthesis - { - UpdateTokenInfo(vParent,tRParen); - } - ) - ) - ; + : (With) => + ( + (With LeftParenthesis predictSchemaItemList[vParent] tRParen:RightParenthesis + { + UpdateTokenInfo(vParent,tRParen); + } + ) + ) + ; predictSchemaItemList [PredictTableReference vParent] { - SchemaDeclarationItem vItem; + SchemaDeclarationItem vItem; } - : vItem = predictSchemaItem - { - AddAndUpdateTokenInfo(vParent, vParent.SchemaDeclarationItems, vItem); - } - (Comma vItem = predictSchemaItem - { - AddAndUpdateTokenInfo(vParent, vParent.SchemaDeclarationItems, vItem); - } - )* - ; + : vItem = predictSchemaItem + { + AddAndUpdateTokenInfo(vParent, vParent.SchemaDeclarationItems, vItem); + } + (Comma vItem = predictSchemaItem + { + AddAndUpdateTokenInfo(vParent, vParent.SchemaDeclarationItems, vItem); + } + )* + ; predictSchemaItem returns [SchemaDeclarationItem vResult = FragmentFactory.CreateFragment()] { - ValueExpression vMapping; - ColumnDefinitionBase vColumn; -} - : vColumn = columnDefinitionBasic - { - vResult.ColumnDefinition = vColumn; - } - (vMapping = stringLiteral - { - vResult.Mapping = vMapping; - } - )? - (As tPredict:Identifier - { - Match(tPredict, CodeGenerationSupporter.Predict); - } - )? - ; + ValueExpression vMapping; + ColumnDefinitionBase vColumn; +} + : vColumn = columnDefinitionBasic + { + vResult.ColumnDefinition = vColumn; + } + (vMapping = stringLiteral + { + vResult.Mapping = vMapping; + } + )? + (As tPredict:Identifier + { + Match(tPredict, CodeGenerationSupporter.Predict); + } + )? + ; mergeTarget[bool indexHintAllowed] returns [TableReferenceWithAlias vResult] { - Identifier vAlias; -} - : - vResult=dmlTarget[indexHintAllowed] - ( - ( - As vAlias = identifier - { - vResult.Alias = vAlias; - } - ) - | - {!NextTokenMatches(CodeGenerationSupporter.Using)}? - ( - vAlias = identifier - { - vResult.Alias = vAlias; - } - ) - | - /* empty */ - ) - ; + Identifier vAlias; +} + : + vResult=dmlTarget[indexHintAllowed] + ( + ( + As vAlias = identifier + { + vResult.Alias = vAlias; + } + ) + | + {!NextTokenMatches(CodeGenerationSupporter.Using)}? + ( + vAlias = identifier + { + vResult.Alias = vAlias; + } + ) + | + /* empty */ + ) + ; changeTableTableReference returns [TableReferenceWithAliasAndColumns vResult] { @@ -20252,20 +20261,20 @@ insertColumn returns [ColumnReferenceExpression vResult = FragmentFactory.Create openRowsetColumn returns [OpenRowsetColumnDefinition vResult = FragmentFactory.CreateFragment()] { ColumnDefinitionBase vColumn; - IntegerLiteral vColumnOrdinal; - StringLiteral vStringLiteral; + IntegerLiteral vColumnOrdinal; + StringLiteral vStringLiteral; } : vColumn = columnDefinitionBasic - { - vResult.ColumnIdentifier = vColumn.ColumnIdentifier; - vResult.DataType = vColumn.DataType; - vResult.Collation = vColumn.Collation; - } - (vColumnOrdinal=integer + { + vResult.ColumnIdentifier = vColumn.ColumnIdentifier; + vResult.DataType = vColumn.DataType; + vResult.Collation = vColumn.Collation; + } + (vColumnOrdinal=integer { vResult.ColumnOrdinal = vColumnOrdinal; } - | vStringLiteral=stringLiteral + | vStringLiteral=stringLiteral { vResult.JsonPath = vStringLiteral; })? @@ -20910,7 +20919,7 @@ openRowsetCosmos returns [OpenRowsetCosmos vResult = FragmentFactory.CreateFragm { long encountered = 0; OpenRowsetCosmosOption vOption; - OpenRowsetColumnDefinition vColumn; + OpenRowsetColumnDefinition vColumn; } : vOption = openRowsetCosmosOptionHint @@ -20971,7 +20980,7 @@ openRowsetBulk returns [BulkOpenRowset vResult = FragmentFactory.CreateFragment< BulkInsertOption vOption; StringLiteral vDataFile; - OpenRowsetColumnDefinition vColumn; + OpenRowsetColumnDefinition vColumn; } : Bulk (( @@ -21006,10 +21015,10 @@ openRowsetBulk returns [BulkOpenRowset vResult = FragmentFactory.CreateFragment< tRParen:RightParenthesis { CheckForDataFileFormatProhibitedOptionsInOpenRowsetBulk(encountered, vDataFile); - CheckForParquetFormatProhibitedOptionsInOpenRowsetBulk(encountered, vResult); + CheckForParquetFormatProhibitedOptionsInOpenRowsetBulk(encountered, vResult); UpdateTokenInfo(vResult,tRParen); } - (tWith:With + (tWith:With { UpdateTokenInfo(vResult,tWith); } @@ -21645,23 +21654,23 @@ simpleGroupByItem [ref bool alreadyEncounteredDistributedAggHint] returns [Expre { vResult.Expression = vExpression; } - ( - // Greedy due to conflict with withCommonTableExpressionsAndXmlNamespaces - options { greedy = true; } : - With LeftParenthesis tDistributedAgg:Identifier tRParen:RightParenthesis - { - Match(tDistributedAgg, CodeGenerationSupporter.DistributedAgg); + ( + // Greedy due to conflict with withCommonTableExpressionsAndXmlNamespaces + options { greedy = true; } : + With LeftParenthesis tDistributedAgg:Identifier tRParen:RightParenthesis + { + Match(tDistributedAgg, CodeGenerationSupporter.DistributedAgg); - if (alreadyEncounteredDistributedAggHint) - ThrowParseErrorException("SQL46129", tDistributedAgg, TSqlParserResource.SQL46129Message); + if (alreadyEncounteredDistributedAggHint) + ThrowParseErrorException("SQL46129", tDistributedAgg, TSqlParserResource.SQL46129Message); - vResult.DistributedAggregation = true; - UpdateTokenInfo(vResult, tDistributedAgg); - UpdateTokenInfo(vResult, tRParen); + vResult.DistributedAggregation = true; + UpdateTokenInfo(vResult, tDistributedAgg); + UpdateTokenInfo(vResult, tRParen); - alreadyEncounteredDistributedAggHint = true; - } - )? + alreadyEncounteredDistributedAggHint = true; + } + )? ; // End of Group By clause @@ -23385,9 +23394,9 @@ createColumnMasterKeyStatement returns [CreateColumnMasterKeyStatement vResult = columnMasterkeyParameter returns [ColumnMasterKeyParameter vResult] : {NextTokenMatches(CodeGenerationSupporter.KeyStoreProviderName)}? vResult = columnMasterKeyStoreProviderNameParameter - | {NextTokenMatches(CodeGenerationSupporter.KeyPath)}? + | {NextTokenMatches(CodeGenerationSupporter.KeyPath)}? vResult = columnMasterKeyPathParameter - | {NextTokenMatches(CodeGenerationSupporter.EnclaveComputations)}? + | {NextTokenMatches(CodeGenerationSupporter.EnclaveComputations)}? vResult = columnMasterKeyEnclaveComputationsParameter ; @@ -23415,14 +23424,14 @@ columnMasterKeyPathParameter returns [ColumnMasterKeyPathParameter vResult = Fra } ; - columnMasterKeyEnclaveComputationsParameter returns [ColumnMasterKeyEnclaveComputationsParameter vResult = FragmentFactory.CreateFragment()] + columnMasterKeyEnclaveComputationsParameter returns [ColumnMasterKeyEnclaveComputationsParameter vResult = FragmentFactory.CreateFragment()] { BinaryLiteral vSignature; } : tEnclaveComputations:Identifier tLeftParens:LeftParenthesis tSignature:Identifier tEquals3:EqualsSign vSignature=binary tRightParens:RightParenthesis { Match(tEnclaveComputations, CodeGenerationSupporter.EnclaveComputations); - Match(tSignature, CodeGenerationSupporter.Signature); + Match(tSignature, CodeGenerationSupporter.Signature); vResult.ParameterKind = ColumnMasterKeyParameterKind.Signature; vResult.Signature = vSignature; } @@ -23867,6 +23876,247 @@ dropSecurityPolicyStatement returns [DropSecurityPolicyStatement vResult = Fragm } ; +createExternalModelStatement returns [CreateExternalModelStatement vResult = FragmentFactory.CreateFragment()] +{ + Identifier vName; + long encounteredOptions = 0; +} + : tModel:Identifier vName = identifier + { + Match(tModel, CodeGenerationSupporter.Model); + vResult.Name = vName; + ThrowPartialAstIfPhaseOne(vResult); + } + authorizationOpt[vResult] + tWith:With LeftParenthesis + ( + {NextTokenMatches(CodeGenerationSupporter.Location)}? + externalModelLocation[vResult] + | + {NextTokenMatches(CodeGenerationSupporter.ApiFormat)}? + externalModelApiFormat[vResult] + | + {NextTokenMatches(CodeGenerationSupporter.ModelType)}? + externalModelModelType[vResult] + | + {NextTokenMatches(CodeGenerationSupporter.ModelName)}? + externalModelModelName[vResult] + | + {NextTokenMatches(CodeGenerationSupporter.Credential)}? + externalModelCredential[vResult] + | + {NextTokenMatches(CodeGenerationSupporter.LocalRuntimePath)}? + externalModelLocalRuntimePath[vResult] + | + {NextTokenMatches(CodeGenerationSupporter.Parameters)}? + externalModelParameters[vResult] + ) + + ( + tComma:Comma + ( + {NextTokenMatches(CodeGenerationSupporter.Location)}? + externalModelLocation[vResult] + | + {NextTokenMatches(CodeGenerationSupporter.ApiFormat)}? + externalModelApiFormat[vResult] + | + {NextTokenMatches(CodeGenerationSupporter.ModelType)}? + externalModelModelType[vResult] + | + {NextTokenMatches(CodeGenerationSupporter.ModelName)}? + externalModelModelName[vResult] + | + {NextTokenMatches(CodeGenerationSupporter.Credential)}? + externalModelCredential[vResult] + | + {NextTokenMatches(CodeGenerationSupporter.LocalRuntimePath)}? + externalModelLocalRuntimePath[vResult] + | + {NextTokenMatches(CodeGenerationSupporter.Parameters)}? + externalModelParameters[vResult] + ) + )* + + tRParen:RightParenthesis + { + UpdateTokenInfo(vResult,tRParen); + } + ; + +externalModelLocation[ExternalModelStatement vParent] +{ + StringLiteral vLocation; +} + : + tLocation:Identifier EqualsSign vLocation = stringLiteral + { + Match(tLocation, CodeGenerationSupporter.Location); + vParent.Location = vLocation; + } + ; + +externalModelApiFormat[ExternalModelStatement vParent] +{ +StringLiteral vApiFormat; +} +: + tApiFormat:Identifier EqualsSign vApiFormat = stringLiteral + { + Match(tApiFormat, CodeGenerationSupporter.ApiFormat); + vParent.ApiFormat = vApiFormat; + } +; + +externalModelModelType[ExternalModelStatement vParent] + : + tModelType:Identifier + { + Match(tModelType, CodeGenerationSupporter.ModelType); + UpdateTokenInfo(vParent, tModelType); + } + EqualsSign + ( + tEmbeddings:Identifier + { + if (TryMatch(tEmbeddings, CodeGenerationSupporter.Embeddings)) + { + vParent.ModelType = ExternalModelTypeOption.EMBEDDINGS; + UpdateTokenInfo(vParent, tEmbeddings); + } + else + { + ThrowIncorrectSyntaxErrorException(tEmbeddings); + } + } + ) + ; + +externalModelModelName[ExternalModelStatement vParent] +{ + StringLiteral vModelName; +} + : + tModelName:Identifier EqualsSign vModelName = stringLiteral + { + Match(tModelName, CodeGenerationSupporter.ModelName); + vParent.ModelName = vModelName; + } + ; + +externalModelCredential[ExternalModelStatement vParent] +{ +Identifier vCredential; +} +: + tCredential:Identifier EqualsSign vCredential = identifier + { + Match(tCredential, CodeGenerationSupporter.Credential); + vParent.Credential = vCredential; + } +; +externalModelLocalRuntimePath[ExternalModelStatement vParent] +{ +StringLiteral vLocalRuntimePath; +} + : + tLocalRuntimePath:Identifier EqualsSign vLocalRuntimePath = stringLiteral + { + Match(tLocalRuntimePath, CodeGenerationSupporter.LocalRuntimePath); + vParent.LocalRuntimePath = vLocalRuntimePath; + } + ; +externalModelParameters[ExternalModelStatement vParent] +{ + StringLiteral vParameters; +} + : + tParameters:Identifier EqualsSign vParameters = stringLiteral + { + Match(tParameters, CodeGenerationSupporter.Parameters); + vParent.Parameters = vParameters; + } + ; + +alterExternalModelStatement returns [AlterExternalModelStatement vResult = FragmentFactory.CreateFragment()] +{ + Identifier vName; + long encounteredOptions = 0; +} + : tModel:Identifier vName = identifier + { + Match(tModel, CodeGenerationSupporter.Model); + vResult.Name = vName; + ThrowPartialAstIfPhaseOne(vResult); + } + tSet:Set LeftParenthesis + ( + {NextTokenMatches(CodeGenerationSupporter.Location)}? + externalModelLocation[vResult] + | + {NextTokenMatches(CodeGenerationSupporter.ApiFormat)}? + externalModelApiFormat[vResult] + | + {NextTokenMatches(CodeGenerationSupporter.ModelType)}? + externalModelModelType[vResult] + | + {NextTokenMatches(CodeGenerationSupporter.ModelName)}? + externalModelModelName[vResult] + | + {NextTokenMatches(CodeGenerationSupporter.Credential)}? + externalModelCredential[vResult] + | + {NextTokenMatches(CodeGenerationSupporter.LocalRuntimePath)}? + externalModelLocalRuntimePath[vResult] + | + {NextTokenMatches(CodeGenerationSupporter.Parameters)}? + externalModelParameters[vResult] + ) + + ( + tComma:Comma + ( + {NextTokenMatches(CodeGenerationSupporter.Location)}? + externalModelLocation[vResult] + | + {NextTokenMatches(CodeGenerationSupporter.ApiFormat)}? + externalModelApiFormat[vResult] + | + {NextTokenMatches(CodeGenerationSupporter.ModelType)}? + externalModelModelType[vResult] + | + {NextTokenMatches(CodeGenerationSupporter.ModelName)}? + externalModelModelName[vResult] + | + {NextTokenMatches(CodeGenerationSupporter.Credential)}? + externalModelCredential[vResult] + | + {NextTokenMatches(CodeGenerationSupporter.LocalRuntimePath)}? + externalModelLocalRuntimePath[vResult] + | + {NextTokenMatches(CodeGenerationSupporter.Parameters)}? + externalModelParameters[vResult] + ) + )* + + tRParen:RightParenthesis + { + UpdateTokenInfo(vResult,tRParen); + } + ; + +dropExternalModelStatement returns [DropExternalModelStatement vResult = FragmentFactory.CreateFragment()] +{ + Identifier vName; +} + : tModel:Identifier vName = identifier + { + Match(tModel, CodeGenerationSupporter.Model); + vResult.Name = vName; + ThrowPartialAstIfPhaseOne(vResult); + } + ; + createExternalDataSourceStatement returns [CreateExternalDataSourceStatement vResult = FragmentFactory.CreateFragment()] { Identifier vName; @@ -23956,11 +24206,11 @@ externalDataSourceType[CreateExternalDataSourceStatement vParent] UpdateTokenInfo(vParent, tExternalDataSourceType); vParent.DataSourceType = ExternalDataSourceType.BLOB_STORAGE; } - else - { - UpdateTokenInfo(vParent, tExternalDataSourceType); + else + { + UpdateTokenInfo(vParent, tExternalDataSourceType); vParent.DataSourceType = ExternalDataSourceType.EXTERNAL_GENERICS; - } + } } ) ; @@ -24130,7 +24380,7 @@ createExternalStreamStatement returns [CreateExternalStreamStatement vResult = F {NextTokenMatches(CodeGenerationSupporter.Location)}? externalStreamLocation[vResult] | - {NextTokenMatches(CodeGenerationSupporter.InputOptions)}? + {NextTokenMatches(CodeGenerationSupporter.InputOptions)}? externalStreamInputOptions[vResult] | {NextTokenMatches(CodeGenerationSupporter.OutputOptions)}? @@ -29497,11 +29747,11 @@ generatedAlwaysClause [ColumnDefinition vResult] } else if (TryMatch(tGeneratedType, CodeGenerationSupporter.TransactionId)) { - vResult.GeneratedAlways = GeneratedAlwaysType.TransactionIdStart; + vResult.GeneratedAlways = GeneratedAlwaysType.TransactionIdStart; } else if (TryMatch(tGeneratedType, CodeGenerationSupporter.SequenceNumber)) - { - vResult.GeneratedAlways = GeneratedAlwaysType.SequenceNumberStart; + { + vResult.GeneratedAlways = GeneratedAlwaysType.SequenceNumberStart; } else { @@ -29965,7 +30215,7 @@ uniqueTableConstraint [IndexAffectingStatement statementType] returns [UniqueCon ( uniqueConstraintEnforcement[vResult] | - uniqueConstraintTailOpt[statementType, vResult] + uniqueConstraintTailOpt[statementType, vResult] ) ; @@ -32159,7 +32409,7 @@ expressionList [TSqlFragment vParent, IList expressions] Identifier vAbsent; } : - ( + ( Null On Null { vNull = FragmentFactory.CreateFragment(); @@ -32199,18 +32449,18 @@ jsonKeyValueExpression returns [JsonKeyValue vResult = FragmentFactory.CreateFra ScalarExpression vKey; ScalarExpression vValue; } - : + : ( vKey=expression - { - vResult.JsonKeyName=vKey; - } + { + vResult.JsonKeyName=vKey; + } Colon vValue=expression - { - vResult.JsonValue=vValue; + { + vResult.JsonValue=vValue; } - ) - ; + ) + ; windowClause returns [WindowClause vResult = FragmentFactory.CreateFragment()] { @@ -32629,19 +32879,19 @@ ignoreRespectNulls [FunctionCall vParent] Identifier vNulls; } : - tIgnoreOrRespect:Identifier + tIgnoreOrRespect:Identifier { - Match(tIgnoreOrRespect, CodeGenerationSupporter.Ignore, CodeGenerationSupporter.Respect); + Match(tIgnoreOrRespect, CodeGenerationSupporter.Ignore, CodeGenerationSupporter.Respect); vIgnoreOrRespect = FragmentFactory.CreateFragment(); - AddAndUpdateTokenInfo(vParent, vParent.IgnoreRespectNulls, vIgnoreOrRespect); - vIgnoreOrRespect.SetUnquotedIdentifier(tIgnoreOrRespect.getText()); + AddAndUpdateTokenInfo(vParent, vParent.IgnoreRespectNulls, vIgnoreOrRespect); + vIgnoreOrRespect.SetUnquotedIdentifier(tIgnoreOrRespect.getText()); } tNulls:Identifier { - Match(tNulls, CodeGenerationSupporter.Nulls); + Match(tNulls, CodeGenerationSupporter.Nulls); vNulls = FragmentFactory.CreateFragment(); - AddAndUpdateTokenInfo(vParent, vParent.IgnoreRespectNulls, vNulls); - vNulls.SetUnquotedIdentifier(tNulls.getText()); + AddAndUpdateTokenInfo(vParent, vParent.IgnoreRespectNulls, vNulls); + vNulls.SetUnquotedIdentifier(tNulls.getText()); } ; @@ -33707,9 +33957,9 @@ nonEmptyString returns [StringLiteral vResult] } ; - defaultValueLiteral returns [ScalarExpression vResult] - : vResult = literal - | vResult = signedIntegerOrReal; + defaultValueLiteral returns [ScalarExpression vResult] + : vResult = literal + | vResult = signedIntegerOrReal; stringLiteral returns [StringLiteral vResult = this.FragmentFactory.CreateFragment()] : tAsciiStringLiteral:AsciiStringLiteral diff --git a/SqlScriptDom/ScriptDom/SqlServer/ScriptGenerator/SqlScriptGenerator.AlterExternalModelStatement.cs b/SqlScriptDom/ScriptDom/SqlServer/ScriptGenerator/SqlScriptGenerator.AlterExternalModelStatement.cs new file mode 100644 index 0000000..3129899 --- /dev/null +++ b/SqlScriptDom/ScriptDom/SqlServer/ScriptGenerator/SqlScriptGenerator.AlterExternalModelStatement.cs @@ -0,0 +1,121 @@ +//------------------------------------------------------------------------------ +// +// Copyright (c) Microsoft Corporation. All rights reserved. +// +//------------------------------------------------------------------------------ + +using System.Globalization; +using System.Collections.Generic; +using Microsoft.SqlServer.TransactSql.ScriptDom; +namespace Microsoft.SqlServer.TransactSql.ScriptDom.ScriptGenerator +{ + partial class SqlScriptGeneratorVisitor + { + public override void ExplicitVisit(AlterExternalModelStatement node) + { + GenerateKeyword(TSqlTokenType.Alter); + GenerateSpaceAndIdentifier(CodeGenerationSupporter.External); + GenerateSpaceAndIdentifier(CodeGenerationSupporter.Model); + GenerateAlterExternalModelStatementBody(node); + } + + protected void GenerateAlterExternalModelStatementBody(AlterExternalModelStatement node) + { + // external model name + GenerateSpaceAndFragmentIfNotNull(node.Name); + + NewLine(); + GenerateKeywordAndSpace(TSqlTokenType.Set); + GenerateKeyword(TSqlTokenType.LeftParenthesis); + bool ifFirst = true; + // external model location + if (node.Location != null) + { + if (!ifFirst) + { + GenerateSymbol(TSqlTokenType.Comma); + } + ifFirst = false; + NewLine(); + GenerateNameEqualsValue(CodeGenerationSupporter.Location, node.Location); + } + + // external model API Format options + if (node.ApiFormat != null) + { + if (!ifFirst) + { + GenerateSymbol(TSqlTokenType.Comma); + } + ifFirst = false; + NewLine(); + GenerateNameEqualsValue(CodeGenerationSupporter.ApiFormat, node.ApiFormat); + } + + // external model Model Type options + if (node.ModelType == ExternalModelTypeOption.EMBEDDINGS) + { + if (!ifFirst) + { + GenerateSymbol(TSqlTokenType.Comma); + } + ifFirst = false; + ExternalModelTypeOption typeOption = ExternalModelTypeOption.EMBEDDINGS; + string externalModelTypeOption = GetValueForEnumKey(_externalModelTypeOption, typeOption); + NewLine(); + GenerateNameEqualsValue(CodeGenerationSupporter.ModelType, externalModelTypeOption); + } + + // external model name options + if (node.ModelName != null) + { + if (!ifFirst) + { + GenerateSymbol(TSqlTokenType.Comma); + } + ifFirst = false; + NewLine(); + GenerateNameEqualsValue(CodeGenerationSupporter.ModelName, node.ModelName); + } + + // external model credential options + if (node.Credential != null) + { + if (!ifFirst) + { + GenerateSymbol(TSqlTokenType.Comma); + } + ifFirst = false; + NewLine(); + GenerateNameEqualsValue(CodeGenerationSupporter.Credential, node.Credential); + } + + // external model parameters options + if (node.Parameters != null) + { + if (!ifFirst) + { + GenerateSymbol(TSqlTokenType.Comma); + } + ifFirst = false; + NewLine(); + GenerateNameEqualsValue(CodeGenerationSupporter.Parameters, node.Parameters); + } + + // external model local runtime path options + if (node.LocalRuntimePath != null) + { + if (!ifFirst) + { + GenerateSymbol(TSqlTokenType.Comma); + } + ifFirst = false; + NewLine(); + GenerateNameEqualsValue(CodeGenerationSupporter.LocalRuntimePath, node.LocalRuntimePath); + } + + NewLine(); + GenerateKeyword(TSqlTokenType.RightParenthesis); + } + } +} diff --git a/SqlScriptDom/ScriptDom/SqlServer/ScriptGenerator/SqlScriptGenerator.DropExternalModelStatement.cs b/SqlScriptDom/ScriptDom/SqlServer/ScriptGenerator/SqlScriptGenerator.DropExternalModelStatement.cs new file mode 100644 index 0000000..707fe8f --- /dev/null +++ b/SqlScriptDom/ScriptDom/SqlServer/ScriptGenerator/SqlScriptGenerator.DropExternalModelStatement.cs @@ -0,0 +1,23 @@ +//------------------------------------------------------------------------------ +// +// Copyright (c) Microsoft Corporation. All rights reserved. +// +//------------------------------------------------------------------------------ + +namespace Microsoft.SqlServer.TransactSql.ScriptDom.ScriptGenerator +{ + partial class SqlScriptGeneratorVisitor + { + // Generates the statement + // DROP EXTERNAL MODEL [{node.Name}] + // + public override void ExplicitVisit(DropExternalModelStatement node) + { + GenerateKeyword(TSqlTokenType.Drop); + GenerateSpaceAndIdentifier(CodeGenerationSupporter.External); + GenerateSpaceAndIdentifier(CodeGenerationSupporter.Model); + + GenerateSpaceAndFragmentIfNotNull(node.Name); + } + } +} diff --git a/SqlScriptDom/ScriptDom/SqlServer/ScriptGenerator/SqlScriptGeneratorVisitor.CreateExternalModelStatement.cs b/SqlScriptDom/ScriptDom/SqlServer/ScriptGenerator/SqlScriptGeneratorVisitor.CreateExternalModelStatement.cs new file mode 100644 index 0000000..8e01b6b --- /dev/null +++ b/SqlScriptDom/ScriptDom/SqlServer/ScriptGenerator/SqlScriptGeneratorVisitor.CreateExternalModelStatement.cs @@ -0,0 +1,122 @@ +//------------------------------------------------------------------------------ +// +// Copyright (c) Microsoft Corporation. All rights reserved. +// +//------------------------------------------------------------------------------ +using System.Collections.Generic; +using System.Globalization; +using Microsoft.SqlServer.TransactSql.ScriptDom; +namespace Microsoft.SqlServer.TransactSql.ScriptDom.ScriptGenerator +{ + partial class SqlScriptGeneratorVisitor + { + public override void ExplicitVisit(CreateExternalModelStatement node) + { + GenerateKeyword(TSqlTokenType.Create); + GenerateSpaceAndIdentifier(CodeGenerationSupporter.External); + GenerateSpaceAndIdentifier(CodeGenerationSupporter.Model); + GenerateCreateExternalModelStatementBody(node); + } + protected static Dictionary _externalModelTypeOption = new Dictionary() + { + {ExternalModelTypeOption.EMBEDDINGS, CodeGenerationSupporter.Embeddings} + }; + + protected void GenerateCreateExternalModelStatementBody(CreateExternalModelStatement node) + { + // external model name + GenerateSpaceAndFragmentIfNotNull(node.Name); + GenerateOwnerIfNotNull(node.Owner); + + NewLine(); + GenerateKeywordAndSpace(TSqlTokenType.With); + GenerateKeyword(TSqlTokenType.LeftParenthesis); + bool ifFirst = true; + + // external model location + if (node.Location != null) + { + NewLine(); + GenerateNameEqualsValue(CodeGenerationSupporter.Location, node.Location); + ifFirst = false; + } + + // external model API Format options + if (node.ApiFormat != null) + { + if (!ifFirst) + { + GenerateSymbol(TSqlTokenType.Comma); + } + ifFirst = false; + NewLine(); + GenerateNameEqualsValue(CodeGenerationSupporter.ApiFormat, node.ApiFormat); + } + + // external model Model Type options + if (node.ModelType == ExternalModelTypeOption.EMBEDDINGS) + { + if (!ifFirst) + { + GenerateSymbol(TSqlTokenType.Comma); + } + ifFirst = false; + ExternalModelTypeOption typeOption = ExternalModelTypeOption.EMBEDDINGS; + string externalModelTypeOption = GetValueForEnumKey(_externalModelTypeOption, typeOption); + NewLine(); + GenerateNameEqualsValue(CodeGenerationSupporter.ModelType, externalModelTypeOption); + } + + // external model name options + if (node.ModelName != null) + { + if (!ifFirst) + { + GenerateSymbol(TSqlTokenType.Comma); + } + ifFirst = false; + NewLine(); + GenerateNameEqualsValue(CodeGenerationSupporter.ModelName, node.ModelName); + } + + // external model credential options + if (node.Credential != null) + { + if (!ifFirst) + { + GenerateSymbol(TSqlTokenType.Comma); + } + ifFirst = false; + NewLine(); + GenerateNameEqualsValue(CodeGenerationSupporter.Credential, node.Credential); + } + + // external model parameters options + if (node.Parameters != null) + { + if (!ifFirst) + { + GenerateSymbol(TSqlTokenType.Comma); + } + ifFirst = false; + NewLine(); + GenerateNameEqualsValue(CodeGenerationSupporter.Parameters, node.Parameters); + } + + // external model local runtime path options + if (node.LocalRuntimePath != null) + { + if (!ifFirst) + { + GenerateSymbol(TSqlTokenType.Comma); + } + ifFirst = false; + NewLine(); + GenerateNameEqualsValue(CodeGenerationSupporter.LocalRuntimePath, node.LocalRuntimePath); + } + + NewLine(); + GenerateKeyword(TSqlTokenType.RightParenthesis); + } + } +} diff --git a/Test/SqlDom/Baselines170/AlterExternalModelStatementTests170.sql b/Test/SqlDom/Baselines170/AlterExternalModelStatementTests170.sql new file mode 100644 index 0000000..e9b590c --- /dev/null +++ b/Test/SqlDom/Baselines170/AlterExternalModelStatementTests170.sql @@ -0,0 +1,27 @@ +ALTER EXTERNAL MODEL abc +SET ( +LOCATION = 'new_location', +API_FORMAT = 'Ollama', +MODEL_TYPE = EMBEDDINGS, +MODEL = 'new_model', +PARAMETERS = '{"key":"new_value"}' +); +ALTER EXTERNAL MODEL abc +SET ( +MODEL = 'new_model' +); +ALTER EXTERNAL MODEL params_model +SET ( +PARAMETERS = '{"temperature":0.7, "max_tokens":2048}' +); +ALTER EXTERNAL MODEL ml_model +SET ( +API_FORMAT = 'OpenAI', +MODEL = 'gpt-3.5-turbo-16k', +PARAMETERS = '{"stream":true}' +); +ALTER EXTERNAL MODEL location_change +SET ( +LOCATION = '/new/model/path', +MODEL_TYPE = EMBEDDINGS +); \ No newline at end of file diff --git a/Test/SqlDom/Baselines170/CreateExternalModelStatementTests170.sql b/Test/SqlDom/Baselines170/CreateExternalModelStatementTests170.sql new file mode 100644 index 0000000..8891476 --- /dev/null +++ b/Test/SqlDom/Baselines170/CreateExternalModelStatementTests170.sql @@ -0,0 +1,33 @@ +CREATE EXTERNAL MODEL abc + AUTHORIZATION dbo +WITH ( +LOCATION = 'sdfasfd', +API_FORMAT = 'Ollama', +MODEL_TYPE = EMBEDDINGS, +MODEL = 'shghfh', +PARAMETERS = '{"key":"valuoipe"}' +); +CREATE EXTERNAL MODEL simple_model +WITH ( +LOCATION = '/models/simple', +API_FORMAT = 'OpenAI', +MODEL_TYPE = EMBEDDINGS, +MODEL = 'gpt-3.5-turbo' +); +CREATE EXTERNAL MODEL advanced_model +WITH ( +LOCATION = 'https://models.example.com/advanced', +API_FORMAT = 'Azure OpenAI', +MODEL_TYPE = EMBEDDINGS, +MODEL = 'bert-base-uncased', +PARAMETERS = '{"batch_size":32,"device":"cuda"}' +); +CREATE EXTERNAL MODEL [model_name] +WITH ( +LOCATION = 'FILE PATH', +API_FORMAT = 'onnx runtime', +MODEL_TYPE = EMBEDDINGS, +MODEL = 'text-embedding-ada-002,etc', +PARAMETERS = '{ "valid":"JSON"}', +LOCAL_RUNTIME_PATH = 'Path on local server to onnx runtime' +); \ No newline at end of file diff --git a/Test/SqlDom/Baselines170/DropExternalModelStatementTests170.sql b/Test/SqlDom/Baselines170/DropExternalModelStatementTests170.sql new file mode 100644 index 0000000..127b413 --- /dev/null +++ b/Test/SqlDom/Baselines170/DropExternalModelStatementTests170.sql @@ -0,0 +1 @@ +DROP EXTERNAL MODEL my_model1; \ No newline at end of file diff --git a/Test/SqlDom/Only170SyntaxTests.cs b/Test/SqlDom/Only170SyntaxTests.cs index 1dc8fc4..686e953 100644 --- a/Test/SqlDom/Only170SyntaxTests.cs +++ b/Test/SqlDom/Only170SyntaxTests.cs @@ -17,7 +17,10 @@ public partial class SqlDomTests new ParserTest170("RegexpTests170.sql", nErrors80: 0, nErrors90: 0, nErrors100: 0, nErrors110: 0, nErrors120: 0, nErrors130: 0, nErrors140: 0, nErrors150: 0, nErrors160: 0), new ParserTest170("AiGenerateChunksTests170.sql", nErrors80: 21, nErrors90: 18, nErrors100: 17, nErrors110: 17, nErrors120: 17, nErrors130: 17, nErrors140: 17, nErrors150: 17, nErrors160: 17), new ParserTest170("JsonFunctionTests170.sql", nErrors80: 9, nErrors90: 8, nErrors100: 30, nErrors110: 30, nErrors120: 30, nErrors130: 30, nErrors140: 30, nErrors150: 30, nErrors160: 30), - new ParserTest170("AiGenerateEmbeddingsTests170.sql", nErrors80: 12, nErrors90: 9, nErrors100: 9, nErrors110: 9, nErrors120: 9, nErrors130: 9, nErrors140: 9, nErrors150: 9, nErrors160: 9) + new ParserTest170("AiGenerateEmbeddingsTests170.sql", nErrors80: 12, nErrors90: 9, nErrors100: 9, nErrors110: 9, nErrors120: 9, nErrors130: 9, nErrors140: 9, nErrors150: 9, nErrors160: 9), + new ParserTest170("CreateExternalModelStatementTests170.sql", nErrors80: 2, nErrors90: 2, nErrors100: 2, nErrors110: 2, nErrors120: 2, nErrors130: 4, nErrors140: 4, nErrors150: 4, nErrors160: 4), + new ParserTest170("AlterExternalModelStatementTests170.sql", nErrors80: 2, nErrors90: 2, nErrors100: 2, nErrors110: 2, nErrors120: 2, nErrors130: 5, nErrors140: 5, nErrors150: 5, nErrors160: 5), + new ParserTest170("DropExternalModelStatementTests170.sql", nErrors80: 1, nErrors90: 1, nErrors100: 1, nErrors110: 1, nErrors120: 1, nErrors130: 1, nErrors140: 1, nErrors150: 1, nErrors160: 1), }; private static readonly ParserTest[] SqlAzure170_TestInfos = diff --git a/Test/SqlDom/ParserErrorsTests.cs b/Test/SqlDom/ParserErrorsTests.cs index 05fcc97..35bd265 100644 --- a/Test/SqlDom/ParserErrorsTests.cs +++ b/Test/SqlDom/ParserErrorsTests.cs @@ -7191,7 +7191,6 @@ public void GenerateChunksNegativeTest170() new ParserErrorInfo(69, "SQL46010", "other")); } - /// /// Negative tests for AI_GENERATE_EMBEDDINGS syntax /// @@ -7255,5 +7254,108 @@ public void GenerateEmbeddingsNegativeTest170() "SELECT AI_GENERATE_EMBEDDINGS('My Default Input Text' USE MODEL NULL)", new ParserErrorInfo(64, "SQL46010", "NULL")); } + + + [TestMethod] + [Priority(0)] + [SqlStudioTestCategory(Category.UnitTest)] + public void CreateExternalModelNegativeTest() + { + // Keyword typos + // + ParserTestUtils.ErrorTest170( + "CREATE EXTERNAL MODLE abc WITH (LOCATION = 'www.somemodellocation.235',API_FORMAT = 'Ollama',MODEL_TYPE = EMBEDDIGS,MODEL = 'shghfh',PARAMETERS = '{\"key\":\"value\"}');", + new ParserErrorInfo(16, "SQL46010", "MODLE")); + + // Invalid Model type + // + ParserTestUtils.ErrorTest170( + "CREATE EXTERNAL MODEL abc WITH (LOCATION = 'www.somemodellocation.235',API_FORMAT = 'Ollama',MODEL_TYPE = EMBEDDIGS,MODEL = 'shghfh',PARAMETERS = '{\"key\":\"value\"}');", + new ParserErrorInfo(106, "SQL46010", "EMBEDDIGS")); + + // Model type not identifier + // + ParserTestUtils.ErrorTest170( + "CREATE EXTERNAL MODEL abc WITH (LOCATION = 'www.somemodellocation.235',API_FORMAT = 'Ollama',MODEL_TYPE = 'EMBEDDINGS',MODEL = 'shghfh',PARAMETERS = '{\"key\":\"value\"}');", + new ParserErrorInfo(106, "SQL46010", "'EMBEDDINGS'")); + + // Missing With Clause + // + ParserTestUtils.ErrorTest170( + "CREATE EXTERNAL MODEL abc (LOCATION = 'www.somemodellocation.235',API_FORMAT = 'Ollama',MODEL_TYPE = EMBEDDINGS,MODEL = 'shghfh',PARAMETERS = '{\"key\":\"value\"}');", + new ParserErrorInfo(26, "SQL46010", "(")); + + // Missing Model Name + // + ParserTestUtils.ErrorTest170( + "CREATE EXTERNAL MODEL WITH (LOCATION = 'www.somemodellocation.235',API_FORMAT = 'Ollama',MODEL_TYPE = EMBEDDINGS,MODEL = 'shghfh',PARAMETERS = '{\"key\":\"value\"}');", + new ParserErrorInfo(16, "SQL46010", "MODEL")); + } + + [TestMethod] + [Priority(0)] + [SqlStudioTestCategory(Category.UnitTest)] + public void AlterExternalModelNegativeTest() + { + // Keyword typos + // + ParserTestUtils.ErrorTest170( + "ALTER EXTERNAL MODLE abc SET (LOCATION = 'www.somemodellocation.235',API_FORMAT = 'Ollama',MODEL_TYPE = EMBEDDINGS,MODEL = 'shghfh',PARAMETERS = '{\"key\":\"value\"}');", + new ParserErrorInfo(15, "SQL46010", "MODLE")); + + // Invalid Model type + // + ParserTestUtils.ErrorTest170( + "ALTER EXTERNAL MODEL abc SET (MODEL_TYPE = EMBEDDINS);", + new ParserErrorInfo(43, "SQL46010", "EMBEDDINS")); + + // Model type not identifier + // + ParserTestUtils.ErrorTest170( + "ALTER EXTERNAL MODEL abc SET (LOCATION = 'www.somemodellocation.235',API_FORMAT = 'Ollama',MODEL_TYPE = 'EMBEDDINGS',MODEL = 'shghfh',PARAMETERS = '{\"key\":\"value\"}');", + new ParserErrorInfo(104, "SQL46010", "'EMBEDDINGS'")); + + // Missing SET Clause + // + ParserTestUtils.ErrorTest170( + "ALTER EXTERNAL MODEL abc (MODEL = 'shghfh');", + new ParserErrorInfo(25, "SQL46010", "(")); + + // WITH instead of SET clause + // + ParserTestUtils.ErrorTest170( + "ALTER EXTERNAL MODEL abc WITH (LOCATION = 'www.somemodellocation.235',API_FORMAT = 'Ollama',MODEL_TYPE = EMBEDDINGS,MODEL = 'shghfh',PARAMETERS = '{\"key\":\"value\"}');", + new ParserErrorInfo(25, "SQL46010", "WITH")); + + // Missing Model Name + // + ParserTestUtils.ErrorTest170( + "ALTER EXTERNAL MODEL SET (LOCATION = 'www.somemodellocation.235',API_FORMAT = 'Ollama',MODEL_TYPE = EMBEDDINGS,MODEL = 'shghfh',PARAMETERS = '{\"key\":\"value\"}');", + new ParserErrorInfo(15, "SQL46010", "MODEL")); + } + + [TestMethod] + [Priority(0)] + [SqlStudioTestCategory(Category.UnitTest)] + public void DropExternalModelNegativeTest() + { + // Keyword typos + // + ParserTestUtils.ErrorTest170( + "DROP EXTERNAL MODLE abc;", + new ParserErrorInfo(14, "SQL46010", "MODLE")); + + // Missing Model Name + // + ParserTestUtils.ErrorTest170( + "DROP EXTERNAL MODEL;", + new ParserErrorInfo(14, "SQL46010", "MODEL")); + + // Extra keyword + // + ParserTestUtils.ErrorTest170( + "DROP EXTERNAL MODEL abc WITH (LOCATION = 'www.somemodellocation.235');", + new ParserErrorInfo(29, "SQL46010", "(")); + } } } diff --git a/Test/SqlDom/TestScripts/AlterExternalModelStatementTests170.sql b/Test/SqlDom/TestScripts/AlterExternalModelStatementTests170.sql new file mode 100644 index 0000000..e9b590c --- /dev/null +++ b/Test/SqlDom/TestScripts/AlterExternalModelStatementTests170.sql @@ -0,0 +1,27 @@ +ALTER EXTERNAL MODEL abc +SET ( +LOCATION = 'new_location', +API_FORMAT = 'Ollama', +MODEL_TYPE = EMBEDDINGS, +MODEL = 'new_model', +PARAMETERS = '{"key":"new_value"}' +); +ALTER EXTERNAL MODEL abc +SET ( +MODEL = 'new_model' +); +ALTER EXTERNAL MODEL params_model +SET ( +PARAMETERS = '{"temperature":0.7, "max_tokens":2048}' +); +ALTER EXTERNAL MODEL ml_model +SET ( +API_FORMAT = 'OpenAI', +MODEL = 'gpt-3.5-turbo-16k', +PARAMETERS = '{"stream":true}' +); +ALTER EXTERNAL MODEL location_change +SET ( +LOCATION = '/new/model/path', +MODEL_TYPE = EMBEDDINGS +); \ No newline at end of file diff --git a/Test/SqlDom/TestScripts/CreateExternalModelStatementTests170.sql b/Test/SqlDom/TestScripts/CreateExternalModelStatementTests170.sql new file mode 100644 index 0000000..8891476 --- /dev/null +++ b/Test/SqlDom/TestScripts/CreateExternalModelStatementTests170.sql @@ -0,0 +1,33 @@ +CREATE EXTERNAL MODEL abc + AUTHORIZATION dbo +WITH ( +LOCATION = 'sdfasfd', +API_FORMAT = 'Ollama', +MODEL_TYPE = EMBEDDINGS, +MODEL = 'shghfh', +PARAMETERS = '{"key":"valuoipe"}' +); +CREATE EXTERNAL MODEL simple_model +WITH ( +LOCATION = '/models/simple', +API_FORMAT = 'OpenAI', +MODEL_TYPE = EMBEDDINGS, +MODEL = 'gpt-3.5-turbo' +); +CREATE EXTERNAL MODEL advanced_model +WITH ( +LOCATION = 'https://models.example.com/advanced', +API_FORMAT = 'Azure OpenAI', +MODEL_TYPE = EMBEDDINGS, +MODEL = 'bert-base-uncased', +PARAMETERS = '{"batch_size":32,"device":"cuda"}' +); +CREATE EXTERNAL MODEL [model_name] +WITH ( +LOCATION = 'FILE PATH', +API_FORMAT = 'onnx runtime', +MODEL_TYPE = EMBEDDINGS, +MODEL = 'text-embedding-ada-002,etc', +PARAMETERS = '{ "valid":"JSON"}', +LOCAL_RUNTIME_PATH = 'Path on local server to onnx runtime' +); \ No newline at end of file diff --git a/Test/SqlDom/TestScripts/DropExternalModelStatementTests170.sql b/Test/SqlDom/TestScripts/DropExternalModelStatementTests170.sql new file mode 100644 index 0000000..127b413 --- /dev/null +++ b/Test/SqlDom/TestScripts/DropExternalModelStatementTests170.sql @@ -0,0 +1 @@ +DROP EXTERNAL MODEL my_model1; \ No newline at end of file From bc6b46cbee82ba8509fd66962003334caa873878 Mon Sep 17 00:00:00 2001 From: Zi Chen Date: Mon, 4 Aug 2025 20:50:42 +0000 Subject: [PATCH 07/12] Merged PR 1758352: Add ScriptDom support for VECTOR_SEARCH function # Pull Request Template for ScriptDom ## Description Added parser and script generator support for VECTOR_SEARCH Before submitting your pull request, please ensure you have completed the following: ## Code Change - [ ] The [Common checklist](https://msdata.visualstudio.com/SQLToolsAndLibraries/_git/Common?path=/Templates/PR%20Checklist%20for%20SQLToolsAndLibraries.md&version=GBmain&_a=preview) has been reviewed and followed - [ ] Code changes are accompanied by appropriate unit tests - [ ] Identified and included SMEs needed to review code changes - [ ] Follow the [steps](https://msdata.visualstudio.com/SQLToolsAndLibraries/_wiki/wikis/SQLToolsAndLibraries.wiki/33838/Adding-or-Extending-TSql-support-in-Sql-Dom?anchor=make-the-changes-in) here to make changes in the code ## Testing - [ ] Follow the [steps](https://msdata.visualstudio.com/SQLToolsAndLibraries/_wiki/wikis/SQLToolsAndLibraries.wiki/33838/Adding-or-Extending-TSql-support-in-Sql-Dom?anchor=to-extend-the-tests-do-the-following%3A) here to add new tests for your feature ## Documentation - [ ] Update relevant documentation in the [wiki](https://dev.azure.com/msdata/SQLToolsAndLibraries/_wiki/wikis/SQLToolsAndLibraries.wiki) and the README.md file ## Additional Information Please provide any additional information that might be helpful for the reviewers Add ScriptDom support for VECTOR_SEARCH function ---- #### AI description (iteration 1) #### PR Classification This pull request introduces a new feature by adding native ScriptDom support for the VECTOR_SEARCH function. #### PR Summary The changes implement parsing, AST definition, and script generation for VECTOR_SEARCH, along with a comprehensive suite of tests to validate error handling and syntax compliance. - **`SqlScriptDom/ScriptDom/SqlServer/ScriptGenerator/SqlScriptGeneratorVisitor.VectorSearchTableReference.cs`**: New file added to generate script output for the VECTOR_SEARCH function. - **`SqlScriptDom/Parser/TSql/TSql170.g` & `SqlDom/Parser/TSql/Ast.xml`**: Extended grammar rules and AST definitions to support VECTOR_SEARCH. - **`SqlScriptDom/Parser/TSql/CodeGenerationSupporter.cs`**: Added new constants (e.g., COSINE, DOT, EUCLIDEAN, METRIC, TOP_N, VECTOR_SEARCH) required for vector search processing. - **`Test/SqlDom/ParserErrorsTests.cs` and related vector test scripts**: Introduced extensive tests covering syntax errors and parameter validations for VECTOR_SEARCH. Related work items: #4440435 --- SqlScriptDom/Parser/TSql/Ast.xml | 8 ++ .../Parser/TSql/CodeGenerationSupporter.cs | 7 + SqlScriptDom/Parser/TSql/TSql170.g | 43 ++++++ ...ratorVisitor.VectorSearchTableReference.cs | 56 ++++++++ .../Baselines170/VectorFunctionTests170.sql | 23 ++++ Test/SqlDom/Only170SyntaxTests.cs | 1 + Test/SqlDom/ParserErrorsTests.cs | 126 ++++++++++++++++++ .../TestScripts/VectorFunctionTests170.sql | 25 ++++ 8 files changed, 289 insertions(+) create mode 100644 SqlScriptDom/ScriptDom/SqlServer/ScriptGenerator/SqlScriptGeneratorVisitor.VectorSearchTableReference.cs create mode 100644 Test/SqlDom/Baselines170/VectorFunctionTests170.sql create mode 100644 Test/SqlDom/TestScripts/VectorFunctionTests170.sql diff --git a/SqlScriptDom/Parser/TSql/Ast.xml b/SqlScriptDom/Parser/TSql/Ast.xml index 942ac65..842a787 100644 --- a/SqlScriptDom/Parser/TSql/Ast.xml +++ b/SqlScriptDom/Parser/TSql/Ast.xml @@ -4752,4 +4752,12 @@ + + + + + + + + diff --git a/SqlScriptDom/Parser/TSql/CodeGenerationSupporter.cs b/SqlScriptDom/Parser/TSql/CodeGenerationSupporter.cs index 2e6edfb..ba75024 100644 --- a/SqlScriptDom/Parser/TSql/CodeGenerationSupporter.cs +++ b/SqlScriptDom/Parser/TSql/CodeGenerationSupporter.cs @@ -249,6 +249,7 @@ internal static class CodeGenerationSupporter internal const string CopyCommand = "COPY"; internal const string CopyOnly = "COPY_ONLY"; internal const string Correlated = "CORRELATED"; + internal const string Cosine = "COSINE"; internal const string Count = "COUNT"; internal const string CountBig = "COUNT_BIG"; internal const string Counter = "COUNTER"; @@ -325,6 +326,7 @@ internal static class CodeGenerationSupporter internal const string Document = "DOCUMENT"; internal const string DollarSign = "$"; internal const string DollarPartition = "$PARTITION"; + internal const string Dot = "DOT"; internal const string Drop = "DROP"; internal const string DropExisting = "DROP_EXISTING"; internal const string DTSBuffers = "DTS_BUFFERS"; @@ -364,6 +366,7 @@ internal static class CodeGenerationSupporter internal const string ErrorFileCredential = "ERRORFILE_CREDENTIAL"; internal const string EscapeChar = "ESCAPECHAR"; internal const string EstimateOnly = "ESTIMATEONLY"; + internal const string Euclidean = "EUCLIDEAN"; internal const string Event = "EVENT"; internal const string EventRetentionMode = "EVENT_RETENTION_MODE"; internal const string Exclamation = "!"; @@ -625,6 +628,7 @@ internal static class CodeGenerationSupporter internal const string Message = "MESSAGE"; internal const string MessageForwarding = "MESSAGE_FORWARDING"; internal const string MessageForwardSize = "MESSAGE_FORWARD_SIZE"; + internal const string Metric = "METRIC"; internal const string MigrationState = "MIGRATION_STATE"; internal const string Metric = "METRIC"; internal const string Min = "MIN"; @@ -930,6 +934,7 @@ internal static class CodeGenerationSupporter internal const string ShrinkDb = "SHRINKDB"; internal const string Sid = "SID"; internal const string Signature = "SIGNATURE"; + internal const string SimilarTo = "SIMILAR_TO"; internal const string Simple = "SIMPLE"; internal const string SingleBlob = "SINGLE_BLOB"; internal const string SingleClob = "SINGLE_CLOB"; @@ -1039,6 +1044,7 @@ internal static class CodeGenerationSupporter internal const string Timer = "TIMER"; internal const string TimeStamp = "TIMESTAMP"; internal const string TinyInt = "TINYINT"; + internal const string TopN = "TOP_N"; internal const string TornPageDetection = "TORN_PAGE_DETECTION"; internal const string TrackCausality = "TRACK_CAUSALITY"; internal const string TrackColumnsUpdated = "TRACK_COLUMNS_UPDATED"; @@ -1091,6 +1097,7 @@ internal static class CodeGenerationSupporter internal const string Varp = "VARP"; internal const string VDevNo = "VDEVNO"; internal const string Vector = "Vector"; + internal const string VectorSearch = "VECTOR_SEARCH"; internal const string Verbose = "VERBOSE"; internal const string VerboseLogging = "VerboseLogging"; internal const string VerifyOnly = "VERIFYONLY"; diff --git a/SqlScriptDom/Parser/TSql/TSql170.g b/SqlScriptDom/Parser/TSql/TSql170.g index 1276434..4baae9d 100644 --- a/SqlScriptDom/Parser/TSql/TSql170.g +++ b/SqlScriptDom/Parser/TSql/TSql170.g @@ -19148,6 +19148,8 @@ selectTableReferenceElementWithoutJoinParenthesis[SubDmlFlags subDmlFlags] retur | vResult=subDmlTableReference[subDmlFlags] | {NextTokenMatches(CodeGenerationSupporter.Predict)}? vResult=predictTableReference[subDmlFlags] + | {NextTokenMatches(CodeGenerationSupporter.VectorSearch)}? + vResult=vectorSearchTableReference | vResult=schemaObjectOrFunctionTableReference ; @@ -19243,6 +19245,47 @@ aiGenerateFixedChunksTableReference [ScalarExpression vSource, Identifier vChunk )? ; +vectorSearchTableReference returns [VectorSearchTableReference vResult = FragmentFactory.CreateFragment()] +{ + TableReferenceWithAlias vTable; + ColumnReferenceExpression vColumn; + ScalarExpression vSimilarTo; + StringLiteral vMetric; + IntegerLiteral vTopN; +} + : + tVectorSearch:Identifier LeftParenthesis + { + Match(tVectorSearch, CodeGenerationSupporter.VectorSearch); + UpdateTokenInfo(vResult, tVectorSearch); + } + Table EqualsSign vTable = mergeTarget[false] + { + vResult.Table = vTable; + } + Comma Column EqualsSign vColumn = fixedColumn + { + vResult.Column = vColumn; + } + Comma tSimilarTo:Identifier EqualsSign vSimilarTo = expression + { + Match(tSimilarTo, CodeGenerationSupporter.SimilarTo); + vResult.SimilarTo = vSimilarTo; + } + Comma tMetric:Identifier EqualsSign vMetric = stringLiteral + { + Match(tMetric, CodeGenerationSupporter.Metric); + MatchString(vMetric, CodeGenerationSupporter.Cosine, CodeGenerationSupporter.Dot, CodeGenerationSupporter.Euclidean); + vResult.Metric = vMetric; + } + Comma tTopN:Identifier EqualsSign vTopN = integer + { + Match(tTopN, CodeGenerationSupporter.TopN); + vResult.TopN = vTopN; + } + RightParenthesis simpleTableReferenceAliasOpt[vResult] + ; + predictTableReference[SubDmlFlags subDmlFlags] returns [PredictTableReference vResult] : {NextTokenMatches(CodeGenerationSupporter.Predict)}? diff --git a/SqlScriptDom/ScriptDom/SqlServer/ScriptGenerator/SqlScriptGeneratorVisitor.VectorSearchTableReference.cs b/SqlScriptDom/ScriptDom/SqlServer/ScriptGenerator/SqlScriptGeneratorVisitor.VectorSearchTableReference.cs new file mode 100644 index 0000000..55e4730 --- /dev/null +++ b/SqlScriptDom/ScriptDom/SqlServer/ScriptGenerator/SqlScriptGeneratorVisitor.VectorSearchTableReference.cs @@ -0,0 +1,56 @@ +//------------------------------------------------------------------------------ +// +// Copyright (c) Microsoft Corporation. All rights reserved. +// +//------------------------------------------------------------------------------ + +namespace Microsoft.SqlServer.TransactSql.ScriptDom.ScriptGenerator +{ + partial class SqlScriptGeneratorVisitor + { + /// + /// Script generator visitor for VECTOR_SEARCH function: https://learn.microsoft.com/sql/t-sql/functions/vector-search-transact-sql + /// Syntax: + /// VECTOR_SEARCH( + /// TABLE = object[AS source_table_alias], + /// COLUMN = vector_column, + /// SIMILAR_TO = query_vector, + /// METRIC = { 'cosine' | 'dot' | 'euclidean' }, + /// TOP_N = k + /// ) [AS result_table_alias] + /// + public override void ExplicitVisit(VectorSearchTableReference node) + { + AlignmentPoint start = new AlignmentPoint(); + MarkAndPushAlignmentPoint(start); + + GenerateIdentifier(CodeGenerationSupporter.VectorSearch); + GenerateSymbol(TSqlTokenType.LeftParenthesis); + + NewLineAndIndent(); + GenerateNameEqualsValue(TSqlTokenType.Table, node.Table); + GenerateSymbol(TSqlTokenType.Comma); + + NewLineAndIndent(); + GenerateNameEqualsValue(TSqlTokenType.Column, node.Column); + GenerateSymbol(TSqlTokenType.Comma); + + NewLineAndIndent(); + GenerateNameEqualsValue(CodeGenerationSupporter.SimilarTo, node.SimilarTo); + GenerateSymbol(TSqlTokenType.Comma); + + NewLineAndIndent(); + GenerateNameEqualsValue(CodeGenerationSupporter.Metric, node.Metric); + GenerateSymbol(TSqlTokenType.Comma); + + NewLineAndIndent(); + GenerateNameEqualsValue(CodeGenerationSupporter.TopN, node.TopN); + + NewLine(); + GenerateSymbol(TSqlTokenType.RightParenthesis); + GenerateSpaceAndAlias(node.Alias); + + PopAlignmentPoint(); + } + } +} diff --git a/Test/SqlDom/Baselines170/VectorFunctionTests170.sql b/Test/SqlDom/Baselines170/VectorFunctionTests170.sql new file mode 100644 index 0000000..dc893ed --- /dev/null +++ b/Test/SqlDom/Baselines170/VectorFunctionTests170.sql @@ -0,0 +1,23 @@ +DECLARE @qv AS VECTOR (1536) = AI_GENERATE_EMBEDDINGS(N'Pink Floyd music style' USE MODEL Ada2Embeddings); + +SELECT t.id, + s.distance, + t.title +FROM VECTOR_SEARCH( + TABLE = [dbo].[wikipedia_articles_embeddings] AS t, + COLUMN = [content_vector], + SIMILAR_TO = @qv, + METRIC = 'cosine', + TOP_N = 10 + ) AS s +ORDER BY s.distance; + +SELECT * +FROM VECTOR_SEARCH( + TABLE = wikipedia_articles_embeddings, + COLUMN = dbo.wikipedia_articles_embeddings.content_vector, + SIMILAR_TO = @qv, + METRIC = 'dot', + TOP_N = 10 + ) +ORDER BY distance; \ No newline at end of file diff --git a/Test/SqlDom/Only170SyntaxTests.cs b/Test/SqlDom/Only170SyntaxTests.cs index 686e953..ef94b35 100644 --- a/Test/SqlDom/Only170SyntaxTests.cs +++ b/Test/SqlDom/Only170SyntaxTests.cs @@ -21,6 +21,7 @@ public partial class SqlDomTests new ParserTest170("CreateExternalModelStatementTests170.sql", nErrors80: 2, nErrors90: 2, nErrors100: 2, nErrors110: 2, nErrors120: 2, nErrors130: 4, nErrors140: 4, nErrors150: 4, nErrors160: 4), new ParserTest170("AlterExternalModelStatementTests170.sql", nErrors80: 2, nErrors90: 2, nErrors100: 2, nErrors110: 2, nErrors120: 2, nErrors130: 5, nErrors140: 5, nErrors150: 5, nErrors160: 5), new ParserTest170("DropExternalModelStatementTests170.sql", nErrors80: 1, nErrors90: 1, nErrors100: 1, nErrors110: 1, nErrors120: 1, nErrors130: 1, nErrors140: 1, nErrors150: 1, nErrors160: 1), + new ParserTest170("VectorFunctionTests170.sql", nErrors80: 1, nErrors90: 1, nErrors100: 2, nErrors110: 2, nErrors120: 2, nErrors130: 2, nErrors140: 2, nErrors150: 2, nErrors160: 2), }; private static readonly ParserTest[] SqlAzure170_TestInfos = diff --git a/Test/SqlDom/ParserErrorsTests.cs b/Test/SqlDom/ParserErrorsTests.cs index 35bd265..6a46317 100644 --- a/Test/SqlDom/ParserErrorsTests.cs +++ b/Test/SqlDom/ParserErrorsTests.cs @@ -7357,5 +7357,131 @@ public void DropExternalModelNegativeTest() "DROP EXTERNAL MODEL abc WITH (LOCATION = 'www.somemodellocation.235');", new ParserErrorInfo(29, "SQL46010", "(")); } + + [TestMethod] + [Priority(0)] + [SqlStudioTestCategory(Category.UnitTest)] + public void VectorSearchErrorTest170() + { + // Missing required parameters: TABLE + ParserTestUtils.ErrorTest170( + "SELECT * FROM VECTOR_SEARCH()", + new ParserErrorInfo(28, "SQL46010", ")")); + + // Missing required parameters: COLUMN + ParserTestUtils.ErrorTest170( + "SELECT * FROM VECTOR_SEARCH(TABLE = tbl1)", + new ParserErrorInfo(40, "SQL46010", ")")); + + // Missing required parameters: SIMILAR_TO + ParserTestUtils.ErrorTest170( + "SELECT * FROM VECTOR_SEARCH(TABLE = tbl1, COLUMN = col1)", + new ParserErrorInfo(55, "SQL46010", ")")); + + // Missing required parameters: METRIC + ParserTestUtils.ErrorTest170( + "SELECT * FROM VECTOR_SEARCH(TABLE = tbl1, COLUMN = col1, SIMILAR_TO = query_vector)", + new ParserErrorInfo(82, "SQL46010", ")")); + + // Missing required parameters: TOP_N + ParserTestUtils.ErrorTest170( + "SELECT * FROM VECTOR_SEARCH(TABLE = tbl1, COLUMN = col1, SIMILAR_TO = query_vector, METRIC = 'dot')", + new ParserErrorInfo(98, "SQL46010", ")")); + + // Invalid order: COLUMN before TABLE + ParserTestUtils.ErrorTest170( + "SELECT * FROM VECTOR_SEARCH(COLUMN = col1, TABLE = tbl1, SIMILAR_TO = query_vector, METRIC = 'dot', TOP_N = 5)", + new ParserErrorInfo(28, "SQL46010", "COLUMN")); + + // Invalid order: SIMILAR_TO before COLUMN + ParserTestUtils.ErrorTest170( + "SELECT * FROM VECTOR_SEARCH(TABLE = tbl1, SIMILAR_TO = query_vector, COLUMN = col1, METRIC = 'dot', TOP_N = 5)", + new ParserErrorInfo(42, "SQL46010", "SIMILAR_TO")); + + // Invalid order: METRIC before SIMILAR_TO + ParserTestUtils.ErrorTest170( + "SELECT * FROM VECTOR_SEARCH(TABLE = tbl1, COLUMN = col1, METRIC = 'dot', SIMILAR_TO = query_vector, TOP_N = 5)", + new ParserErrorInfo(57, "SQL46005", "SIMILAR_TO", "METRIC")); + + // Invalid value: TABLE = 'tbl1' (should be identifier, not string) + ParserTestUtils.ErrorTest170( + "SELECT * FROM VECTOR_SEARCH(TABLE = 'tbl1', COLUMN = col1, SIMILAR_TO = query_vector, METRIC = 'dot', TOP_N = 5)", + new ParserErrorInfo(36, "SQL46010", "'tbl1'")); + + // Invalid value: TABLE = 123 (should be identifier, not integer) + ParserTestUtils.ErrorTest170( + "SELECT * FROM VECTOR_SEARCH(TABLE = 123, COLUMN = col1, SIMILAR_TO = query_vector, METRIC = 'dot', TOP_N = 5)", + new ParserErrorInfo(36, "SQL46010", "123")); + + // Invalid value: COLUMN = 'col1' (should be identifier, not string) + ParserTestUtils.ErrorTest170( + "SELECT * FROM VECTOR_SEARCH(TABLE = tbl1, COLUMN = 'col1', SIMILAR_TO = query_vector, METRIC = 'dot', TOP_N = 5)", + new ParserErrorInfo(51, "SQL46010", "'col1'")); + + // Invalid value: COLUMN = 123 (should be identifier, not integer) + ParserTestUtils.ErrorTest170( + "SELECT * FROM VECTOR_SEARCH(TABLE = tbl1, COLUMN = 123, SIMILAR_TO = query_vector, METRIC = 'dot', TOP_N = 5)", + new ParserErrorInfo(51, "SQL46010", "123")); + + // Invalid value: METRIC = dot (should be string literal) + ParserTestUtils.ErrorTest170( + "SELECT * FROM VECTOR_SEARCH(TABLE = tbl1, COLUMN = col1, SIMILAR_TO = query_vector, METRIC = dot, TOP_N = 5)", + new ParserErrorInfo(93, "SQL46010", "dot")); + + // Invalid value: METRIC = 'invalid_value' (should be either 'cosine', 'dot', or 'euclidean') + ParserTestUtils.ErrorTest170( + "SELECT * FROM VECTOR_SEARCH(TABLE = tbl1, COLUMN = col1, SIMILAR_TO = query_vector, METRIC = 'invalid_value', TOP_N = 5)", + new ParserErrorInfo(93, "SQL46010", "'invalid_value'")); + + // Invalid value: TOP_N = '5' (should be integer, not string) + ParserTestUtils.ErrorTest170( + "SELECT * FROM VECTOR_SEARCH(TABLE = tbl1, COLUMN = col1, SIMILAR_TO = query_vector, METRIC = 'dot', TOP_N = '5')", + new ParserErrorInfo(108, "SQL46010", "'5'")); + + // Invalid value: TOP_N = -5 (should be positive integer) + ParserTestUtils.ErrorTest170( + "SELECT * FROM VECTOR_SEARCH(TABLE = tbl1, COLUMN = col1, SIMILAR_TO = query_vector, METRIC = 'dot', TOP_N = -5)", + new ParserErrorInfo(108, "SQL46010", "-")); + + // Missing value after equals for TABLE + ParserTestUtils.ErrorTest170( + "SELECT * FROM VECTOR_SEARCH(TABLE = , COLUMN = col1, SIMILAR_TO = query_vector, METRIC = 'dot', TOP_N = 5)", + new ParserErrorInfo(36, "SQL46010", ",")); + + // Missing value after equals for COLUMN + ParserTestUtils.ErrorTest170( + "SELECT * FROM VECTOR_SEARCH(TABLE = tbl1, COLUMN = , SIMILAR_TO = query_vector, METRIC = 'dot', TOP_N = 5)", + new ParserErrorInfo(51, "SQL46010", ",")); + + // Missing value after equals for SIMILAR_TO + ParserTestUtils.ErrorTest170( + "SELECT * FROM VECTOR_SEARCH(TABLE = tbl1, COLUMN = col1, SIMILAR_TO = , METRIC = 'dot', TOP_N = 5)", + new ParserErrorInfo(70, "SQL46010", ",")); + + // Missing value after equals for METRIC + ParserTestUtils.ErrorTest170( + "SELECT * FROM VECTOR_SEARCH(TABLE = tbl1, COLUMN = col1, SIMILAR_TO = query_vector, METRIC = , TOP_N = 5)", + new ParserErrorInfo(93, "SQL46010", ",")); + + // Missing value after equals for TOP_N + ParserTestUtils.ErrorTest170( + "SELECT * FROM VECTOR_SEARCH(TABLE = tbl1, COLUMN = col1, SIMILAR_TO = query_vector, METRIC = 'dot', TOP_N = )", + new ParserErrorInfo(108, "SQL46010", ")")); + + // Extra parameter + ParserTestUtils.ErrorTest170( + "SELECT * FROM VECTOR_SEARCH(TABLE = tbl1, COLUMN = col1, SIMILAR_TO = query_vector, METRIC = 'dot', TOP_N = 5, EXTRA_PARAM = 'value')", + new ParserErrorInfo(109, "SQL46010", ",")); + + // Extra comma at end + ParserTestUtils.ErrorTest170( + "SELECT * FROM VECTOR_SEARCH(TABLE = tbl1, COLUMN = col1, SIMILAR_TO = query_vector, METRIC = 'dot', TOP_N = 5,)", + new ParserErrorInfo(109, "SQL46010", ",")); + + // Function call with constant input, not keyword params + ParserTestUtils.ErrorTest170( + "SELECT * FROM VECTOR_SEARCH('tbl1', 'col1', 'query_vector', 'dot', 5)", + new ParserErrorInfo(28, "SQL46010", "'tbl1'")); + } } } diff --git a/Test/SqlDom/TestScripts/VectorFunctionTests170.sql b/Test/SqlDom/TestScripts/VectorFunctionTests170.sql new file mode 100644 index 0000000..1d1c679 --- /dev/null +++ b/Test/SqlDom/TestScripts/VectorFunctionTests170.sql @@ -0,0 +1,25 @@ +DECLARE @qv VECTOR(1536) = AI_GENERATE_EMBEDDINGS(N'Pink Floyd music style' USE MODEL Ada2Embeddings); + +SELECT + t.id, s.distance, t.title +FROM + VECTOR_SEARCH( + TABLE = [dbo].[wikipedia_articles_embeddings] as t, + COLUMN = [content_vector], + SIMILAR_TO = @qv, + METRIC = 'cosine', + TOP_N = 10 + ) AS s +ORDER BY s.distance + +SELECT + * +FROM + VECTOR_SEARCH( + TABLE = wikipedia_articles_embeddings, + COLUMN = dbo.wikipedia_articles_embeddings.content_vector, + SIMILAR_TO = @qv, + METRIC = 'dot', + TOP_N = 10 + ) +ORDER BY distance \ No newline at end of file From db55dd55a189c00fc78c170c0191e19abc63767e Mon Sep 17 00:00:00 2001 From: Urvashi Raj Date: Wed, 6 Aug 2025 03:59:05 +0000 Subject: [PATCH 08/12] Merged PR 1757817: Added EM Permission support MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Code Change - [ ] The [Common checklist](https://msdata.visualstudio.com/SQLToolsAndLibraries/_git/Common?path=/Templates/PR%20Checklist%20for%20SQLToolsAndLibraries.md&version=GBmain&_a=preview) has been reviewed and followed - [ ] Code changes are accompanied by appropriate unit tests - [ ] Identified and included SMEs needed to review code changes - [ ] Follow the [steps](https://msdata.visualstudio.com/SQLToolsAndLibraries/_wiki/wikis/SQLToolsAndLibraries.wiki/33838/Adding-or-Extending-TSql-support-in-Sql-Dom?anchor=make-the-changes-in) here to make changes in the code ## Testing - [ ] Follow the [steps](https://msdata.visualstudio.com/SQLToolsAndLibraries/_wiki/wikis/SQLToolsAndLibraries.wiki/33838/Adding-or-Extending-TSql-support-in-Sql-Dom?anchor=to-extend-the-tests-do-the-following%3A) here to add new tests for your feature ## Documentation - [ ] Update relevant documentation in the [wiki](https://dev.azure.com/msdata/SQLToolsAndLibraries/_wiki/wikis/SQLToolsAndLibraries.wiki) and the README.md file ## Additional Information Please provide any additional information that might be helpful for the reviewers Added EM Permission support ---- #### AI description (iteration 1) #### PR Classification This PR implements a new feature by adding support for External Model permissions in ScriptDOM, enabling the parsing and generation of new permission-related T-SQL statements. #### PR Summary The pull request enhances ScriptDOM to correctly handle External Model permissions—covering GRANT, DENY, REVOKE, and ALTER AUTHORIZATION operations—and integrates comprehensive tests and token generation support to improve IntelliSense and syntax-highlighting as outlined in the linked work items. - **`Test/SqlDom/ExternalModelPermissionUnitTests.cs`**: Adds unit tests for External Model permission statements, including backward compatibility checks. - **`Test/SqlDom/Baselines170/SecurityStatementExternalModelTests170.sql` & `Test/SqlDom/TestScripts/SecurityStatementExternalModelTests170.sql`**: Introduce baseline and test scripts with various External Model permission scenarios. - **`SqlScriptDom/Parser/TSql/TSql170ParserBaseInternal.cs` & `SqlScriptDom/Parser/TSql/SecurityObjectKind.cs`**: Update parsing logic and the security object enumeration to recognize and handle External Model keywords. - **`SqlScriptDom/ScriptDom/SqlServer/ScriptGenerator/SqlScriptGeneratorVisitor.SecurityTargetObject.cs`**: Enhances token generation to support the External Model permission syntax. Related work items: #4088625, #4088633 --- .../Parser/TSql/SecurityObjectKind.cs | 1 + SqlScriptDom/Parser/TSql/TSql170.g | 2 +- .../Parser/TSql/TSql170ParserBaseInternal.cs | 24 +++++++ ...ptGeneratorVisitor.SecurityTargetObject.cs | 8 ++- ...SecurityStatementExternalModelTests170.sql | 71 +++++++++++++++++++ Test/SqlDom/Only170SyntaxTests.cs | 2 + ...SecurityStatementExternalModelTests170.sql | 47 ++++++++++++ 7 files changed, 152 insertions(+), 3 deletions(-) create mode 100644 Test/SqlDom/Baselines170/SecurityStatementExternalModelTests170.sql create mode 100644 Test/SqlDom/TestScripts/SecurityStatementExternalModelTests170.sql diff --git a/SqlScriptDom/Parser/TSql/SecurityObjectKind.cs b/SqlScriptDom/Parser/TSql/SecurityObjectKind.cs index 8d72de8..4d1cf17 100644 --- a/SqlScriptDom/Parser/TSql/SecurityObjectKind.cs +++ b/SqlScriptDom/Parser/TSql/SecurityObjectKind.cs @@ -42,6 +42,7 @@ public enum SecurityObjectKind SearchPropertyList = 23, ServerRole = 24, AvailabilityGroup = 25, + ExternalModel = 26, } #pragma warning restore 1591 diff --git a/SqlScriptDom/Parser/TSql/TSql170.g b/SqlScriptDom/Parser/TSql/TSql170.g index 4baae9d..a3545e2 100644 --- a/SqlScriptDom/Parser/TSql/TSql170.g +++ b/SqlScriptDom/Parser/TSql/TSql170.g @@ -13951,7 +13951,7 @@ securityTargetObjectCommon[SecurityTargetObject vParent] ( vIdentifier2=securityStatementPermission { - vParent.ObjectKind = ParseSecurityObjectKind(vIdentifier1, vIdentifier2); + vParent.ObjectKind = ParseSecurityObjectKindTSql170(vIdentifier1, vIdentifier2); } | vIdentifier2=securityStatementPermission vIdentifier3=securityStatementPermission diff --git a/SqlScriptDom/Parser/TSql/TSql170ParserBaseInternal.cs b/SqlScriptDom/Parser/TSql/TSql170ParserBaseInternal.cs index 5c332e5..2ca8412 100644 --- a/SqlScriptDom/Parser/TSql/TSql170ParserBaseInternal.cs +++ b/SqlScriptDom/Parser/TSql/TSql170ParserBaseInternal.cs @@ -46,5 +46,29 @@ public TSql170ParserBaseInternal(bool initialQuotedIdentifiersOn) } #endregion + + /// + /// Parses security object kind with support for External Model (TSql170+) + /// + /// The first identifier. + /// The second identifier. + /// The security object kind. + protected SecurityObjectKind ParseSecurityObjectKindTSql170(Identifier identifier1, Identifier identifier2) + { + if (identifier1 == null) + { + throw new ArgumentNullException(nameof(identifier1)); + } + + switch (identifier1.Value.ToUpperInvariant()) + { + case CodeGenerationSupporter.External: + Match(identifier2, CodeGenerationSupporter.Model); + return SecurityObjectKind.ExternalModel; + default: + // Fall back to the base class implementation for all other cases + return TSql160ParserBaseInternal.ParseSecurityObjectKind(identifier1, identifier2); + } + } } } diff --git a/SqlScriptDom/ScriptDom/SqlServer/ScriptGenerator/SqlScriptGeneratorVisitor.SecurityTargetObject.cs b/SqlScriptDom/ScriptDom/SqlServer/ScriptGenerator/SqlScriptGeneratorVisitor.SecurityTargetObject.cs index ebceaa0..571e65d 100644 --- a/SqlScriptDom/ScriptDom/SqlServer/ScriptGenerator/SqlScriptGeneratorVisitor.SecurityTargetObject.cs +++ b/SqlScriptDom/ScriptDom/SqlServer/ScriptGenerator/SqlScriptGeneratorVisitor.SecurityTargetObject.cs @@ -13,7 +13,7 @@ partial class SqlScriptGeneratorVisitor protected static Dictionary> _securityObjectKindGenerators = new Dictionary>() { - { SecurityObjectKind.AvailabilityGroup, new List() { + { SecurityObjectKind.AvailabilityGroup, new List() { new IdentifierGenerator(CodeGenerationSupporter.Availability, true), new IdentifierGenerator(CodeGenerationSupporter.Group) }}, @@ -96,11 +96,15 @@ partial class SqlScriptGeneratorVisitor { SecurityObjectKind.FullTextStopList, new List() { new IdentifierGenerator(CodeGenerationSupporter.Fulltext, true), new KeywordGenerator(TSqlTokenType.StopList)}}, - + { SecurityObjectKind.SearchPropertyList, new List() { new IdentifierGenerator(CodeGenerationSupporter.Search, true), new IdentifierGenerator(CodeGenerationSupporter.Property, true), new IdentifierGenerator(CodeGenerationSupporter.List)}}, + + { SecurityObjectKind.ExternalModel, new List() { + new IdentifierGenerator(CodeGenerationSupporter.External, true), + new IdentifierGenerator(CodeGenerationSupporter.Model) }}, }; diff --git a/Test/SqlDom/Baselines170/SecurityStatementExternalModelTests170.sql b/Test/SqlDom/Baselines170/SecurityStatementExternalModelTests170.sql new file mode 100644 index 0000000..a64b7b7 --- /dev/null +++ b/Test/SqlDom/Baselines170/SecurityStatementExternalModelTests170.sql @@ -0,0 +1,71 @@ +GRANT ALTER ANY EXTERNAL MODEL TO datascientist; + +GRANT ALTER ANY EXTERNAL MODEL TO PUBLIC + WITH GRANT OPTION; + +GRANT ALTER ANY EXTERNAL MODEL TO mlrole, adminrole + AS dbo; + +GRANT EXECUTE + ON EXTERNAL MODEL::MyPredictionModel TO analyst; + +GRANT EXECUTE + ON EXTERNAL MODEL::schema1.ModelName TO user1, user2 + WITH GRANT OPTION; + +GRANT CONTROL + ON EXTERNAL MODEL::dbo.SalesModel TO mladmin + AS dbo; + +GRANT VIEW DEFINITION + ON EXTERNAL MODEL::MySchema.MyModel TO PUBLIC; + +DENY ALTER ANY EXTERNAL MODEL TO restricteduser; + +DENY EXECUTE + ON EXTERNAL MODEL::TestModel TO guest CASCADE; + +DENY CONTROL + ON EXTERNAL MODEL::test.model1 TO user1, user2 CASCADE + AS admin; + +REVOKE ALTER ANY EXTERNAL MODEL TO olduser; + +REVOKE GRANT OPTION FOR EXECUTE + ON EXTERNAL MODEL::MyModel TO tempuser CASCADE; + +REVOKE CONTROL + ON EXTERNAL MODEL::prod.RecommendationModel TO contractor + AS manager; + +REVOKE EXECUTE + ON EXTERNAL MODEL::dbo.CustomerModel TO PUBLIC CASCADE; + +ALTER AUTHORIZATION + ON EXTERNAL MODEL::MyModel + TO newowner; + +ALTER AUTHORIZATION + ON EXTERNAL MODEL::schema1.ModelName + TO SCHEMA OWNER; + +ALTER AUTHORIZATION + ON EXTERNAL MODEL::test.experimentalmodel + TO datateam; + +GRANT EXECUTE, VIEW DEFINITION + ON EXTERNAL MODEL::MultiPermModel TO analyst; + +DENY EXECUTE, CONTROL + ON EXTERNAL MODEL::RestrictedModel TO guest; + +REVOKE EXECUTE, VIEW DEFINITION + ON EXTERNAL MODEL::OldModel TO formeranalyst; + +GRANT EXECUTE + ON EXTERNAL MODEL::MyDatabase.MySchema.ModelWithSpaces TO testuser; + +DENY ALTER ANY EXTERNAL MODEL TO userwithspaces; + +REVOKE CONTROL + ON EXTERNAL MODEL::QuotedModel TO QuotedUser; \ No newline at end of file diff --git a/Test/SqlDom/Only170SyntaxTests.cs b/Test/SqlDom/Only170SyntaxTests.cs index ef94b35..a339cf7 100644 --- a/Test/SqlDom/Only170SyntaxTests.cs +++ b/Test/SqlDom/Only170SyntaxTests.cs @@ -22,6 +22,8 @@ public partial class SqlDomTests new ParserTest170("AlterExternalModelStatementTests170.sql", nErrors80: 2, nErrors90: 2, nErrors100: 2, nErrors110: 2, nErrors120: 2, nErrors130: 5, nErrors140: 5, nErrors150: 5, nErrors160: 5), new ParserTest170("DropExternalModelStatementTests170.sql", nErrors80: 1, nErrors90: 1, nErrors100: 1, nErrors110: 1, nErrors120: 1, nErrors130: 1, nErrors140: 1, nErrors150: 1, nErrors160: 1), new ParserTest170("VectorFunctionTests170.sql", nErrors80: 1, nErrors90: 1, nErrors100: 2, nErrors110: 2, nErrors120: 2, nErrors130: 2, nErrors140: 2, nErrors150: 2, nErrors160: 2), + new ParserTest170("SecurityStatementExternalModelTests170.sql", nErrors80: 2, nErrors90: 17, nErrors100: 17, nErrors110: 17, nErrors120: 17, nErrors130: 17, nErrors140: 17, nErrors150: 17, nErrors160: 17), + }; private static readonly ParserTest[] SqlAzure170_TestInfos = diff --git a/Test/SqlDom/TestScripts/SecurityStatementExternalModelTests170.sql b/Test/SqlDom/TestScripts/SecurityStatementExternalModelTests170.sql new file mode 100644 index 0000000..04442d6 --- /dev/null +++ b/Test/SqlDom/TestScripts/SecurityStatementExternalModelTests170.sql @@ -0,0 +1,47 @@ +GRANT ALTER ANY EXTERNAL MODEL TO datascientist; +GRANT ALTER ANY EXTERNAL MODEL TO PUBLIC + WITH GRANT OPTION; +GRANT ALTER ANY EXTERNAL MODEL TO mlrole, adminrole + AS dbo; +GRANT EXECUTE + ON EXTERNAL MODEL::MyPredictionModel TO analyst; +GRANT EXECUTE + ON EXTERNAL MODEL::schema1.ModelName TO user1, user2 + WITH GRANT OPTION; +GRANT CONTROL + ON EXTERNAL MODEL::dbo.SalesModel TO mladmin + AS dbo; +GRANT VIEW DEFINITION + ON EXTERNAL MODEL::MySchema.MyModel TO PUBLIC; +DENY ALTER ANY EXTERNAL MODEL TO restricteduser; +DENY EXECUTE + ON EXTERNAL MODEL::TestModel TO guest + CASCADE; +DENY CONTROL + ON EXTERNAL MODEL::test.model1 TO user1, user2 + CASCADE + AS admin; +REVOKE ALTER ANY EXTERNAL MODEL FROM olduser; +REVOKE GRANT OPTION FOR EXECUTE + ON EXTERNAL MODEL::MyModel FROM tempuser + CASCADE; +REVOKE CONTROL + ON EXTERNAL MODEL::prod.RecommendationModel FROM contractor + AS manager; +REVOKE EXECUTE + ON EXTERNAL MODEL::dbo.CustomerModel FROM PUBLIC + CASCADE; +ALTER AUTHORIZATION ON EXTERNAL MODEL::MyModel TO newowner; +ALTER AUTHORIZATION ON EXTERNAL MODEL::schema1.ModelName TO SCHEMA OWNER; +ALTER AUTHORIZATION ON EXTERNAL MODEL::test.experimentalmodel TO datateam; +GRANT EXECUTE, VIEW DEFINITION + ON EXTERNAL MODEL::MultiPermModel TO analyst; +DENY EXECUTE, CONTROL + ON EXTERNAL MODEL::RestrictedModel TO guest; +REVOKE EXECUTE, VIEW DEFINITION + ON EXTERNAL MODEL::OldModel FROM formeranalyst; +GRANT EXECUTE + ON EXTERNAL MODEL::MyDatabase.MySchema.ModelWithSpaces TO testuser; +DENY ALTER ANY EXTERNAL MODEL TO userwithspaces; +REVOKE CONTROL + ON EXTERNAL MODEL::QuotedModel FROM QuotedUser; \ No newline at end of file From bc528d95a4ba60510d0694c733c529eea4ad3e6c Mon Sep 17 00:00:00 2001 From: MerlinBot Date: Wed, 6 Aug 2025 21:18:47 +0000 Subject: [PATCH 09/12] Update .NET SDK to latest patch version (8.0.413) --- global.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/global.json b/global.json index 3fc0ba6..b812260 100644 --- a/global.json +++ b/global.json @@ -1,6 +1,6 @@ { "sdk": { - "version": "8.0.412", + "version": "8.0.413", "rollForward": "latestMajor" }, "msbuild-sdks": { From 5e0176a9874fcba65260f17e991c52ea8ff2e4ec Mon Sep 17 00:00:00 2001 From: Leila Lali Date: Thu, 7 Aug 2025 21:05:52 +0000 Subject: [PATCH 10/12] Merged PR 1764425: Adding release notes for 170.82.0 # Pull Request Template for ScriptDom ## Description Please provide a detailed description, include the link to the design specification or SQL feature document for the new TSQL syntaxes. Make sure to add links to the Github or DevDiv issue Before submitting your pull request, please ensure you have completed the following: ## Code Change - [ ] The [Common checklist](https://msdata.visualstudio.com/SQLToolsAndLibraries/_git/Common?path=/Templates/PR%20Checklist%20for%20SQLToolsAndLibraries.md&version=GBmain&_a=preview) has been reviewed and followed - [ ] Code changes are accompanied by appropriate unit tests - [ ] Identified and included SMEs needed to review code changes - [ ] Follow the [steps](https://msdata.visualstudio.com/SQLToolsAndLibraries/_wiki/wikis/SQLToolsAndLibraries.wiki/33838/Adding-or-Extending-TSql-support-in-Sql-Dom?anchor=make-the-changes-in) here to make changes in the code ## Testing - [ ] Follow the [steps](https://msdata.visualstudio.com/SQLToolsAndLibraries/_wiki/wikis/SQLToolsAndLibraries.wiki/33838/Adding-or-Extending-TSql-support-in-Sql-Dom?anchor=to-extend-the-tests-do-the-following%3A) here to add new tests for your feature ## Documentation - [ ] Update relevant documentation in the [wiki](https://dev.azure.com/msdata/SQLToolsAndLibraries/_wiki/wikis/SQLToolsAndLibraries.wiki) and the README.md file ## Additional Information Please provide any additional information that might be helpful for the reviewers Adding release notes for 170.82.0 --- release-notes/170/170.82.0.md | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 release-notes/170/170.82.0.md diff --git a/release-notes/170/170.82.0.md b/release-notes/170/170.82.0.md new file mode 100644 index 0000000..59e0112 --- /dev/null +++ b/release-notes/170/170.82.0.md @@ -0,0 +1,27 @@ +# Release Notes + +## Microsoft.SqlServer.TransactSql.ScriptDom 170.82.0 +This update brings the below changes over the previous release: + +### Target Platform Support + +* .NET Framework 4.7.2 (Windows x86, Windows x64) +* .NET 8 (Windows x86, Windows x64, Linux, macOS) +* .NET Standard 2.0+ (Windows x86, Windows x64, Linux, macOS) + +### Dependencies +* Updates.NET SDK to latest patch version 8.0.413 + +#### .NET Framework +#### .NET Core + +### New Features +* Adds support for `AI_GENERATE_CHUNKS`, `JSON_OBJECTAGG`, `AI_GENERATE_EMBEDDINGS` and `VECTOR_SEARCH` Functions. +* Adds supports for `EXTERNAL MODEL` object type. + +### Fixed + +### Changes +* Updated Fabric DW syntax to better match the definition for identity columns. + +### Known Issues From 76ce80bf575f17467a5321abe29fd133b99afdc6 Mon Sep 17 00:00:00 2001 From: Leila Lali Date: Thu, 7 Aug 2025 14:17:09 -0700 Subject: [PATCH 11/12] remove extras --- SqlScriptDom/Parser/TSql/CodeGenerationSupporter.cs | 4 ---- 1 file changed, 4 deletions(-) diff --git a/SqlScriptDom/Parser/TSql/CodeGenerationSupporter.cs b/SqlScriptDom/Parser/TSql/CodeGenerationSupporter.cs index ba75024..1106636 100644 --- a/SqlScriptDom/Parser/TSql/CodeGenerationSupporter.cs +++ b/SqlScriptDom/Parser/TSql/CodeGenerationSupporter.cs @@ -230,7 +230,6 @@ internal static class CodeGenerationSupporter internal const string CompressionDelay = "COMPRESSION_DELAY"; internal const string CompressAllRowGroups = "COMPRESS_ALL_ROW_GROUPS"; internal const string Concat = "CONCAT"; - internal const string Cosine = "COSINE"; internal const string Configuration = "CONFIGURATION"; internal const string ConnectionOptions = "CONNECTION_OPTIONS"; internal const string Contained = "CONTAINED"; @@ -307,7 +306,6 @@ internal static class CodeGenerationSupporter internal const string Description = "DESCRIPTION"; internal const string DesiredState = "DESIRED_STATE"; internal const string DiskANN = "DISKANN"; - internal const string Dot = "DOT"; internal const string Delay = "DELAY"; internal const string DelayedDurability = "DELAYED_DURABILITY"; internal const string DelimitedText = "DELIMITEDTEXT"; @@ -359,7 +357,6 @@ internal static class CodeGenerationSupporter internal const string EnvironmentVariables = "ENVIRONMENT_VARIABLES"; internal const string Equal = "="; internal const string Error = "ERROR"; - internal const string Euclidean = "EUCLIDEAN"; internal const string ErrorBrokerConversations = "ERROR_BROKER_CONVERSATIONS"; internal const string ErrorDataSource = "ERRORFILE_DATA_SOURCE"; internal const string ErrorFile = "ERRORFILE"; @@ -630,7 +627,6 @@ internal static class CodeGenerationSupporter internal const string MessageForwardSize = "MESSAGE_FORWARD_SIZE"; internal const string Metric = "METRIC"; internal const string MigrationState = "MIGRATION_STATE"; - internal const string Metric = "METRIC"; internal const string Min = "MIN"; internal const string MinGrantPercent = "MIN_GRANT_PERCENT"; internal const string MinCpuPercent = "MIN_CPU_PERCENT"; From 76434dacc84109bd5aa0d1d40b6bce7a37332b64 Mon Sep 17 00:00:00 2001 From: Leila Lali Date: Thu, 7 Aug 2025 15:50:16 -0700 Subject: [PATCH 12/12] fixing the build --- SqlScriptDom/Parser/TSql/TSql170.g | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SqlScriptDom/Parser/TSql/TSql170.g b/SqlScriptDom/Parser/TSql/TSql170.g index a3545e2..a8f9bf2 100644 --- a/SqlScriptDom/Parser/TSql/TSql170.g +++ b/SqlScriptDom/Parser/TSql/TSql170.g @@ -32081,7 +32081,7 @@ aiGenerateEmbeddingsFunctionCall vResult.Input = vInput; } - tUse:Use // ← your reserved keyword + tUse:Use // your reserved keyword { UpdateTokenInfo(vResult, tUse); }