diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index dbdf428..f4b000c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -117,23 +117,37 @@ Before sending a Pull Request, please do the following: ### Helpful notes for SQLDOM extensions -1. For changing the DOM classes, modify the XML file (the C# code is generated based on this during the build process) `$(EnlistmentRoot)\Source\SqlDom\SqlScriptDom\Parser\TSql\Ast.xml`. Change Ast.xml to put the class pieces on their appropriate statements. - 1. The build process is defined in `$(EnlistmentRoot)\Source\SqlDom\SqlScriptDom\SqlScriptDom.props` (Target Name="CreateAST") - 2. The generated files are dropped in `$(EnlistmentRoot)\obj\\\SqlScriptDom.csproj\` +1. For changing the DOM classes, modify the XML file (the C# code is generated based on this during the build process) `SqlScriptDom\Parser\TSql\Ast.xml`. Change Ast.xml to put the class pieces on their appropriate statements. + 1. The build process is defined in `SqlScriptDom\GenerateFiles.props` (Target Name="CreateAST") + 2. The generated files are dropped in `obj\SqlScriptDom\AnyCPU\\\Microsoft.SqlServer.TransactSql.ScriptDom.csproj\` + + Regenerating generated sources (what to run and when) + --------------------------------------------------- + When you modify `Source\SqlDom\SqlScriptDom\Parser\TSql\Ast.xml` or any `TSql<#>.g` grammar file, the C# parser and DOM sources are produced by MSBuild generation targets (for example `CreateAST`). These targets are invoked automatically during a normal build, so in most cases you can simply run: + + ```powershell + dotnet build Source\SqlDom\SqlScriptDom\Microsoft.SqlServer.TransactSql.ScriptDom.csproj -c Debug + ``` + + If you only want to run generation targets (no compile) or need more detailed generation logs, invoke the MSBuild targets directly: + + ```powershell + dotnet msbuild Source\SqlDom\SqlScriptDom\Microsoft.SqlServer.TransactSql.ScriptDom.csproj -t:GLexerParserCompile;GSqlTokenTypesCompile;CreateAST -p:Configuration=Debug + ``` + + Generated files are written into the `obj` folder for that project (for example `obj\SqlScriptDom\AnyCPU\\\Microsoft.SqlServer.TransactSql.ScriptDom.csproj\`). If antlr or related tools are missing, see `Directory.Build.props` for `AntlrLocation` and follow the repo guidance to supply the binaries. 2. For changing the parser, modify the .g file here: -`$(EnlistmentRoot)\Source\SqlDom\SqlScriptDom\Parser\TSql\TSql<#>.g` where # is the version (ie - 100, 120, 130). This will usually be the latest number if adding new grammar. Change the Tsql(xxx).g file to parse the new syntax. - 1. The build process is defined in `$(EnlistmentRoot)\Source\SqlDom\SqlScriptDom\SqlScriptDom.props` (Target Name="CreateAST") - 2. The generated files are dropped in `$(EnlistmentRoot)\obj\x86|x64\Debug|Release\sqlscriptdom.csproj\` +`SqlScriptDom\Parser\TSql\TSql<#>.g` where # is the version (ie - 100, 120, 130). This will usually be the latest number if adding new grammar. Change the Tsql(xxx).g file to parse the new syntax. + 1. The build process is defined in `SqlScriptDom\GenerateFiles.props` (Target Name="CreateAST") + 2. The generated files are dropped in `obj\SqlScriptDom\AnyCPU\\\Microsoft.SqlServer.TransactSql.ScriptDom.csproj\` -3. For changing the ScriptGenerator, modify the appropriate file (i.e. Visitor that accepts the modified DOM class) in here: `$(EnlistmentRoot)\Source\SqlDom\SqlScriptDom\ScriptDom\SqlServer\ScriptGenerator`. - 1. To add a new ScriptGenerator, you need to add the file to `$(EnlistmentRoot)\Source\SqlDom\SqlScriptDom\SqlScriptDom.props` +3. For changing the ScriptGenerator, modify the appropriate file (i.e. Visitor that accepts the modified DOM class) in here: `SqlScriptDom\ScriptDom\SqlServer\ScriptGenerator`. 1. Change The visitors SqlScriptGenerator.X to use the new piece from AST.XML 1. If you're adding syntax that's Azure-only or Standalone-only, implement appropriate Versioning Visitor for your constructs. -4. When adding/removing new files please add/remove an entry to/from `$(EnlistmentRoot)\Source\SqlDom\SqlScriptDom\SqlScriptDom.csproj` -5. To extend the tests do the following: +4. To extend the tests do the following: 1. Baselines# needs to be updated or added with the appropriate .sql file as expected results. 1. The Only#SyntaxTests.cs needs to be extended or added to specify the appropriate TestScripts script. 1. Positive tests go in Only#SyntaxTests.cs if adding new grammar. diff --git a/SqlScriptDom/Parser/TSql/Ast.xml b/SqlScriptDom/Parser/TSql/Ast.xml index c3de308..347befd 100644 --- a/SqlScriptDom/Parser/TSql/Ast.xml +++ b/SqlScriptDom/Parser/TSql/Ast.xml @@ -271,6 +271,11 @@ Summary="The xml data type option."/> + + + + + @@ -2520,6 +2525,10 @@ + + + + diff --git a/SqlScriptDom/Parser/TSql/CodeGenerationSupporter.cs b/SqlScriptDom/Parser/TSql/CodeGenerationSupporter.cs index e59bb88..0a64b28 100644 --- a/SqlScriptDom/Parser/TSql/CodeGenerationSupporter.cs +++ b/SqlScriptDom/Parser/TSql/CodeGenerationSupporter.cs @@ -427,6 +427,8 @@ internal static class CodeGenerationSupporter internal const string FlushIntervalSecondsAlt = "DATA_FLUSH_INTERVAL_SECONDS"; internal const string Fn = "FN"; internal const string Float = "FLOAT"; + internal const string Float16 = "FLOAT16"; + internal const string Float32 = "FLOAT32"; internal const string For = "FOR"; internal const string ForceFailoverAllowDataLoss = "FORCE_FAILOVER_ALLOW_DATA_LOSS"; internal const string ForceScan = "FORCESCAN"; @@ -718,6 +720,8 @@ internal static class CodeGenerationSupporter internal const string OperatorAudit = "OPERATOR_AUDIT"; internal const string Optimistic = "OPTIMISTIC"; internal const string Optimize = "OPTIMIZE"; + internal const string OptimizedLocking = "OPTIMIZED_LOCKING"; + internal const string OptimizeForArraySearch = "OPTIMIZE_FOR_ARRAY_SEARCH"; internal const string OptimizeForSequentialKey = "OPTIMIZE_FOR_SEQUENTIAL_KEY"; internal const string OptimizerQueue = "OPTIMIZER_QUEUE"; internal const string Order = "ORDER"; diff --git a/SqlScriptDom/Parser/TSql/DatabaseOptionKind.cs b/SqlScriptDom/Parser/TSql/DatabaseOptionKind.cs index c3118e2..84d3832 100644 --- a/SqlScriptDom/Parser/TSql/DatabaseOptionKind.cs +++ b/SqlScriptDom/Parser/TSql/DatabaseOptionKind.cs @@ -98,7 +98,8 @@ public enum DatabaseOptionKind Ledger = 68, ManualCutover = 69, - PerformCutover = 70 + PerformCutover = 70, + OptimizedLocking = 71 } #pragma warning restore 1591 diff --git a/SqlScriptDom/Parser/TSql/IndexOptionHelper.cs b/SqlScriptDom/Parser/TSql/IndexOptionHelper.cs index 1a0545f..4a739a1 100644 --- a/SqlScriptDom/Parser/TSql/IndexOptionHelper.cs +++ b/SqlScriptDom/Parser/TSql/IndexOptionHelper.cs @@ -46,6 +46,8 @@ private IndexOptionHelper() AddOptionMapping(IndexOptionKind.VectorMetric, CodeGenerationSupporter.Metric, SqlVersionFlags.TSql170AndAbove); AddOptionMapping(IndexOptionKind.VectorType, CodeGenerationSupporter.Type, SqlVersionFlags.TSql170AndAbove); + AddOptionMapping(IndexOptionKind.OptimizeForArraySearch, CodeGenerationSupporter.OptimizeForArraySearch, SqlVersionFlags.TSql170AndAbove); + } internal static readonly IndexOptionHelper Instance = new IndexOptionHelper(); diff --git a/SqlScriptDom/Parser/TSql/IndexOptionKind.cs b/SqlScriptDom/Parser/TSql/IndexOptionKind.cs index 7f2914a..ae69cb5 100644 --- a/SqlScriptDom/Parser/TSql/IndexOptionKind.cs +++ b/SqlScriptDom/Parser/TSql/IndexOptionKind.cs @@ -40,6 +40,7 @@ public enum IndexOptionKind XmlCompression = 23, VectorMetric = 24, VectorType = 25, + OptimizeForArraySearch = 26, } #pragma warning restore 1591 diff --git a/SqlScriptDom/Parser/TSql/TSql170.g b/SqlScriptDom/Parser/TSql/TSql170.g index 89c95cb..d746306 100644 --- a/SqlScriptDom/Parser/TSql/TSql170.g +++ b/SqlScriptDom/Parser/TSql/TSql170.g @@ -3064,17 +3064,14 @@ alterDbModify returns [AlterDatabaseStatement vResult = null] ; alterDbModifyAzureOptions returns [AlterDatabaseSetStatement vResult = FragmentFactory.CreateFragment()] -{ - bool hasManualCutover = false; -} : azureOptions[vResult, vResult.Options] ( With tManualCutover:Identifier { Match(tManualCutover, CodeGenerationSupporter.ManualCutover); - hasManualCutover = true; vResult.WithManualCutover = true; + UpdateTokenInfo(vResult, tManualCutover); } )? ; @@ -3247,6 +3244,8 @@ dbOptionStateItem[ref ulong encounteredOptions] returns [DatabaseOption vResult vResult = changeTrackingDbOption | {NextTokenMatches(CodeGenerationSupporter.AcceleratedDatabaseRecovery)}? vResult = acceleratedDatabaseRecoveryOption + | {NextTokenMatches(CodeGenerationSupporter.OptimizedLocking)}? + vResult = optimizedLockingOption | {NextTokenMatches(CodeGenerationSupporter.Containment)}? vResult = dbContainmentOption | {NextTokenMatches(CodeGenerationSupporter.Hadr)}? @@ -3560,6 +3559,30 @@ acceleratedDatabaseRecoveryOption returns [AcceleratedDatabaseRecoveryDatabaseOp ) ; +optimizedLockingOption returns [OptimizedLockingDatabaseOption vResult = FragmentFactory.CreateFragment()] + : tOptimizedLocking:Identifier + { + Match(tOptimizedLocking, CodeGenerationSupporter.OptimizedLocking); + vResult.OptionKind = DatabaseOptionKind.OptimizedLocking; + UpdateTokenInfo(vResult, tOptimizedLocking); + } + ( + (EqualsSign tOff:Off + { + vResult.OptionState = OptionState.Off; + UpdateTokenInfo(vResult, tOff); + } + ) + | + (EqualsSign tOn:On + { + vResult.OptionState = OptionState.On; + UpdateTokenInfo(vResult, tOn); + } + ) + ) + ; + changeTrackingOnOptions [ChangeTrackingDatabaseOption vParent] { bool autoCleanupEncountered = false; @@ -23930,7 +23953,6 @@ dropSecurityPolicyStatement returns [DropSecurityPolicyStatement vResult = Fragm createExternalModelStatement returns [CreateExternalModelStatement vResult = FragmentFactory.CreateFragment()] { Identifier vName; - long encounteredOptions = 0; } : tModel:Identifier vName = identifier { @@ -24092,7 +24114,6 @@ externalModelParameters[ExternalModelStatement vParent] alterExternalModelStatement returns [AlterExternalModelStatement vResult = FragmentFactory.CreateFragment()] { Identifier vName; - long encounteredOptions = 0; } : tModel:Identifier vName = identifier { @@ -30680,6 +30701,32 @@ xmlDataType [SchemaObjectName vName] returns [XmlDataTypeReference vResult = Fra )? ; +vectorDataType [SchemaObjectName vName] returns [VectorDataTypeReference vResult = FragmentFactory.CreateFragment()] +{ + vResult.Name = vName; + vResult.UpdateTokenInfo(vName); + + IntegerLiteral vDimension = null; + Identifier vBaseType = null; +} + : + ( LeftParenthesis vDimension=integer + { + vResult.Dimension = vDimension; + } + ( + Comma vBaseType=identifier + { + vResult.BaseType = vBaseType; + } + )? + tRParen:RightParenthesis + { + UpdateTokenInfo(vResult,tRParen); + } + ) + ; + scalarDataType returns [DataTypeReference vResult = null] { SchemaObjectName vName; @@ -30700,6 +30747,9 @@ scalarDataType returns [DataTypeReference vResult = null] ( {isXmlDataType}? vResult = xmlDataType[vName] + | + {typeOption == SqlDataTypeOption.Vector}? + vResult = vectorDataType[vName] | {typeOption != SqlDataTypeOption.None}? vResult = sqlDataTypeWithoutNational[vName, typeOption] @@ -32103,6 +32153,7 @@ aiGenerateEmbeddingsFunctionCall ScalarExpression vInput; SchemaObjectName vModelName; ScalarExpression vParams = null; + ScalarExpression vParamsInner; } : tFunc:Identifier LeftParenthesis @@ -32114,8 +32165,7 @@ aiGenerateEmbeddingsFunctionCall { vResult.Input = vInput; } - - tUse:Use // your reserved keyword + tUse:Use { UpdateTokenInfo(vResult, tUse); } @@ -32123,21 +32173,97 @@ aiGenerateEmbeddingsFunctionCall tModel:Identifier { Match(tModel, CodeGenerationSupporter.Model); - } - - vModelName=schemaObjectThreePartName - { - vResult.ModelName = vModelName; - } - - ( - tParams:Identifier + UpdateTokenInfo(vResult, tModel); + } + + // --- MODEL NAME: single-part only (strict) --------------------------------------------- + // We accept exactly **one identifier token** after "USE MODEL". + // + // Why: + // - Users may store model names that *visually* contain dots or spaces, e.g. [dbo.MyDefaultModel]. + // When bracket-delimited, the lexer emits this as a **single** token (QuotedIdentifier), so it's OK. + // - True multipart names (db.schema.model) must be rejected here, so we do NOT consume any Dot tokens. + // + // Allowed (single token): + // USE MODEL MyDefaultModel -- Identifier + // USE MODEL [dbo.MyDefaultModel] -- QuotedIdentifier (one token; dot lives inside the brackets) + // + // Rejected (multipart): + // USE MODEL dbo.MyDefaultModel -- Identifier '.' Identifier (two tokens + Dot) + // USE MODEL [dbo].[MyDefaultModel] -- QuotedIdentifier '.' QuotedIdentifier + // + // Token notes: + // - Identifier : unquoted identifier; cannot contain spaces or '.'. + // - QuotedIdentifier : bracket-delimited; may contain spaces and '.' inside the brackets. + ( + vModelId:Identifier + { + vModelName = this.FragmentFactory.CreateFragment(); + vModelName.Identifiers.Add(this.CreateIdentifierFromToken(vModelId)); + vResult.ModelName = vModelName; + } + | + vModelQId:QuotedIdentifier { - Match(tParams, CodeGenerationSupporter.Parameters); + vModelName = this.FragmentFactory.CreateFragment(); + vModelName.Identifiers.Add(this.CreateIdentifierFromToken(vModelQId)); + vResult.ModelName = vModelName; } - LeftParenthesis + ) + + // --- Optional PARAMETERS clause --------------------------------------------------------- + // Shape: [PARAMETERS ()] | [PARAMETERS ] + // Goals: + // 1) Accept a general **expression** as the PARAMETERS value. + // 2) Preserve user-written parentheses by constructing a ParenthesisExpression node + // for the "( )" variant so pretty-printing round-trips exactly. + // 3) **Reject** a bare JSON string literal (e.g., PARAMETERS '{...}'); callers must pass + // a parsed JSON expression (e.g., TRY_CONVERT(JSON, N'{}')). + // Notes: + // - Some builds tokenize PARAMETERS as a keyword; others as an Identifier. Support both. + ( + ( + // PARAMETERS as a true keyword token. + tParamsKw:Parameters + { + UpdateTokenInfo(vResult, tParamsKw); + } + | + // PARAMETERS as an identifier token; enforce its text equals "Parameters". + tParams:Identifier + { + Match(tParams, CodeGenerationSupporter.Parameters); + UpdateTokenInfo(vResult, tParams); + } + ) + + // ---- Value of PARAMETERS ----------------------------------------------------------- + ( + // Variant A: user wrote parentheses around the value: PARAMETERS ( ) + // Build a ParenthesisExpression so the printer re-emits parens. + tLP:LeftParenthesis + vParamsInner=expression + tRP:RightParenthesis + { + ParenthesisExpression p = this.FragmentFactory.CreateFragment(); + p.Expression = vParamsInner; + vParams = p; + + // Attach LP/RP token info for accurate script generation. + UpdateTokenInfo(p, tLP); + UpdateTokenInfo(p, tRP); + } + | + // Variant B: bare expression without surrounding parentheses. + // Guardrail: Disallow a leading string literal so that + // PARAMETERS '{"dimensions":768}' + // is a **syntax error**, while + // PARAMETERS TRY_CONVERT(JSON, N'{}') + // is allowed. + // LA(1) is the next token type; block both ASCII ('...') and Unicode (N'...') strings. + { LA(1) != AsciiStringLiteral && LA(1) != UnicodeStringLiteral }? vParams=expression - RightParenthesis + ) { vResult.OptionalParameters = vParams; } diff --git a/SqlScriptDom/Parser/TSql/TSqlFabricDW.g b/SqlScriptDom/Parser/TSql/TSqlFabricDW.g index 3f1cb10..9e1fea8 100644 --- a/SqlScriptDom/Parser/TSql/TSqlFabricDW.g +++ b/SqlScriptDom/Parser/TSql/TSqlFabricDW.g @@ -3221,6 +3221,8 @@ dbOptionStateItem[ref ulong encounteredOptions] returns [DatabaseOption vResult vResult = changeTrackingDbOption | {NextTokenMatches(CodeGenerationSupporter.AcceleratedDatabaseRecovery)}? vResult = acceleratedDatabaseRecoveryOption + | {NextTokenMatches(CodeGenerationSupporter.OptimizedLocking)}? + vResult = optimizedLockingOption | {NextTokenMatches(CodeGenerationSupporter.Containment)}? vResult = dbContainmentOption | {NextTokenMatches(CodeGenerationSupporter.Hadr)}? @@ -3534,6 +3536,30 @@ acceleratedDatabaseRecoveryOption returns [AcceleratedDatabaseRecoveryDatabaseOp ) ; +optimizedLockingOption returns [OptimizedLockingDatabaseOption vResult = FragmentFactory.CreateFragment()] + : tOptimizedLocking:Identifier + { + Match(tOptimizedLocking, CodeGenerationSupporter.OptimizedLocking); + vResult.OptionKind = DatabaseOptionKind.OptimizedLocking; + UpdateTokenInfo(vResult, tOptimizedLocking); + } + ( + (EqualsSign tOff:Off + { + vResult.OptionState = OptionState.Off; + UpdateTokenInfo(vResult, tOff); + } + ) + | + (EqualsSign tOn:On + { + vResult.OptionState = OptionState.On; + UpdateTokenInfo(vResult, tOn); + } + ) + ) + ; + changeTrackingOnOptions [ChangeTrackingDatabaseOption vParent] { bool autoCleanupEncountered = false; diff --git a/SqlScriptDom/ScriptDom/SqlServer/ScriptGenerator/SqlScriptGeneratorVisitor.AiGenerateEmbeddingsFunction.cs b/SqlScriptDom/ScriptDom/SqlServer/ScriptGenerator/SqlScriptGeneratorVisitor.AiGenerateEmbeddingsFunction.cs index 0efdc60..299a1c3 100644 --- a/SqlScriptDom/ScriptDom/SqlServer/ScriptGenerator/SqlScriptGeneratorVisitor.AiGenerateEmbeddingsFunction.cs +++ b/SqlScriptDom/ScriptDom/SqlServer/ScriptGenerator/SqlScriptGeneratorVisitor.AiGenerateEmbeddingsFunction.cs @@ -36,9 +36,8 @@ public override void ExplicitVisit(AIGenerateEmbeddingsFunctionCall node) { GenerateSpace(); GenerateIdentifierWithoutCasing(CodeGenerationSupporter.Parameters); - GenerateSpaceAndSymbol(TSqlTokenType.LeftParenthesis); + GenerateSpace(); GenerateFragmentIfNotNull(node.OptionalParameters); - GenerateSymbol(TSqlTokenType.RightParenthesis); } // Emit closing parenthesis diff --git a/SqlScriptDom/ScriptDom/SqlServer/ScriptGenerator/SqlScriptGeneratorVisitor.OptimizedLockingAlterDatabaseOption.cs b/SqlScriptDom/ScriptDom/SqlServer/ScriptGenerator/SqlScriptGeneratorVisitor.OptimizedLockingAlterDatabaseOption.cs new file mode 100644 index 0000000..6ee34d9 --- /dev/null +++ b/SqlScriptDom/ScriptDom/SqlServer/ScriptGenerator/SqlScriptGeneratorVisitor.OptimizedLockingAlterDatabaseOption.cs @@ -0,0 +1,31 @@ +//------------------------------------------------------------------------------ +// +// Copyright (c) Microsoft Corporation. All rights reserved. +// +//------------------------------------------------------------------------------ +using Microsoft.SqlServer.TransactSql.ScriptDom; + +namespace Microsoft.SqlServer.TransactSql.ScriptDom.ScriptGenerator +{ + partial class SqlScriptGeneratorVisitor + { + public override void ExplicitVisit(OptimizedLockingDatabaseOption node) + { + System.Diagnostics.Debug.Assert(node.OptionKind == DatabaseOptionKind.OptimizedLocking); + GenerateIdentifier(CodeGenerationSupporter.OptimizedLocking); + GenerateSpace(); + + switch (node.OptionState) + { + case OptionState.Off: + GenerateSymbolAndSpace(TSqlTokenType.EqualsSign); + GenerateKeyword(TSqlTokenType.Off); + break; + case OptionState.On: + GenerateSymbolAndSpace(TSqlTokenType.EqualsSign); + GenerateKeyword(TSqlTokenType.On); + break; + } + } + } +} diff --git a/SqlScriptDom/ScriptDom/SqlServer/ScriptGenerator/SqlScriptGeneratorVisitor.VectorDataType.cs b/SqlScriptDom/ScriptDom/SqlServer/ScriptGenerator/SqlScriptGeneratorVisitor.VectorDataType.cs new file mode 100644 index 0000000..c651659 --- /dev/null +++ b/SqlScriptDom/ScriptDom/SqlServer/ScriptGenerator/SqlScriptGeneratorVisitor.VectorDataType.cs @@ -0,0 +1,24 @@ +//------------------------------------------------------------------------------ +// +// Copyright (c) Microsoft Corporation. All rights reserved. +// +//------------------------------------------------------------------------------ + +namespace Microsoft.SqlServer.TransactSql.ScriptDom.ScriptGenerator +{ + partial class SqlScriptGeneratorVisitor + { + public override void ExplicitVisit(VectorDataTypeReference node) + { + GenerateIdentifier(CodeGenerationSupporter.Vector); + GenerateSymbol(TSqlTokenType.LeftParenthesis); + GenerateFragmentIfNotNull(node.Dimension); + if (node.BaseType != null) + { + GenerateSymbol(TSqlTokenType.Comma); + GenerateSpaceAndFragmentIfNotNull(node.BaseType); + } + GenerateSymbol(TSqlTokenType.RightParenthesis); + } + } +} diff --git a/Test/SqlDom/Baselines170/AiGenerateEmbeddingsTests170.sql b/Test/SqlDom/Baselines170/AiGenerateEmbeddingsTests170.sql index 72493c7..1728248 100644 --- a/Test/SqlDom/Baselines170/AiGenerateEmbeddingsTests170.sql +++ b/Test/SqlDom/Baselines170/AiGenerateEmbeddingsTests170.sql @@ -1,8 +1,10 @@ SELECT AI_GENERATE_EMBEDDINGS('My Default Input Text' USE MODEL MyDefaultModel); SELECT AI_GENERATE_EMBEDDINGS(N'My Default Input Text' USE MODEL MyDefaultModel); SELECT AI_GENERATE_EMBEDDINGS('My Default Input Text' USE MODEL MyDefaultModel PARAMETERS (TRY_CONVERT (JSON, N'{}'))); -SELECT AI_GENERATE_EMBEDDINGS('My Default Input Text' USE MODEL dbo.MyDefaultModel); -SELECT AI_GENERATE_EMBEDDINGS('My Default Input Text' USE MODEL MyDatabase.dbo.MyDefaultModel); +SELECT AI_GENERATE_EMBEDDINGS('My Default Input Text' USE MODEL MyDefaultModel PARAMETERS TRY_CONVERT (JSON, N'{}')); +SELECT AI_GENERATE_EMBEDDINGS('My Default Input Text' USE MODEL [My Default Model]); +SELECT AI_GENERATE_EMBEDDINGS('My Default Input Text' USE MODEL [dbo.MyDefaultModel]); +SELECT AI_GENERATE_EMBEDDINGS('' USE MODEL MyDefaultModel); GO CREATE FUNCTION dbo.AI_GENERATE_EMBEDDINGS @@ -35,7 +37,7 @@ GO CREATE VIEW dbo.MyEmbeddingView AS -SELECT AI_GENERATE_EMBEDDINGS(N'View Input' USE MODEL dbo.MyDefaultModel) AS EmbeddingResult; +SELECT AI_GENERATE_EMBEDDINGS(N'View Input' USE MODEL MyDefaultModel) AS EmbeddingResult; GO CREATE TABLE dbo.SimpleEmbeddingTable ( diff --git a/Test/SqlDom/Baselines170/JsonIndexTests170.sql b/Test/SqlDom/Baselines170/JsonIndexTests170.sql index f6010de..c6c72e2 100644 --- a/Test/SqlDom/Baselines170/JsonIndexTests170.sql +++ b/Test/SqlDom/Baselines170/JsonIndexTests170.sql @@ -26,4 +26,10 @@ CREATE JSON INDEX [IX JSON Index] CREATE JSON INDEX IX_JSON_Complex ON dbo.Documents (Content) - FOR ('$.metadata.title', '$.content.sections[*].text', '$.tags[*]'); \ No newline at end of file + FOR ('$.metadata.title', '$.content.sections[*].text', '$.tags[*]'); + +CREATE JSON INDEX IX_JSON + ON dbo.Users (JsonData) WITH (OPTIMIZE_FOR_ARRAY_SEARCH = ON); + +CREATE JSON INDEX IX_JSON + ON dbo.Users (JsonData) WITH (PAD_INDEX = ON, DROP_EXISTING = ON); \ No newline at end of file diff --git a/Test/SqlDom/Baselines170/OptimizedLockingTests170.sql b/Test/SqlDom/Baselines170/OptimizedLockingTests170.sql new file mode 100644 index 0000000..2e0d73d --- /dev/null +++ b/Test/SqlDom/Baselines170/OptimizedLockingTests170.sql @@ -0,0 +1,5 @@ +ALTER DATABASE db + SET OPTIMIZED_LOCKING = ON; + +ALTER DATABASE db + SET OPTIMIZED_LOCKING = OFF; diff --git a/Test/SqlDom/Baselines170/VectorFunctionTests170.sql b/Test/SqlDom/Baselines170/VectorFunctionTests170.sql index 2176223..9ab5f85 100644 --- a/Test/SqlDom/Baselines170/VectorFunctionTests170.sql +++ b/Test/SqlDom/Baselines170/VectorFunctionTests170.sql @@ -1,4 +1,4 @@ -DECLARE @qv AS VECTOR (1536) = AI_GENERATE_EMBEDDINGS(N'Pink Floyd music style' USE MODEL Ada2Embeddings); +DECLARE @qv AS VECTOR(1536) = AI_GENERATE_EMBEDDINGS(N'Pink Floyd music style' USE MODEL Ada2Embeddings); SELECT t.id, s.distance, diff --git a/Test/SqlDom/Baselines170/VectorTypeSecondParameterTests170.sql b/Test/SqlDom/Baselines170/VectorTypeSecondParameterTests170.sql new file mode 100644 index 0000000..c41e7f2 --- /dev/null +++ b/Test/SqlDom/Baselines170/VectorTypeSecondParameterTests170.sql @@ -0,0 +1,17 @@ +CREATE TABLE tbl ( + embedding VECTOR(1) +); + +CREATE TABLE tbl ( + embedding VECTOR(1, float32) +); + +CREATE TABLE tbl ( + embedding VECTOR(1, FLOAT16) +); + +DECLARE @embedding AS VECTOR(2); + +DECLARE @embedding AS VECTOR(2, FLOAT32); + +DECLARE @embedding AS VECTOR(2, float16); \ No newline at end of file diff --git a/Test/SqlDom/Only170SyntaxTests.cs b/Test/SqlDom/Only170SyntaxTests.cs index 949e22d..c0d7765 100644 --- a/Test/SqlDom/Only170SyntaxTests.cs +++ b/Test/SqlDom/Only170SyntaxTests.cs @@ -13,21 +13,23 @@ public partial class SqlDomTests private static readonly ParserTest[] Only170TestInfos = { new ParserTest170("RegexpTVFTests170.sql", nErrors80: 1, nErrors90: 1, nErrors100: 0, nErrors110: 0, nErrors120: 0, nErrors130: 0, nErrors140: 0, nErrors150: 0, nErrors160: 0), - new ParserTest170("JsonIndexTests170.sql", nErrors80: 2, nErrors90: 8, nErrors100: 8, nErrors110: 8, nErrors120: 8, nErrors130: 8, nErrors140: 8, nErrors150: 8, nErrors160: 8), + new ParserTest170("JsonIndexTests170.sql", nErrors80: 2, nErrors90: 10, nErrors100: 10, nErrors110: 10, nErrors120: 10, nErrors130: 10, nErrors140: 10, nErrors150: 10, nErrors160: 10), new ParserTest170("VectorIndexTests170.sql", nErrors80: 2, nErrors90: 12, nErrors100: 12, nErrors110: 12, nErrors120: 12, nErrors130: 12, nErrors140: 12, nErrors150: 12, nErrors160: 12), new ParserTest170("AlterDatabaseManualCutoverTests170.sql", nErrors80: 4, nErrors90: 4, nErrors100: 4, nErrors110: 4, nErrors120: 4, nErrors130: 4, nErrors140: 4, nErrors150: 4, nErrors160: 4), new ParserTest170("CreateColumnStoreIndexTests170.sql", nErrors80: 3, nErrors90: 3, nErrors100: 3, nErrors110: 3, nErrors120: 3, nErrors130: 0, nErrors140: 0, nErrors150: 0, nErrors160: 0), new ParserTest170("RegexpTests170.sql", nErrors80: 0, nErrors90: 0, nErrors100: 0, nErrors110: 0, nErrors120: 0, nErrors130: 0, nErrors140: 0, nErrors150: 0, nErrors160: 0), new ParserTest170("AiGenerateChunksTests170.sql", nErrors80: 19, nErrors90: 16, nErrors100: 15, nErrors110: 15, nErrors120: 15, nErrors130: 15, nErrors140: 15, nErrors150: 15, nErrors160: 15), new ParserTest170("JsonFunctionTests170.sql", nErrors80: 13, nErrors90: 8, nErrors100: 38, nErrors110: 38, nErrors120: 38, nErrors130: 38, nErrors140: 38, nErrors150: 38, nErrors160: 38), - new ParserTest170("AiGenerateEmbeddingsTests170.sql", nErrors80: 12, nErrors90: 9, nErrors100: 9, nErrors110: 9, nErrors120: 9, nErrors130: 9, nErrors140: 9, nErrors150: 9, nErrors160: 9), + new ParserTest170("AiGenerateEmbeddingsTests170.sql", nErrors80: 14, nErrors90: 11, nErrors100: 11, nErrors110: 11, nErrors120: 11, nErrors130: 11, nErrors140: 11, nErrors150: 11, nErrors160: 11), new ParserTest170("CreateExternalModelStatementTests170.sql", nErrors80: 2, nErrors90: 2, nErrors100: 2, nErrors110: 2, nErrors120: 2, nErrors130: 4, nErrors140: 4, nErrors150: 4, nErrors160: 4), new ParserTest170("AlterExternalModelStatementTests170.sql", nErrors80: 2, nErrors90: 2, nErrors100: 2, nErrors110: 2, nErrors120: 2, nErrors130: 5, nErrors140: 5, nErrors150: 5, nErrors160: 5), new ParserTest170("DropExternalModelStatementTests170.sql", nErrors80: 1, nErrors90: 1, nErrors100: 1, nErrors110: 1, nErrors120: 1, nErrors130: 1, nErrors140: 1, nErrors150: 1, nErrors160: 1), new ParserTest170("VectorFunctionTests170.sql", nErrors80: 1, nErrors90: 1, nErrors100: 3, nErrors110: 3, nErrors120: 3, nErrors130: 3, nErrors140: 3, nErrors150: 3, nErrors160: 3), new ParserTest170("SecurityStatementExternalModelTests170.sql", nErrors80: 2, nErrors90: 17, nErrors100: 17, nErrors110: 17, nErrors120: 17, nErrors130: 17, nErrors140: 17, nErrors150: 17, nErrors160: 17), - new ParserTest170("CreateEventSessionNotLikePredicate.sql", nErrors80: 2, nErrors90: 1, nErrors100: 1, nErrors110: 1, nErrors120: 1, nErrors130: 0, nErrors140: 0, nErrors150: 0, nErrors160: 0), - new ParserTest170("RegexpLikeTests170.sql", nErrors80: 15, nErrors90: 15, nErrors100: 15, nErrors110: 18, nErrors120: 18, nErrors130: 18, nErrors140: 18, nErrors150: 18, nErrors160: 18) + new ParserTest170("VectorTypeSecondParameterTests170.sql", nErrors80: 2, nErrors90: 2, nErrors100: 2, nErrors110: 2, nErrors120: 2, nErrors130: 2, nErrors140: 2, nErrors150: 2, nErrors160: 2), + new ParserTest170("RegexpLikeTests170.sql", nErrors80: 15, nErrors90: 15, nErrors100: 15, nErrors110: 18, nErrors120: 18, nErrors130: 18, nErrors140: 18, nErrors150: 18, nErrors160: 18), + new ParserTest170("OptimizedLockingTests170.sql", nErrors80: 2, nErrors90: 2, nErrors100: 2, nErrors110: 2, nErrors120: 2, nErrors130: 2, nErrors140: 2, nErrors150: 2, nErrors160: 2), + new ParserTest170("CreateEventSessionNotLikePredicate.sql", nErrors80: 2, nErrors90: 1, nErrors100: 1, nErrors110: 1, nErrors120: 1, nErrors130: 0, nErrors140: 0, nErrors150: 0, nErrors160: 0) }; private static readonly ParserTest[] SqlAzure170_TestInfos = diff --git a/Test/SqlDom/ParserErrorsTests.cs b/Test/SqlDom/ParserErrorsTests.cs index b793a14..3644819 100644 --- a/Test/SqlDom/ParserErrorsTests.cs +++ b/Test/SqlDom/ParserErrorsTests.cs @@ -7315,11 +7315,6 @@ public void GenerateEmbeddingsNegativeTest170() "SELECT AI_GENERATE_EMBEDDINGS('My Default Input Text' PARAMETERS (TRY_CONVERT(JSON, N'{}')))", new ParserErrorInfo(54, "SQL46010", "PARAMETERS")); - // PARAMETERS missing parentheses - ParserTestUtils.ErrorTest170( - "SELECT AI_GENERATE_EMBEDDINGS('My Default Input Text' USE MODEL MyModel PARAMETERS TRY_CONVERT(JSON, N'{}'))", - new ParserErrorInfo(83, "SQL46010", "TRY_CONVERT")); - // Invalid expression inside PARAMETERS (missing closing parenthesis) ParserTestUtils.ErrorTest170( "SELECT AI_GENERATE_EMBEDDINGS('My Default Input Text' USE MODEL MyModel PARAMETERS (TRY_CONVERT(JSON, N'{}')", @@ -7344,6 +7339,21 @@ public void GenerateEmbeddingsNegativeTest170() ParserTestUtils.ErrorTest170( "SELECT AI_GENERATE_EMBEDDINGS('My Default Input Text' USE MODEL NULL)", new ParserErrorInfo(64, "SQL46010", "NULL")); + + // Invalid model reference: schema-qualified name after USE MODEL + ParserTestUtils.ErrorTest170( + "SELECT AI_GENERATE_EMBEDDINGS('My Default Input Text' USE MODEL [dbo].[MyDefaultModel])", + new ParserErrorInfo(69, "SQL46010", ".")); + + // Invalid model reference: database + schema-qualified name after USE MODEL + ParserTestUtils.ErrorTest170( + "SELECT AI_GENERATE_EMBEDDINGS('My Default Input Text' USE MODEL [MyDatabase].[dbo].[MyDefaultModel])", + new ParserErrorInfo(76, "SQL46010", ".")); + + // Varchar Optional Parameters + ParserTestUtils.ErrorTest170( + "SELECT AI_GENERATE_EMBEDDINGS('My Default Input Text' USE MODEL MyDefaultModel PARAMETERS '{\"dimensions\" : 768 }')", + new ParserErrorInfo(90, "SQL46010", "'{\"dimensions\" : 768 }'")); } diff --git a/Test/SqlDom/TestScripts/AiGenerateEmbeddingsTests170.sql b/Test/SqlDom/TestScripts/AiGenerateEmbeddingsTests170.sql index 72493c7..1728248 100644 --- a/Test/SqlDom/TestScripts/AiGenerateEmbeddingsTests170.sql +++ b/Test/SqlDom/TestScripts/AiGenerateEmbeddingsTests170.sql @@ -1,8 +1,10 @@ SELECT AI_GENERATE_EMBEDDINGS('My Default Input Text' USE MODEL MyDefaultModel); SELECT AI_GENERATE_EMBEDDINGS(N'My Default Input Text' USE MODEL MyDefaultModel); SELECT AI_GENERATE_EMBEDDINGS('My Default Input Text' USE MODEL MyDefaultModel PARAMETERS (TRY_CONVERT (JSON, N'{}'))); -SELECT AI_GENERATE_EMBEDDINGS('My Default Input Text' USE MODEL dbo.MyDefaultModel); -SELECT AI_GENERATE_EMBEDDINGS('My Default Input Text' USE MODEL MyDatabase.dbo.MyDefaultModel); +SELECT AI_GENERATE_EMBEDDINGS('My Default Input Text' USE MODEL MyDefaultModel PARAMETERS TRY_CONVERT (JSON, N'{}')); +SELECT AI_GENERATE_EMBEDDINGS('My Default Input Text' USE MODEL [My Default Model]); +SELECT AI_GENERATE_EMBEDDINGS('My Default Input Text' USE MODEL [dbo.MyDefaultModel]); +SELECT AI_GENERATE_EMBEDDINGS('' USE MODEL MyDefaultModel); GO CREATE FUNCTION dbo.AI_GENERATE_EMBEDDINGS @@ -35,7 +37,7 @@ GO CREATE VIEW dbo.MyEmbeddingView AS -SELECT AI_GENERATE_EMBEDDINGS(N'View Input' USE MODEL dbo.MyDefaultModel) AS EmbeddingResult; +SELECT AI_GENERATE_EMBEDDINGS(N'View Input' USE MODEL MyDefaultModel) AS EmbeddingResult; GO CREATE TABLE dbo.SimpleEmbeddingTable ( diff --git a/Test/SqlDom/TestScripts/JsonIndexTests170.sql b/Test/SqlDom/TestScripts/JsonIndexTests170.sql index 2a0f1ad..b5acf4a 100644 --- a/Test/SqlDom/TestScripts/JsonIndexTests170.sql +++ b/Test/SqlDom/TestScripts/JsonIndexTests170.sql @@ -28,4 +28,12 @@ FOR ('$.data.attributes'); -- JSON index with complex path expressions CREATE JSON INDEX IX_JSON_Complex ON dbo.Documents (Content) -FOR ('$.metadata.title', '$.content.sections[*].text', '$.tags[*]'); \ No newline at end of file +FOR ('$.metadata.title', '$.content.sections[*].text', '$.tags[*]'); + +-- Create JSON index with OPTIMIZE_FOR_ARRAY_SEARCH option +CREATE JSON INDEX IX_JSON ON dbo.Users (JsonData) +WITH (OPTIMIZE_FOR_ARRAY_SEARCH = ON); + +-- Create JSON index with PAD_INDEX and DROP_EXISTING options +CREATE JSON INDEX IX_JSON ON dbo.Users (JsonData) +WITH (PAD_INDEX = ON, DROP_EXISTING = ON); \ No newline at end of file diff --git a/Test/SqlDom/TestScripts/OptimizedLockingTests170.sql b/Test/SqlDom/TestScripts/OptimizedLockingTests170.sql new file mode 100644 index 0000000..aba6b6f --- /dev/null +++ b/Test/SqlDom/TestScripts/OptimizedLockingTests170.sql @@ -0,0 +1,5 @@ +ALTER DATABASE db + SET OPTIMIZED_LOCKING = ON; + +ALTER DATABASE db + SET OPTIMIZED_LOCKING = OFF; \ No newline at end of file diff --git a/Test/SqlDom/TestScripts/VectorTypeSecondParameterTests170.sql b/Test/SqlDom/TestScripts/VectorTypeSecondParameterTests170.sql new file mode 100644 index 0000000..35351bf --- /dev/null +++ b/Test/SqlDom/TestScripts/VectorTypeSecondParameterTests170.sql @@ -0,0 +1,11 @@ +CREATE TABLE tbl (embedding VECTOR(1)) + +CREATE TABLE tbl (embedding VECTOR(1, float32)) + +CREATE TABLE tbl (embedding VECTOR(1, FLOAT16)) + +DECLARE @embedding VECTOR(2) + +DECLARE @embedding VECTOR(2, FLOAT32) + +DECLARE @embedding VECTOR(2, float16) \ No newline at end of file diff --git a/global.json b/global.json index b812260..0f9441a 100644 --- a/global.json +++ b/global.json @@ -1,6 +1,6 @@ { "sdk": { - "version": "8.0.413", + "version": "8.0.414", "rollForward": "latestMajor" }, "msbuild-sdks": { diff --git a/release-notes/170/170.121.2.md b/release-notes/170/170.121.2.md new file mode 100644 index 0000000..d92500b --- /dev/null +++ b/release-notes/170/170.121.2.md @@ -0,0 +1,31 @@ +# Release Notes + +## Microsoft.SqlServer.TransactSql.ScriptDom 170.121.2 +This update brings the below changes over the previous release: + +### Target Platform Support + +* .NET Framework 4.7.2 (Windows x86, Windows x64) +* .NET 8 (Windows x86, Windows x64, Linux, macOS) +* .NET Standard 2.0+ (Windows x86, Windows x64, Linux, macOS) + +### Dependencies +* Updates .NET SDK to latest patch version 8.0.414 + +#### .NET Framework +#### .NET Core + +### New Features +* Adds support for 32-bit float vectors. +* Adds support for Optimized Locking database option. +* Adds OPTIMIZE_FOR_ARRAY_SEARCH option for Json index. +* Adds support for `JSON_QUERY` Functions. + +### Fixed +* Fixes parsing AI_GENERATE_EMBEDDINGS when PARAMETERS comes without parentheses +* Fixes https://github.com/microsoft/SqlScriptDOM/issues/163 +* Fixes https://github.com/microsoft/SqlScriptDOM/issues/160 +* Fixes https://github.com/microsoft/SqlScriptDOM/issues/153 +### Changes + +### Known Issues