From 43a50807440a32fbbf0247953052564304edbe6b Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 30 Jul 2025 17:23:23 +0000 Subject: [PATCH 1/9] Initial plan From 8d6642aa3009a5f6ca0363250a2a8250ff484868 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 30 Jul 2025 17:46:32 +0000 Subject: [PATCH 2/9] Implement vector index support - add grammar, AST classes, and script generators Co-authored-by: llali <17187348+llali@users.noreply.github.com> --- SqlScriptDom/Parser/TSql/Ast.xml | 15 ++++ .../Parser/TSql/CodeGenerationSupporter.cs | 6 ++ SqlScriptDom/Parser/TSql/IndexOptionHelper.cs | 3 + SqlScriptDom/Parser/TSql/IndexOptionKind.cs | 2 + SqlScriptDom/Parser/TSql/TSql170.g | 74 +++++++++++++++++++ SqlScriptDom/Parser/TSql/VectorIndexType.cs | 23 ++++++ .../Parser/TSql/VectorIndexTypeHelper.cs | 21 ++++++ SqlScriptDom/Parser/TSql/VectorMetricType.cs | 25 +++++++ .../Parser/TSql/VectorMetricTypeHelper.cs | 23 ++++++ ...ratorVisitor.CreateVectorIndexStatement.cs | 48 ++++++++++++ ...eneratorVisitor.VectorMetricIndexOption.cs | 21 ++++++ ...tGeneratorVisitor.VectorTypeIndexOption.cs | 21 ++++++ .../Baselines170/VectorIndexTests170.sql | 37 ++++++++++ Test/SqlDom/Only170SyntaxTests.cs | 1 + .../TestScripts/VectorIndexTests170.sql | 46 ++++++++++++ 15 files changed, 366 insertions(+) create mode 100644 SqlScriptDom/Parser/TSql/VectorIndexType.cs create mode 100644 SqlScriptDom/Parser/TSql/VectorIndexTypeHelper.cs create mode 100644 SqlScriptDom/Parser/TSql/VectorMetricType.cs create mode 100644 SqlScriptDom/Parser/TSql/VectorMetricTypeHelper.cs create mode 100644 SqlScriptDom/ScriptDom/SqlServer/ScriptGenerator/SqlScriptGeneratorVisitor.CreateVectorIndexStatement.cs create mode 100644 SqlScriptDom/ScriptDom/SqlServer/ScriptGenerator/SqlScriptGeneratorVisitor.VectorMetricIndexOption.cs create mode 100644 SqlScriptDom/ScriptDom/SqlServer/ScriptGenerator/SqlScriptGeneratorVisitor.VectorTypeIndexOption.cs create mode 100644 Test/SqlDom/Baselines170/VectorIndexTests170.sql create mode 100644 Test/SqlDom/TestScripts/VectorIndexTests170.sql diff --git a/SqlScriptDom/Parser/TSql/Ast.xml b/SqlScriptDom/Parser/TSql/Ast.xml index e5764b2..c8ebb38 100644 --- a/SqlScriptDom/Parser/TSql/Ast.xml +++ b/SqlScriptDom/Parser/TSql/Ast.xml @@ -4607,6 +4607,13 @@ + + + + + + + @@ -4639,6 +4646,14 @@ + + + + + + + + diff --git a/SqlScriptDom/Parser/TSql/CodeGenerationSupporter.cs b/SqlScriptDom/Parser/TSql/CodeGenerationSupporter.cs index 7c00dbe..29696ef 100644 --- a/SqlScriptDom/Parser/TSql/CodeGenerationSupporter.cs +++ b/SqlScriptDom/Parser/TSql/CodeGenerationSupporter.cs @@ -225,6 +225,7 @@ internal static class CodeGenerationSupporter internal const string CompressionDelay = "COMPRESSION_DELAY"; internal const string CompressAllRowGroups = "COMPRESS_ALL_ROW_GROUPS"; internal const string Concat = "CONCAT"; + internal const string Cosine = "COSINE"; internal const string Configuration = "CONFIGURATION"; internal const string ConnectionOptions = "CONNECTION_OPTIONS"; internal const string Contained = "CONTAINED"; @@ -299,6 +300,8 @@ internal static class CodeGenerationSupporter internal const string Dependents = "DEPENDENTS"; internal const string Description = "DESCRIPTION"; internal const string DesiredState = "DESIRED_STATE"; + internal const string DiskANN = "DISKANN"; + internal const string Dot = "DOT"; internal const string Delay = "DELAY"; internal const string DelayedDurability = "DELAYED_DURABILITY"; internal const string DelimitedText = "DELIMITEDTEXT"; @@ -347,6 +350,7 @@ internal static class CodeGenerationSupporter internal const string EnvironmentVariables = "ENVIRONMENT_VARIABLES"; internal const string Equal = "="; internal const string Error = "ERROR"; + internal const string Euclidean = "EUCLIDEAN"; internal const string ErrorBrokerConversations = "ERROR_BROKER_CONVERSATIONS"; internal const string ErrorDataSource = "ERRORFILE_DATA_SOURCE"; internal const string ErrorFile = "ERRORFILE"; @@ -510,6 +514,7 @@ internal static class CodeGenerationSupporter internal const string Json = "JSON"; internal const string JsonArray = "JSON_ARRAY"; internal const string JsonObject = "JSON_OBJECT"; + internal const string Vector = "VECTOR"; internal const string Keep = "KEEP"; internal const string KeepDefaults = "KEEPDEFAULTS"; internal const string KeepFixed = "KEEPFIXED"; @@ -612,6 +617,7 @@ internal static class CodeGenerationSupporter internal const string MessageForwarding = "MESSAGE_FORWARDING"; internal const string MessageForwardSize = "MESSAGE_FORWARD_SIZE"; internal const string MigrationState = "MIGRATION_STATE"; + internal const string Metric = "METRIC"; internal const string Min = "MIN"; internal const string MinGrantPercent = "MIN_GRANT_PERCENT"; internal const string MinCpuPercent = "MIN_CPU_PERCENT"; diff --git a/SqlScriptDom/Parser/TSql/IndexOptionHelper.cs b/SqlScriptDom/Parser/TSql/IndexOptionHelper.cs index d552933..1a0545f 100644 --- a/SqlScriptDom/Parser/TSql/IndexOptionHelper.cs +++ b/SqlScriptDom/Parser/TSql/IndexOptionHelper.cs @@ -43,6 +43,9 @@ private IndexOptionHelper() AddOptionMapping(IndexOptionKind.XmlCompression, CodeGenerationSupporter.XmlCompression, SqlVersionFlags.TSql160AndAbove); + AddOptionMapping(IndexOptionKind.VectorMetric, CodeGenerationSupporter.Metric, SqlVersionFlags.TSql170AndAbove); + AddOptionMapping(IndexOptionKind.VectorType, CodeGenerationSupporter.Type, SqlVersionFlags.TSql170AndAbove); + } internal static readonly IndexOptionHelper Instance = new IndexOptionHelper(); diff --git a/SqlScriptDom/Parser/TSql/IndexOptionKind.cs b/SqlScriptDom/Parser/TSql/IndexOptionKind.cs index 795fa3f..7f2914a 100644 --- a/SqlScriptDom/Parser/TSql/IndexOptionKind.cs +++ b/SqlScriptDom/Parser/TSql/IndexOptionKind.cs @@ -38,6 +38,8 @@ public enum IndexOptionKind WaitAtLowPriority = 21, OptimizeForSequentialKey = 22, XmlCompression = 23, + VectorMetric = 24, + VectorType = 25, } #pragma warning restore 1591 diff --git a/SqlScriptDom/Parser/TSql/TSql170.g b/SqlScriptDom/Parser/TSql/TSql170.g index 031bc61..a0f7863 100644 --- a/SqlScriptDom/Parser/TSql/TSql170.g +++ b/SqlScriptDom/Parser/TSql/TSql170.g @@ -886,6 +886,9 @@ create2005Statements returns [TSqlStatement vResult = null] | {NextTokenMatches(CodeGenerationSupporter.Json)}? vResult=createJsonIndexStatement[null, null] + | + {NextTokenMatches(CodeGenerationSupporter.Vector)}? + vResult=createVectorIndexStatement[null, null] | {NextTokenMatches(CodeGenerationSupporter.Contract)}? vResult=createContractStatement @@ -16848,6 +16851,7 @@ createIndexStatement returns [TSqlStatement vResult = null] vResult=createRelationalIndexStatement[tUnique, isClustered] | vResult=createColumnStoreIndexStatement[tUnique, isClustered] | vResult=createJsonIndexStatement[tUnique, isClustered] + | vResult=createVectorIndexStatement[tUnique, isClustered] ) ) | @@ -17036,6 +17040,50 @@ createJsonIndexStatement [IToken tUnique, bool? isClustered] returns [CreateJson )? ; +createVectorIndexStatement [IToken tUnique, bool? isClustered] returns [CreateVectorIndexStatement vResult = FragmentFactory.CreateFragment()] +{ + Identifier vIdentifier; + SchemaObjectName vSchemaObjectName; + Identifier vVectorColumn; + FileGroupOrPartitionScheme vFileGroupOrPartitionScheme; + + if (tUnique != null) + { + ThrowIncorrectSyntaxErrorException(tUnique); + } + if (isClustered.HasValue) + { + ThrowIncorrectSyntaxErrorException(LT(1)); + } +} + : tVector:Identifier tIndex:Index vIdentifier=identifier + { + Match(tVector, CodeGenerationSupporter.Vector); + vResult.Name = vIdentifier; + } + tOn:On vSchemaObjectName=schemaObjectThreePartName + { + vResult.OnName = vSchemaObjectName; + } + LeftParenthesis vVectorColumn=identifier tRParen:RightParenthesis + { + vResult.VectorColumn = vVectorColumn; + UpdateTokenInfo(vResult, tRParen); + } + ( + // Greedy due to conflict with withCommonTableExpressionsAndXmlNamespaces + options {greedy = true; } : + With + indexOptionList[IndexAffectingStatement.CreateIndex, vResult.IndexOptions, vResult] + )? + ( + On vFileGroupOrPartitionScheme=filegroupOrPartitionScheme + { + vResult.OnFileGroupOrPartitionScheme = vFileGroupOrPartitionScheme; + } + )? + ; + indexKeyColumnList[CreateIndexStatement vParent] { ColumnWithSortOrder vColumnWithSortOrder; @@ -17680,6 +17728,12 @@ indexOption returns [IndexOption vResult = null] | {NextTokenMatches(CodeGenerationSupporter.WaitAtLowPriority)}? vResult=waitAtLowPriorityOption + | + {NextTokenMatches(CodeGenerationSupporter.Metric)}? + vResult=vectorMetricOption + | + {NextTokenMatches(CodeGenerationSupporter.Type)}? + vResult=vectorTypeOption | vResult=indexStateOption ; @@ -27669,6 +27723,26 @@ xmlCompressionOption returns [XmlCompressionOption vResult = FragmentFactory.Cre )? ; +vectorMetricOption returns [VectorMetricIndexOption vResult = FragmentFactory.CreateFragment()] + : tMetric:Identifier EqualsSign tMetricValue:StringLiteral + { + Match(tMetric, CodeGenerationSupporter.Metric); + vResult.OptionKind = IndexOptionKind.VectorMetric; + vResult.MetricType = VectorMetricTypeHelper.Instance.ParseOption(tMetricValue); + UpdateTokenInfo(vResult, tMetricValue); + } + ; + +vectorTypeOption returns [VectorTypeIndexOption vResult = FragmentFactory.CreateFragment()] + : tType:Identifier EqualsSign tTypeValue:StringLiteral + { + Match(tType, CodeGenerationSupporter.Type); + vResult.OptionKind = IndexOptionKind.VectorType; + vResult.VectorType = VectorIndexTypeHelper.Instance.ParseOption(tTypeValue); + UpdateTokenInfo(vResult, tTypeValue); + } + ; + compressionPartitionRange returns [CompressionPartitionRange vResult = FragmentFactory.CreateFragment()] { ScalarExpression vExpression; diff --git a/SqlScriptDom/Parser/TSql/VectorIndexType.cs b/SqlScriptDom/Parser/TSql/VectorIndexType.cs new file mode 100644 index 0000000..8f70a02 --- /dev/null +++ b/SqlScriptDom/Parser/TSql/VectorIndexType.cs @@ -0,0 +1,23 @@ +//------------------------------------------------------------------------------ +// +// Copyright (c) Microsoft Corporation. All rights reserved. +// +//------------------------------------------------------------------------------ + + + +namespace Microsoft.SqlServer.TransactSql.ScriptDom +{ +#pragma warning disable 1591 + + /// + /// The possible values for vector index type + /// + public enum VectorIndexType + { + DiskANN = 0 + } + +#pragma warning restore 1591 +} + \ No newline at end of file diff --git a/SqlScriptDom/Parser/TSql/VectorIndexTypeHelper.cs b/SqlScriptDom/Parser/TSql/VectorIndexTypeHelper.cs new file mode 100644 index 0000000..7cbc0fe --- /dev/null +++ b/SqlScriptDom/Parser/TSql/VectorIndexTypeHelper.cs @@ -0,0 +1,21 @@ +//------------------------------------------------------------------------------ +// +// Copyright (c) Microsoft Corporation. All rights reserved. +// +//------------------------------------------------------------------------------ + + + +namespace Microsoft.SqlServer.TransactSql.ScriptDom +{ + + internal class VectorIndexTypeHelper : OptionsHelper + { + private VectorIndexTypeHelper() + { + AddOptionMapping(VectorIndexType.DiskANN, CodeGenerationSupporter.DiskANN); + } + + public static readonly VectorIndexTypeHelper Instance = new VectorIndexTypeHelper(); + } +} \ No newline at end of file diff --git a/SqlScriptDom/Parser/TSql/VectorMetricType.cs b/SqlScriptDom/Parser/TSql/VectorMetricType.cs new file mode 100644 index 0000000..44b24a8 --- /dev/null +++ b/SqlScriptDom/Parser/TSql/VectorMetricType.cs @@ -0,0 +1,25 @@ +//------------------------------------------------------------------------------ +// +// Copyright (c) Microsoft Corporation. All rights reserved. +// +//------------------------------------------------------------------------------ + + + +namespace Microsoft.SqlServer.TransactSql.ScriptDom +{ +#pragma warning disable 1591 + + /// + /// The possible values for vector index metric + /// + public enum VectorMetricType + { + Cosine = 0, + Dot = 1, + Euclidean = 2 + } + +#pragma warning restore 1591 +} + \ No newline at end of file diff --git a/SqlScriptDom/Parser/TSql/VectorMetricTypeHelper.cs b/SqlScriptDom/Parser/TSql/VectorMetricTypeHelper.cs new file mode 100644 index 0000000..f154655 --- /dev/null +++ b/SqlScriptDom/Parser/TSql/VectorMetricTypeHelper.cs @@ -0,0 +1,23 @@ +//------------------------------------------------------------------------------ +// +// Copyright (c) Microsoft Corporation. All rights reserved. +// +//------------------------------------------------------------------------------ + + + +namespace Microsoft.SqlServer.TransactSql.ScriptDom +{ + + internal class VectorMetricTypeHelper : OptionsHelper + { + private VectorMetricTypeHelper() + { + AddOptionMapping(VectorMetricType.Cosine, CodeGenerationSupporter.Cosine); + AddOptionMapping(VectorMetricType.Dot, CodeGenerationSupporter.Dot); + AddOptionMapping(VectorMetricType.Euclidean, CodeGenerationSupporter.Euclidean); + } + + public static readonly VectorMetricTypeHelper Instance = new VectorMetricTypeHelper(); + } +} \ No newline at end of file diff --git a/SqlScriptDom/ScriptDom/SqlServer/ScriptGenerator/SqlScriptGeneratorVisitor.CreateVectorIndexStatement.cs b/SqlScriptDom/ScriptDom/SqlServer/ScriptGenerator/SqlScriptGeneratorVisitor.CreateVectorIndexStatement.cs new file mode 100644 index 0000000..cb47a2f --- /dev/null +++ b/SqlScriptDom/ScriptDom/SqlServer/ScriptGenerator/SqlScriptGeneratorVisitor.CreateVectorIndexStatement.cs @@ -0,0 +1,48 @@ +//------------------------------------------------------------------------------ +// +// Copyright (c) Microsoft Corporation. All rights reserved. +// +//------------------------------------------------------------------------------ +using System.Collections.Generic; +using Microsoft.SqlServer.TransactSql.ScriptDom; + +namespace Microsoft.SqlServer.TransactSql.ScriptDom.ScriptGenerator +{ + partial class SqlScriptGeneratorVisitor + { + public override void ExplicitVisit(CreateVectorIndexStatement node) + { + GenerateKeyword(TSqlTokenType.Create); + + GenerateSpaceAndIdentifier(CodeGenerationSupporter.Vector); + + GenerateSpaceAndKeyword(TSqlTokenType.Index); + + // name + GenerateSpaceAndFragmentIfNotNull(node.Name); + + NewLineAndIndent(); + GenerateKeyword(TSqlTokenType.On); + GenerateSpaceAndFragmentIfNotNull(node.OnName); + + // Vector column + if (node.VectorColumn != null) + { + GenerateSpace(); + GenerateSymbol(TSqlTokenType.LeftParenthesis); + GenerateFragmentIfNotNull(node.VectorColumn); + GenerateSymbol(TSqlTokenType.RightParenthesis); + } + + GenerateIndexOptions(node.IndexOptions); + + if (node.OnFileGroupOrPartitionScheme != null) + { + NewLineAndIndent(); + GenerateKeyword(TSqlTokenType.On); + + GenerateSpaceAndFragmentIfNotNull(node.OnFileGroupOrPartitionScheme); + } + } + } +} \ No newline at end of file diff --git a/SqlScriptDom/ScriptDom/SqlServer/ScriptGenerator/SqlScriptGeneratorVisitor.VectorMetricIndexOption.cs b/SqlScriptDom/ScriptDom/SqlServer/ScriptGenerator/SqlScriptGeneratorVisitor.VectorMetricIndexOption.cs new file mode 100644 index 0000000..5e8e5bc --- /dev/null +++ b/SqlScriptDom/ScriptDom/SqlServer/ScriptGenerator/SqlScriptGeneratorVisitor.VectorMetricIndexOption.cs @@ -0,0 +1,21 @@ +//------------------------------------------------------------------------------ +// +// Copyright (c) Microsoft Corporation. All rights reserved. +// +//------------------------------------------------------------------------------ + +namespace Microsoft.SqlServer.TransactSql.ScriptDom.ScriptGenerator +{ + partial class SqlScriptGeneratorVisitor + { + public override void ExplicitVisit(VectorMetricIndexOption node) + { + IndexOptionHelper.Instance.GenerateSourceForOption(_writer, node.OptionKind); + GenerateSpaceAndSymbol(TSqlTokenType.EqualsSign); + GenerateSpace(); + GenerateSymbol(TSqlTokenType.SingleQuote); + VectorMetricTypeHelper.Instance.GenerateSourceForOption(_writer, node.MetricType); + GenerateSymbol(TSqlTokenType.SingleQuote); + } + } +} \ No newline at end of file diff --git a/SqlScriptDom/ScriptDom/SqlServer/ScriptGenerator/SqlScriptGeneratorVisitor.VectorTypeIndexOption.cs b/SqlScriptDom/ScriptDom/SqlServer/ScriptGenerator/SqlScriptGeneratorVisitor.VectorTypeIndexOption.cs new file mode 100644 index 0000000..6c7f6d9 --- /dev/null +++ b/SqlScriptDom/ScriptDom/SqlServer/ScriptGenerator/SqlScriptGeneratorVisitor.VectorTypeIndexOption.cs @@ -0,0 +1,21 @@ +//------------------------------------------------------------------------------ +// +// Copyright (c) Microsoft Corporation. All rights reserved. +// +//------------------------------------------------------------------------------ + +namespace Microsoft.SqlServer.TransactSql.ScriptDom.ScriptGenerator +{ + partial class SqlScriptGeneratorVisitor + { + public override void ExplicitVisit(VectorTypeIndexOption node) + { + IndexOptionHelper.Instance.GenerateSourceForOption(_writer, node.OptionKind); + GenerateSpaceAndSymbol(TSqlTokenType.EqualsSign); + GenerateSpace(); + GenerateSymbol(TSqlTokenType.SingleQuote); + VectorIndexTypeHelper.Instance.GenerateSourceForOption(_writer, node.VectorType); + GenerateSymbol(TSqlTokenType.SingleQuote); + } + } +} \ No newline at end of file diff --git a/Test/SqlDom/Baselines170/VectorIndexTests170.sql b/Test/SqlDom/Baselines170/VectorIndexTests170.sql new file mode 100644 index 0000000..81fa86d --- /dev/null +++ b/Test/SqlDom/Baselines170/VectorIndexTests170.sql @@ -0,0 +1,37 @@ +CREATE VECTOR INDEX IX_Vector_Basic + ON dbo.Documents (VectorData); + +CREATE VECTOR INDEX IX_Vector_Cosine + ON dbo.Documents (VectorData) WITH (METRIC = 'cosine'); + +CREATE VECTOR INDEX IX_Vector_Dot + ON dbo.Documents (VectorData) WITH (METRIC = 'dot'); + +CREATE VECTOR INDEX IX_Vector_Euclidean + ON dbo.Documents (VectorData) WITH (METRIC = 'euclidean'); + +CREATE VECTOR INDEX IX_Vector_DiskANN + ON dbo.Documents (VectorData) WITH (TYPE = 'DiskANN'); + +CREATE VECTOR INDEX IX_Vector_Complete + ON dbo.Documents (VectorData) WITH (METRIC = 'cosine', TYPE = 'DiskANN'); + +CREATE VECTOR INDEX IX_Vector_MaxDop + ON dbo.Documents (VectorData) WITH (MAXDOP = 4); + +CREATE VECTOR INDEX IX_Vector_AllOptions + ON dbo.Documents (VectorData) WITH (METRIC = 'dot', TYPE = 'DiskANN', MAXDOP = 8); + +CREATE VECTOR INDEX IX_Vector_Schema + ON MySchema.MyTable (VectorColumn) WITH (METRIC = 'euclidean'); + +CREATE VECTOR INDEX [IX Vector Index] + ON [dbo].[Documents] ([Vector Data]) WITH (METRIC = 'cosine'); + +CREATE VECTOR INDEX IX_Vector_Filegroup + ON dbo.Documents (VectorData) WITH (METRIC = 'cosine') + ON [PRIMARY]; + +CREATE VECTOR INDEX IX_Vector_DefaultFG + ON dbo.Documents (VectorData) WITH (METRIC = 'dot') + ON "default"; \ No newline at end of file diff --git a/Test/SqlDom/Only170SyntaxTests.cs b/Test/SqlDom/Only170SyntaxTests.cs index 654fcdb..29836d6 100644 --- a/Test/SqlDom/Only170SyntaxTests.cs +++ b/Test/SqlDom/Only170SyntaxTests.cs @@ -11,6 +11,7 @@ public partial class SqlDomTests { new ParserTest170("RegexpTVFTests170.sql", nErrors80: 1, nErrors90: 1, nErrors100: 0, nErrors110: 0, nErrors120: 0, nErrors130: 0, nErrors140: 0, nErrors150: 0, nErrors160: 0), new ParserTest170("JsonIndexTests170.sql", nErrors80: 2, nErrors90: 8, nErrors100: 8, nErrors110: 8, nErrors120: 8, nErrors130: 8, nErrors140: 8, nErrors150: 8, nErrors160: 8), + new ParserTest170("VectorIndexTests170.sql", nErrors80: 12, nErrors90: 12, nErrors100: 12, nErrors110: 12, nErrors120: 12, nErrors130: 12, nErrors140: 12, nErrors150: 12, nErrors160: 12), new ParserTest170("AlterDatabaseManualCutoverTests170.sql", nErrors80: 4, nErrors90: 4, nErrors100: 4, nErrors110: 4, nErrors120: 4, nErrors130: 4, nErrors140: 4, nErrors150: 4, nErrors160: 4), new ParserTest170("CreateColumnStoreIndexTests170.sql", nErrors80: 3, nErrors90: 3, nErrors100: 3, nErrors110: 3, nErrors120: 3, nErrors130: 0, nErrors140: 0, nErrors150: 0, nErrors160: 0) }; diff --git a/Test/SqlDom/TestScripts/VectorIndexTests170.sql b/Test/SqlDom/TestScripts/VectorIndexTests170.sql new file mode 100644 index 0000000..d3a1a79 --- /dev/null +++ b/Test/SqlDom/TestScripts/VectorIndexTests170.sql @@ -0,0 +1,46 @@ +-- Basic VECTOR index creation +CREATE VECTOR INDEX IX_Vector_Basic ON dbo.Documents (VectorData); + +-- VECTOR index with METRIC option +CREATE VECTOR INDEX IX_Vector_Cosine ON dbo.Documents (VectorData) +WITH (METRIC = 'cosine'); + +CREATE VECTOR INDEX IX_Vector_Dot ON dbo.Documents (VectorData) +WITH (METRIC = 'dot'); + +CREATE VECTOR INDEX IX_Vector_Euclidean ON dbo.Documents (VectorData) +WITH (METRIC = 'euclidean'); + +-- VECTOR index with TYPE option +CREATE VECTOR INDEX IX_Vector_DiskANN ON dbo.Documents (VectorData) +WITH (TYPE = 'DiskANN'); + +-- VECTOR index with both METRIC and TYPE options +CREATE VECTOR INDEX IX_Vector_Complete ON dbo.Documents (VectorData) +WITH (METRIC = 'cosine', TYPE = 'DiskANN'); + +-- VECTOR index with MAXDOP option +CREATE VECTOR INDEX IX_Vector_MaxDop ON dbo.Documents (VectorData) +WITH (MAXDOP = 4); + +-- VECTOR index with multiple options +CREATE VECTOR INDEX IX_Vector_AllOptions ON dbo.Documents (VectorData) +WITH (METRIC = 'dot', TYPE = 'DiskANN', MAXDOP = 8); + +-- VECTOR index on schema-qualified table +CREATE VECTOR INDEX IX_Vector_Schema ON MySchema.MyTable (VectorColumn) +WITH (METRIC = 'euclidean'); + +-- VECTOR index with quoted identifiers +CREATE VECTOR INDEX [IX Vector Index] ON [dbo].[Documents] ([Vector Data]) +WITH (METRIC = 'cosine'); + +-- VECTOR index with filegroup +CREATE VECTOR INDEX IX_Vector_Filegroup ON dbo.Documents (VectorData) +WITH (METRIC = 'cosine') +ON [PRIMARY]; + +-- VECTOR index with filegroup as string +CREATE VECTOR INDEX IX_Vector_DefaultFG ON dbo.Documents (VectorData) +WITH (METRIC = 'dot') +ON "default"; \ No newline at end of file From 97a1590452041d95ee030aca11a9b9fa145c361e Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 31 Jul 2025 00:02:29 +0000 Subject: [PATCH 3/9] Consolidate vector index options and remove duplicate Vector constant Co-authored-by: llali <17187348+llali@users.noreply.github.com> --- SqlScriptDom/Parser/TSql/Ast.xml | 5 +-- .../Parser/TSql/CodeGenerationSupporter.cs | 1 - SqlScriptDom/Parser/TSql/TSql170.g | 41 ++++++++++--------- ...criptGeneratorVisitor.VectorIndexOption.cs | 36 ++++++++++++++++ ...eneratorVisitor.VectorMetricIndexOption.cs | 21 ---------- ...tGeneratorVisitor.VectorTypeIndexOption.cs | 21 ---------- 6 files changed, 58 insertions(+), 67 deletions(-) create mode 100644 SqlScriptDom/ScriptDom/SqlServer/ScriptGenerator/SqlScriptGeneratorVisitor.VectorIndexOption.cs delete mode 100644 SqlScriptDom/ScriptDom/SqlServer/ScriptGenerator/SqlScriptGeneratorVisitor.VectorMetricIndexOption.cs delete mode 100644 SqlScriptDom/ScriptDom/SqlServer/ScriptGenerator/SqlScriptGeneratorVisitor.VectorTypeIndexOption.cs diff --git a/SqlScriptDom/Parser/TSql/Ast.xml b/SqlScriptDom/Parser/TSql/Ast.xml index c8ebb38..ed6cf2f 100644 --- a/SqlScriptDom/Parser/TSql/Ast.xml +++ b/SqlScriptDom/Parser/TSql/Ast.xml @@ -4646,12 +4646,9 @@ - + - - - diff --git a/SqlScriptDom/Parser/TSql/CodeGenerationSupporter.cs b/SqlScriptDom/Parser/TSql/CodeGenerationSupporter.cs index 29696ef..4deaa86 100644 --- a/SqlScriptDom/Parser/TSql/CodeGenerationSupporter.cs +++ b/SqlScriptDom/Parser/TSql/CodeGenerationSupporter.cs @@ -514,7 +514,6 @@ internal static class CodeGenerationSupporter internal const string Json = "JSON"; internal const string JsonArray = "JSON_ARRAY"; internal const string JsonObject = "JSON_OBJECT"; - internal const string Vector = "VECTOR"; internal const string Keep = "KEEP"; internal const string KeepDefaults = "KEEPDEFAULTS"; internal const string KeepFixed = "KEEPFIXED"; diff --git a/SqlScriptDom/Parser/TSql/TSql170.g b/SqlScriptDom/Parser/TSql/TSql170.g index a0f7863..6948e82 100644 --- a/SqlScriptDom/Parser/TSql/TSql170.g +++ b/SqlScriptDom/Parser/TSql/TSql170.g @@ -17730,10 +17730,10 @@ indexOption returns [IndexOption vResult = null] vResult=waitAtLowPriorityOption | {NextTokenMatches(CodeGenerationSupporter.Metric)}? - vResult=vectorMetricOption + vResult=vectorIndexOption | {NextTokenMatches(CodeGenerationSupporter.Type)}? - vResult=vectorTypeOption + vResult=vectorIndexOption | vResult=indexStateOption ; @@ -27723,24 +27723,25 @@ xmlCompressionOption returns [XmlCompressionOption vResult = FragmentFactory.Cre )? ; -vectorMetricOption returns [VectorMetricIndexOption vResult = FragmentFactory.CreateFragment()] - : tMetric:Identifier EqualsSign tMetricValue:StringLiteral - { - Match(tMetric, CodeGenerationSupporter.Metric); - vResult.OptionKind = IndexOptionKind.VectorMetric; - vResult.MetricType = VectorMetricTypeHelper.Instance.ParseOption(tMetricValue); - UpdateTokenInfo(vResult, tMetricValue); - } - ; - -vectorTypeOption returns [VectorTypeIndexOption vResult = FragmentFactory.CreateFragment()] - : tType:Identifier EqualsSign tTypeValue:StringLiteral - { - Match(tType, CodeGenerationSupporter.Type); - vResult.OptionKind = IndexOptionKind.VectorType; - vResult.VectorType = VectorIndexTypeHelper.Instance.ParseOption(tTypeValue); - UpdateTokenInfo(vResult, tTypeValue); - } +vectorIndexOption returns [VectorIndexOption vResult = FragmentFactory.CreateFragment()] + : + ( + tMetric:Identifier EqualsSign tMetricValue:StringLiteral + { + Match(tMetric, CodeGenerationSupporter.Metric); + vResult.OptionKind = IndexOptionKind.VectorMetric; + vResult.MetricType = VectorMetricTypeHelper.Instance.ParseOption(tMetricValue); + UpdateTokenInfo(vResult, tMetricValue); + } + | + tType:Identifier EqualsSign tTypeValue:StringLiteral + { + Match(tType, CodeGenerationSupporter.Type); + vResult.OptionKind = IndexOptionKind.VectorType; + vResult.VectorType = VectorIndexTypeHelper.Instance.ParseOption(tTypeValue); + UpdateTokenInfo(vResult, tTypeValue); + } + ) ; compressionPartitionRange returns [CompressionPartitionRange vResult = FragmentFactory.CreateFragment()] diff --git a/SqlScriptDom/ScriptDom/SqlServer/ScriptGenerator/SqlScriptGeneratorVisitor.VectorIndexOption.cs b/SqlScriptDom/ScriptDom/SqlServer/ScriptGenerator/SqlScriptGeneratorVisitor.VectorIndexOption.cs new file mode 100644 index 0000000..85a2652 --- /dev/null +++ b/SqlScriptDom/ScriptDom/SqlServer/ScriptGenerator/SqlScriptGeneratorVisitor.VectorIndexOption.cs @@ -0,0 +1,36 @@ +//------------------------------------------------------------------------------ +// +// Copyright (c) Microsoft Corporation. All rights reserved. +// +//------------------------------------------------------------------------------ + +using Microsoft.SqlServer.TransactSql.ScriptDom; +using System.Diagnostics; + +namespace Microsoft.SqlServer.TransactSql.ScriptDom.ScriptGenerator +{ + partial class SqlScriptGeneratorVisitor + { + public override void ExplicitVisit(VectorIndexOption node) + { + switch (node.OptionKind) + { + case IndexOptionKind.VectorMetric: + GenerateKeyword(TSqlTokenType.Identifier, CodeGenerationSupporter.Metric); + GenerateSpaceAndKeyword(TSqlTokenType.EqualsSign); + GenerateSpace(); + VectorMetricTypeHelper.Instance.GenerateSourceFromEnum(this, node.MetricType); + break; + case IndexOptionKind.VectorType: + GenerateKeyword(TSqlTokenType.Identifier, CodeGenerationSupporter.Type); + GenerateSpaceAndKeyword(TSqlTokenType.EqualsSign); + GenerateSpace(); + VectorIndexTypeHelper.Instance.GenerateSourceFromEnum(this, node.VectorType); + break; + default: + Debug.Assert(false, "An unhandled option is encountered."); + break; + } + } + } +} \ No newline at end of file diff --git a/SqlScriptDom/ScriptDom/SqlServer/ScriptGenerator/SqlScriptGeneratorVisitor.VectorMetricIndexOption.cs b/SqlScriptDom/ScriptDom/SqlServer/ScriptGenerator/SqlScriptGeneratorVisitor.VectorMetricIndexOption.cs deleted file mode 100644 index 5e8e5bc..0000000 --- a/SqlScriptDom/ScriptDom/SqlServer/ScriptGenerator/SqlScriptGeneratorVisitor.VectorMetricIndexOption.cs +++ /dev/null @@ -1,21 +0,0 @@ -//------------------------------------------------------------------------------ -// -// Copyright (c) Microsoft Corporation. All rights reserved. -// -//------------------------------------------------------------------------------ - -namespace Microsoft.SqlServer.TransactSql.ScriptDom.ScriptGenerator -{ - partial class SqlScriptGeneratorVisitor - { - public override void ExplicitVisit(VectorMetricIndexOption node) - { - IndexOptionHelper.Instance.GenerateSourceForOption(_writer, node.OptionKind); - GenerateSpaceAndSymbol(TSqlTokenType.EqualsSign); - GenerateSpace(); - GenerateSymbol(TSqlTokenType.SingleQuote); - VectorMetricTypeHelper.Instance.GenerateSourceForOption(_writer, node.MetricType); - GenerateSymbol(TSqlTokenType.SingleQuote); - } - } -} \ No newline at end of file diff --git a/SqlScriptDom/ScriptDom/SqlServer/ScriptGenerator/SqlScriptGeneratorVisitor.VectorTypeIndexOption.cs b/SqlScriptDom/ScriptDom/SqlServer/ScriptGenerator/SqlScriptGeneratorVisitor.VectorTypeIndexOption.cs deleted file mode 100644 index 6c7f6d9..0000000 --- a/SqlScriptDom/ScriptDom/SqlServer/ScriptGenerator/SqlScriptGeneratorVisitor.VectorTypeIndexOption.cs +++ /dev/null @@ -1,21 +0,0 @@ -//------------------------------------------------------------------------------ -// -// Copyright (c) Microsoft Corporation. All rights reserved. -// -//------------------------------------------------------------------------------ - -namespace Microsoft.SqlServer.TransactSql.ScriptDom.ScriptGenerator -{ - partial class SqlScriptGeneratorVisitor - { - public override void ExplicitVisit(VectorTypeIndexOption node) - { - IndexOptionHelper.Instance.GenerateSourceForOption(_writer, node.OptionKind); - GenerateSpaceAndSymbol(TSqlTokenType.EqualsSign); - GenerateSpace(); - GenerateSymbol(TSqlTokenType.SingleQuote); - VectorIndexTypeHelper.Instance.GenerateSourceForOption(_writer, node.VectorType); - GenerateSymbol(TSqlTokenType.SingleQuote); - } - } -} \ No newline at end of file From 0c53470483dff96a988a5a593275f53a8f594caf Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 1 Aug 2025 17:02:13 +0000 Subject: [PATCH 4/9] Revert to original design with separate VectorMetricIndexOption and VectorTypeIndexOption classes Co-authored-by: llali <17187348+llali@users.noreply.github.com> --- SqlScriptDom/Parser/TSql/Ast.xml | 5 ++- SqlScriptDom/Parser/TSql/TSql170.g | 41 +++++++++---------- ...criptGeneratorVisitor.VectorIndexOption.cs | 36 ---------------- ...eneratorVisitor.VectorMetricIndexOption.cs | 21 ++++++++++ ...tGeneratorVisitor.VectorTypeIndexOption.cs | 21 ++++++++++ 5 files changed, 66 insertions(+), 58 deletions(-) delete mode 100644 SqlScriptDom/ScriptDom/SqlServer/ScriptGenerator/SqlScriptGeneratorVisitor.VectorIndexOption.cs create mode 100644 SqlScriptDom/ScriptDom/SqlServer/ScriptGenerator/SqlScriptGeneratorVisitor.VectorMetricIndexOption.cs create mode 100644 SqlScriptDom/ScriptDom/SqlServer/ScriptGenerator/SqlScriptGeneratorVisitor.VectorTypeIndexOption.cs diff --git a/SqlScriptDom/Parser/TSql/Ast.xml b/SqlScriptDom/Parser/TSql/Ast.xml index ed6cf2f..c8ebb38 100644 --- a/SqlScriptDom/Parser/TSql/Ast.xml +++ b/SqlScriptDom/Parser/TSql/Ast.xml @@ -4646,9 +4646,12 @@ - + + + + diff --git a/SqlScriptDom/Parser/TSql/TSql170.g b/SqlScriptDom/Parser/TSql/TSql170.g index 6948e82..a0f7863 100644 --- a/SqlScriptDom/Parser/TSql/TSql170.g +++ b/SqlScriptDom/Parser/TSql/TSql170.g @@ -17730,10 +17730,10 @@ indexOption returns [IndexOption vResult = null] vResult=waitAtLowPriorityOption | {NextTokenMatches(CodeGenerationSupporter.Metric)}? - vResult=vectorIndexOption + vResult=vectorMetricOption | {NextTokenMatches(CodeGenerationSupporter.Type)}? - vResult=vectorIndexOption + vResult=vectorTypeOption | vResult=indexStateOption ; @@ -27723,25 +27723,24 @@ xmlCompressionOption returns [XmlCompressionOption vResult = FragmentFactory.Cre )? ; -vectorIndexOption returns [VectorIndexOption vResult = FragmentFactory.CreateFragment()] - : - ( - tMetric:Identifier EqualsSign tMetricValue:StringLiteral - { - Match(tMetric, CodeGenerationSupporter.Metric); - vResult.OptionKind = IndexOptionKind.VectorMetric; - vResult.MetricType = VectorMetricTypeHelper.Instance.ParseOption(tMetricValue); - UpdateTokenInfo(vResult, tMetricValue); - } - | - tType:Identifier EqualsSign tTypeValue:StringLiteral - { - Match(tType, CodeGenerationSupporter.Type); - vResult.OptionKind = IndexOptionKind.VectorType; - vResult.VectorType = VectorIndexTypeHelper.Instance.ParseOption(tTypeValue); - UpdateTokenInfo(vResult, tTypeValue); - } - ) +vectorMetricOption returns [VectorMetricIndexOption vResult = FragmentFactory.CreateFragment()] + : tMetric:Identifier EqualsSign tMetricValue:StringLiteral + { + Match(tMetric, CodeGenerationSupporter.Metric); + vResult.OptionKind = IndexOptionKind.VectorMetric; + vResult.MetricType = VectorMetricTypeHelper.Instance.ParseOption(tMetricValue); + UpdateTokenInfo(vResult, tMetricValue); + } + ; + +vectorTypeOption returns [VectorTypeIndexOption vResult = FragmentFactory.CreateFragment()] + : tType:Identifier EqualsSign tTypeValue:StringLiteral + { + Match(tType, CodeGenerationSupporter.Type); + vResult.OptionKind = IndexOptionKind.VectorType; + vResult.VectorType = VectorIndexTypeHelper.Instance.ParseOption(tTypeValue); + UpdateTokenInfo(vResult, tTypeValue); + } ; compressionPartitionRange returns [CompressionPartitionRange vResult = FragmentFactory.CreateFragment()] diff --git a/SqlScriptDom/ScriptDom/SqlServer/ScriptGenerator/SqlScriptGeneratorVisitor.VectorIndexOption.cs b/SqlScriptDom/ScriptDom/SqlServer/ScriptGenerator/SqlScriptGeneratorVisitor.VectorIndexOption.cs deleted file mode 100644 index 85a2652..0000000 --- a/SqlScriptDom/ScriptDom/SqlServer/ScriptGenerator/SqlScriptGeneratorVisitor.VectorIndexOption.cs +++ /dev/null @@ -1,36 +0,0 @@ -//------------------------------------------------------------------------------ -// -// Copyright (c) Microsoft Corporation. All rights reserved. -// -//------------------------------------------------------------------------------ - -using Microsoft.SqlServer.TransactSql.ScriptDom; -using System.Diagnostics; - -namespace Microsoft.SqlServer.TransactSql.ScriptDom.ScriptGenerator -{ - partial class SqlScriptGeneratorVisitor - { - public override void ExplicitVisit(VectorIndexOption node) - { - switch (node.OptionKind) - { - case IndexOptionKind.VectorMetric: - GenerateKeyword(TSqlTokenType.Identifier, CodeGenerationSupporter.Metric); - GenerateSpaceAndKeyword(TSqlTokenType.EqualsSign); - GenerateSpace(); - VectorMetricTypeHelper.Instance.GenerateSourceFromEnum(this, node.MetricType); - break; - case IndexOptionKind.VectorType: - GenerateKeyword(TSqlTokenType.Identifier, CodeGenerationSupporter.Type); - GenerateSpaceAndKeyword(TSqlTokenType.EqualsSign); - GenerateSpace(); - VectorIndexTypeHelper.Instance.GenerateSourceFromEnum(this, node.VectorType); - break; - default: - Debug.Assert(false, "An unhandled option is encountered."); - break; - } - } - } -} \ No newline at end of file diff --git a/SqlScriptDom/ScriptDom/SqlServer/ScriptGenerator/SqlScriptGeneratorVisitor.VectorMetricIndexOption.cs b/SqlScriptDom/ScriptDom/SqlServer/ScriptGenerator/SqlScriptGeneratorVisitor.VectorMetricIndexOption.cs new file mode 100644 index 0000000..5e8e5bc --- /dev/null +++ b/SqlScriptDom/ScriptDom/SqlServer/ScriptGenerator/SqlScriptGeneratorVisitor.VectorMetricIndexOption.cs @@ -0,0 +1,21 @@ +//------------------------------------------------------------------------------ +// +// Copyright (c) Microsoft Corporation. All rights reserved. +// +//------------------------------------------------------------------------------ + +namespace Microsoft.SqlServer.TransactSql.ScriptDom.ScriptGenerator +{ + partial class SqlScriptGeneratorVisitor + { + public override void ExplicitVisit(VectorMetricIndexOption node) + { + IndexOptionHelper.Instance.GenerateSourceForOption(_writer, node.OptionKind); + GenerateSpaceAndSymbol(TSqlTokenType.EqualsSign); + GenerateSpace(); + GenerateSymbol(TSqlTokenType.SingleQuote); + VectorMetricTypeHelper.Instance.GenerateSourceForOption(_writer, node.MetricType); + GenerateSymbol(TSqlTokenType.SingleQuote); + } + } +} \ No newline at end of file diff --git a/SqlScriptDom/ScriptDom/SqlServer/ScriptGenerator/SqlScriptGeneratorVisitor.VectorTypeIndexOption.cs b/SqlScriptDom/ScriptDom/SqlServer/ScriptGenerator/SqlScriptGeneratorVisitor.VectorTypeIndexOption.cs new file mode 100644 index 0000000..6c7f6d9 --- /dev/null +++ b/SqlScriptDom/ScriptDom/SqlServer/ScriptGenerator/SqlScriptGeneratorVisitor.VectorTypeIndexOption.cs @@ -0,0 +1,21 @@ +//------------------------------------------------------------------------------ +// +// Copyright (c) Microsoft Corporation. All rights reserved. +// +//------------------------------------------------------------------------------ + +namespace Microsoft.SqlServer.TransactSql.ScriptDom.ScriptGenerator +{ + partial class SqlScriptGeneratorVisitor + { + public override void ExplicitVisit(VectorTypeIndexOption node) + { + IndexOptionHelper.Instance.GenerateSourceForOption(_writer, node.OptionKind); + GenerateSpaceAndSymbol(TSqlTokenType.EqualsSign); + GenerateSpace(); + GenerateSymbol(TSqlTokenType.SingleQuote); + VectorIndexTypeHelper.Instance.GenerateSourceForOption(_writer, node.VectorType); + GenerateSymbol(TSqlTokenType.SingleQuote); + } + } +} \ No newline at end of file From 3c465551799ca9cfd2da35359fcb0a8eb39e064d Mon Sep 17 00:00:00 2001 From: Leila Lali Date: Fri, 1 Aug 2025 15:15:33 -0700 Subject: [PATCH 5/9] fixing the parser --- SqlScriptDom/Parser/TSql/OptionsHelper.cs | 2 +- SqlScriptDom/Parser/TSql/TSql170.g | 9 +++++---- SqlScriptDom/Parser/TSql/VectorIndexTypeHelper.cs | 2 +- SqlScriptDom/Parser/TSql/VectorMetricTypeHelper.cs | 8 +++++--- .../SqlScriptGeneratorVisitor.VectorMetricIndexOption.cs | 2 -- .../SqlScriptGeneratorVisitor.VectorTypeIndexOption.cs | 2 -- Test/SqlDom/Only170SyntaxTests.cs | 2 +- 7 files changed, 13 insertions(+), 14 deletions(-) diff --git a/SqlScriptDom/Parser/TSql/OptionsHelper.cs b/SqlScriptDom/Parser/TSql/OptionsHelper.cs index 284ec4e..7262ab6 100644 --- a/SqlScriptDom/Parser/TSql/OptionsHelper.cs +++ b/SqlScriptDom/Parser/TSql/OptionsHelper.cs @@ -135,7 +135,7 @@ internal SqlVersionFlags MapSqlVersionToSqlVersionFlags(SqlVersion sqlVersion) } } - internal OptionType ParseOption(antlr.IToken token, SqlVersionFlags version) + internal virtual OptionType ParseOption(antlr.IToken token, SqlVersionFlags version) { OptionInfo optionInfo; if (_stringToOptionInfo.TryGetValue(token.getText(), out optionInfo) && diff --git a/SqlScriptDom/Parser/TSql/TSql170.g b/SqlScriptDom/Parser/TSql/TSql170.g index a0f7863..1ef4339 100644 --- a/SqlScriptDom/Parser/TSql/TSql170.g +++ b/SqlScriptDom/Parser/TSql/TSql170.g @@ -27724,22 +27724,23 @@ xmlCompressionOption returns [XmlCompressionOption vResult = FragmentFactory.Cre ; vectorMetricOption returns [VectorMetricIndexOption vResult = FragmentFactory.CreateFragment()] - : tMetric:Identifier EqualsSign tMetricValue:StringLiteral + : tMetric:Identifier EqualsSign tMetricValue:AsciiStringLiteral { Match(tMetric, CodeGenerationSupporter.Metric); vResult.OptionKind = IndexOptionKind.VectorMetric; vResult.MetricType = VectorMetricTypeHelper.Instance.ParseOption(tMetricValue); - UpdateTokenInfo(vResult, tMetricValue); + + UpdateTokenInfo(vResult, tMetric); } ; vectorTypeOption returns [VectorTypeIndexOption vResult = FragmentFactory.CreateFragment()] - : tType:Identifier EqualsSign tTypeValue:StringLiteral + : tType:Identifier EqualsSign tTypeValue:AsciiStringLiteral { Match(tType, CodeGenerationSupporter.Type); vResult.OptionKind = IndexOptionKind.VectorType; vResult.VectorType = VectorIndexTypeHelper.Instance.ParseOption(tTypeValue); - UpdateTokenInfo(vResult, tTypeValue); + UpdateTokenInfo(vResult, tType); } ; diff --git a/SqlScriptDom/Parser/TSql/VectorIndexTypeHelper.cs b/SqlScriptDom/Parser/TSql/VectorIndexTypeHelper.cs index 7cbc0fe..602617f 100644 --- a/SqlScriptDom/Parser/TSql/VectorIndexTypeHelper.cs +++ b/SqlScriptDom/Parser/TSql/VectorIndexTypeHelper.cs @@ -13,7 +13,7 @@ internal class VectorIndexTypeHelper : OptionsHelper { private VectorIndexTypeHelper() { - AddOptionMapping(VectorIndexType.DiskANN, CodeGenerationSupporter.DiskANN); + AddOptionMapping(VectorIndexType.DiskANN, "'" + CodeGenerationSupporter.DiskANN + "'"); } public static readonly VectorIndexTypeHelper Instance = new VectorIndexTypeHelper(); diff --git a/SqlScriptDom/Parser/TSql/VectorMetricTypeHelper.cs b/SqlScriptDom/Parser/TSql/VectorMetricTypeHelper.cs index f154655..6b1479a 100644 --- a/SqlScriptDom/Parser/TSql/VectorMetricTypeHelper.cs +++ b/SqlScriptDom/Parser/TSql/VectorMetricTypeHelper.cs @@ -6,6 +6,8 @@ +using static Microsoft.SqlServer.TransactSql.ScriptDom.SensitivityClassification; + namespace Microsoft.SqlServer.TransactSql.ScriptDom { @@ -13,9 +15,9 @@ internal class VectorMetricTypeHelper : OptionsHelper { private VectorMetricTypeHelper() { - AddOptionMapping(VectorMetricType.Cosine, CodeGenerationSupporter.Cosine); - AddOptionMapping(VectorMetricType.Dot, CodeGenerationSupporter.Dot); - AddOptionMapping(VectorMetricType.Euclidean, CodeGenerationSupporter.Euclidean); + AddOptionMapping(VectorMetricType.Cosine, "'" + CodeGenerationSupporter.Cosine + "'"); + AddOptionMapping(VectorMetricType.Dot, "'" + CodeGenerationSupporter.Dot + "'"); + AddOptionMapping(VectorMetricType.Euclidean, "'" + CodeGenerationSupporter.Euclidean + "'"); } public static readonly VectorMetricTypeHelper Instance = new VectorMetricTypeHelper(); diff --git a/SqlScriptDom/ScriptDom/SqlServer/ScriptGenerator/SqlScriptGeneratorVisitor.VectorMetricIndexOption.cs b/SqlScriptDom/ScriptDom/SqlServer/ScriptGenerator/SqlScriptGeneratorVisitor.VectorMetricIndexOption.cs index 5e8e5bc..9beeeaf 100644 --- a/SqlScriptDom/ScriptDom/SqlServer/ScriptGenerator/SqlScriptGeneratorVisitor.VectorMetricIndexOption.cs +++ b/SqlScriptDom/ScriptDom/SqlServer/ScriptGenerator/SqlScriptGeneratorVisitor.VectorMetricIndexOption.cs @@ -13,9 +13,7 @@ public override void ExplicitVisit(VectorMetricIndexOption node) IndexOptionHelper.Instance.GenerateSourceForOption(_writer, node.OptionKind); GenerateSpaceAndSymbol(TSqlTokenType.EqualsSign); GenerateSpace(); - GenerateSymbol(TSqlTokenType.SingleQuote); VectorMetricTypeHelper.Instance.GenerateSourceForOption(_writer, node.MetricType); - GenerateSymbol(TSqlTokenType.SingleQuote); } } } \ No newline at end of file diff --git a/SqlScriptDom/ScriptDom/SqlServer/ScriptGenerator/SqlScriptGeneratorVisitor.VectorTypeIndexOption.cs b/SqlScriptDom/ScriptDom/SqlServer/ScriptGenerator/SqlScriptGeneratorVisitor.VectorTypeIndexOption.cs index 6c7f6d9..85bcadb 100644 --- a/SqlScriptDom/ScriptDom/SqlServer/ScriptGenerator/SqlScriptGeneratorVisitor.VectorTypeIndexOption.cs +++ b/SqlScriptDom/ScriptDom/SqlServer/ScriptGenerator/SqlScriptGeneratorVisitor.VectorTypeIndexOption.cs @@ -13,9 +13,7 @@ public override void ExplicitVisit(VectorTypeIndexOption node) IndexOptionHelper.Instance.GenerateSourceForOption(_writer, node.OptionKind); GenerateSpaceAndSymbol(TSqlTokenType.EqualsSign); GenerateSpace(); - GenerateSymbol(TSqlTokenType.SingleQuote); VectorIndexTypeHelper.Instance.GenerateSourceForOption(_writer, node.VectorType); - GenerateSymbol(TSqlTokenType.SingleQuote); } } } \ No newline at end of file diff --git a/Test/SqlDom/Only170SyntaxTests.cs b/Test/SqlDom/Only170SyntaxTests.cs index 29836d6..05ade64 100644 --- a/Test/SqlDom/Only170SyntaxTests.cs +++ b/Test/SqlDom/Only170SyntaxTests.cs @@ -11,7 +11,7 @@ public partial class SqlDomTests { new ParserTest170("RegexpTVFTests170.sql", nErrors80: 1, nErrors90: 1, nErrors100: 0, nErrors110: 0, nErrors120: 0, nErrors130: 0, nErrors140: 0, nErrors150: 0, nErrors160: 0), new ParserTest170("JsonIndexTests170.sql", nErrors80: 2, nErrors90: 8, nErrors100: 8, nErrors110: 8, nErrors120: 8, nErrors130: 8, nErrors140: 8, nErrors150: 8, nErrors160: 8), - new ParserTest170("VectorIndexTests170.sql", nErrors80: 12, nErrors90: 12, nErrors100: 12, nErrors110: 12, nErrors120: 12, nErrors130: 12, nErrors140: 12, nErrors150: 12, nErrors160: 12), + new ParserTest170("VectorIndexTests170.sql", nErrors80: 2, nErrors90: 12, nErrors100: 12, nErrors110: 12, nErrors120: 12, nErrors130: 12, nErrors140: 12, nErrors150: 12, nErrors160: 12), new ParserTest170("AlterDatabaseManualCutoverTests170.sql", nErrors80: 4, nErrors90: 4, nErrors100: 4, nErrors110: 4, nErrors120: 4, nErrors130: 4, nErrors140: 4, nErrors150: 4, nErrors160: 4), new ParserTest170("CreateColumnStoreIndexTests170.sql", nErrors80: 3, nErrors90: 3, nErrors100: 3, nErrors110: 3, nErrors120: 3, nErrors130: 0, nErrors140: 0, nErrors150: 0, nErrors160: 0) }; From df44c721f792cfe42088676553674f8be98dd218 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 1 Aug 2025 22:24:11 +0000 Subject: [PATCH 6/9] Update vector index option values to uppercase in baseline test file Co-authored-by: llali <17187348+llali@users.noreply.github.com> --- .../Baselines170/VectorIndexTests170.sql | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/Test/SqlDom/Baselines170/VectorIndexTests170.sql b/Test/SqlDom/Baselines170/VectorIndexTests170.sql index 81fa86d..729b3ab 100644 --- a/Test/SqlDom/Baselines170/VectorIndexTests170.sql +++ b/Test/SqlDom/Baselines170/VectorIndexTests170.sql @@ -2,36 +2,36 @@ CREATE VECTOR INDEX IX_Vector_Basic ON dbo.Documents (VectorData); CREATE VECTOR INDEX IX_Vector_Cosine - ON dbo.Documents (VectorData) WITH (METRIC = 'cosine'); + ON dbo.Documents (VectorData) WITH (METRIC = 'COSINE'); CREATE VECTOR INDEX IX_Vector_Dot - ON dbo.Documents (VectorData) WITH (METRIC = 'dot'); + ON dbo.Documents (VectorData) WITH (METRIC = 'DOT'); CREATE VECTOR INDEX IX_Vector_Euclidean - ON dbo.Documents (VectorData) WITH (METRIC = 'euclidean'); + ON dbo.Documents (VectorData) WITH (METRIC = 'EUCLIDEAN'); CREATE VECTOR INDEX IX_Vector_DiskANN - ON dbo.Documents (VectorData) WITH (TYPE = 'DiskANN'); + ON dbo.Documents (VectorData) WITH (TYPE = 'DISKANN'); CREATE VECTOR INDEX IX_Vector_Complete - ON dbo.Documents (VectorData) WITH (METRIC = 'cosine', TYPE = 'DiskANN'); + ON dbo.Documents (VectorData) WITH (METRIC = 'COSINE', TYPE = 'DISKANN'); CREATE VECTOR INDEX IX_Vector_MaxDop ON dbo.Documents (VectorData) WITH (MAXDOP = 4); CREATE VECTOR INDEX IX_Vector_AllOptions - ON dbo.Documents (VectorData) WITH (METRIC = 'dot', TYPE = 'DiskANN', MAXDOP = 8); + ON dbo.Documents (VectorData) WITH (METRIC = 'DOT', TYPE = 'DISKANN', MAXDOP = 8); CREATE VECTOR INDEX IX_Vector_Schema - ON MySchema.MyTable (VectorColumn) WITH (METRIC = 'euclidean'); + ON MySchema.MyTable (VectorColumn) WITH (METRIC = 'EUCLIDEAN'); CREATE VECTOR INDEX [IX Vector Index] - ON [dbo].[Documents] ([Vector Data]) WITH (METRIC = 'cosine'); + ON [dbo].[Documents] ([Vector Data]) WITH (METRIC = 'COSINE'); CREATE VECTOR INDEX IX_Vector_Filegroup - ON dbo.Documents (VectorData) WITH (METRIC = 'cosine') + ON dbo.Documents (VectorData) WITH (METRIC = 'COSINE') ON [PRIMARY]; CREATE VECTOR INDEX IX_Vector_DefaultFG - ON dbo.Documents (VectorData) WITH (METRIC = 'dot') + ON dbo.Documents (VectorData) WITH (METRIC = 'DOT') ON "default"; \ No newline at end of file From e0432d2f1f286e0ebd2af837a58e15f7027b020c Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 1 Aug 2025 22:56:09 +0000 Subject: [PATCH 7/9] Add comprehensive vector index error test cases to ParserErrorsTests Co-authored-by: zijchen <13544267+zijchen@users.noreply.github.com> --- Test/SqlDom/ParserErrorsTests.cs | 93 ++++++++++++++++++++++++++++++++ 1 file changed, 93 insertions(+) diff --git a/Test/SqlDom/ParserErrorsTests.cs b/Test/SqlDom/ParserErrorsTests.cs index f7632b5..24e78e4 100644 --- a/Test/SqlDom/ParserErrorsTests.cs +++ b/Test/SqlDom/ParserErrorsTests.cs @@ -6950,5 +6950,98 @@ Description NVARCHAR(200) "; ParserTestUtils.ErrorTestFabricDW(identityColumnSyntax2, new ParserErrorInfo(identityColumnSyntax2.IndexOf("IDENTITY(") + 8, "SQL46010", "(")); } + + /// + /// Negative tests for VECTOR INDEX syntax + /// + [TestMethod] + [Priority(0)] + [SqlStudioTestCategory(Category.UnitTest)] + public void VectorIndexNegativeTests() + { + // Missing VECTOR keyword + ParserTestUtils.ErrorTest170("CREATE INDEX IX_Test ON dbo.Documents (VectorData)", + new ParserErrorInfo(7, "SQL46010", "INDEX")); + + // Missing INDEX keyword + ParserTestUtils.ErrorTest170("CREATE VECTOR IX_Test ON dbo.Documents (VectorData)", + new ParserErrorInfo(14, "SQL46010", "IX_Test")); + + // Missing table name + ParserTestUtils.ErrorTest170("CREATE VECTOR INDEX IX_Test ON (VectorData)", + new ParserErrorInfo(35, "SQL46010", "(")); + + // Missing column specification + ParserTestUtils.ErrorTest170("CREATE VECTOR INDEX IX_Test ON dbo.Documents", + new ParserErrorInfo(45, "SQL46029", "")); + + // Empty column list + ParserTestUtils.ErrorTest170("CREATE VECTOR INDEX IX_Test ON dbo.Documents ()", + new ParserErrorInfo(46, "SQL46010", ")")); + + // Multiple columns (not supported for vector indexes) + ParserTestUtils.ErrorTest170("CREATE VECTOR INDEX IX_Test ON dbo.Documents (VectorData, OtherColumn)", + new ParserErrorInfo(57, "SQL46010", ",")); + + // Invalid metric value + ParserTestUtils.ErrorTest170("CREATE VECTOR INDEX IX_Test ON dbo.Documents (VectorData) WITH (METRIC = 'invalid')", + new ParserErrorInfo(76, "SQL46010", "'invalid'")); + + // Invalid type value + ParserTestUtils.ErrorTest170("CREATE VECTOR INDEX IX_Test ON dbo.Documents (VectorData) WITH (TYPE = 'invalid')", + new ParserErrorInfo(74, "SQL46010", "'invalid'")); + + // Missing option value + ParserTestUtils.ErrorTest170("CREATE VECTOR INDEX IX_Test ON dbo.Documents (VectorData) WITH (METRIC = )", + new ParserErrorInfo(76, "SQL46010", ")")); + + // Empty option value + ParserTestUtils.ErrorTest170("CREATE VECTOR INDEX IX_Test ON dbo.Documents (VectorData) WITH (METRIC = '')", + new ParserErrorInfo(76, "SQL46063", "METRIC")); + + // Duplicate METRIC options + ParserTestUtils.ErrorTest170("CREATE VECTOR INDEX IX_Test ON dbo.Documents (VectorData) WITH (METRIC = 'cosine', METRIC = 'dot')", + new ParserErrorInfo(80, "SQL46049", "METRIC")); + + // Duplicate TYPE options + ParserTestUtils.ErrorTest170("CREATE VECTOR INDEX IX_Test ON dbo.Documents (VectorData) WITH (TYPE = 'DiskANN', TYPE = 'DiskANN')", + new ParserErrorInfo(78, "SQL46049", "TYPE")); + + // Missing WITH keyword when options are present + ParserTestUtils.ErrorTest170("CREATE VECTOR INDEX IX_Test ON dbo.Documents (VectorData) (METRIC = 'cosine')", + new ParserErrorInfo(63, "SQL46010", "(")); + + // Missing parentheses around options + ParserTestUtils.ErrorTest170("CREATE VECTOR INDEX IX_Test ON dbo.Documents (VectorData) WITH METRIC = 'cosine'", + new ParserErrorInfo(68, "SQL46010", "METRIC")); + + // Invalid option name + ParserTestUtils.ErrorTest170("CREATE VECTOR INDEX IX_Test ON dbo.Documents (VectorData) WITH (INVALID_OPTION = 'value')", + new ParserErrorInfo(68, "SQL46010", "INVALID_OPTION")); + + // Metric value without quotes + ParserTestUtils.ErrorTest170("CREATE VECTOR INDEX IX_Test ON dbo.Documents (VectorData) WITH (METRIC = cosine)", + new ParserErrorInfo(76, "SQL46010", "cosine")); + + // Type value without quotes + ParserTestUtils.ErrorTest170("CREATE VECTOR INDEX IX_Test ON dbo.Documents (VectorData) WITH (TYPE = DiskANN)", + new ParserErrorInfo(74, "SQL46010", "DiskANN")); + + // MAXDOP with invalid value + ParserTestUtils.ErrorTest170("CREATE VECTOR INDEX IX_Test ON dbo.Documents (VectorData) WITH (MAXDOP = 'invalid')", + new ParserErrorInfo(76, "SQL46010", "'invalid'")); + + // MAXDOP with negative value + ParserTestUtils.ErrorTest170("CREATE VECTOR INDEX IX_Test ON dbo.Documents (VectorData) WITH (MAXDOP = -1)", + new ParserErrorInfo(76, "SQL46010", "-")); + + // Missing equals sign in option + ParserTestUtils.ErrorTest170("CREATE VECTOR INDEX IX_Test ON dbo.Documents (VectorData) WITH (METRIC 'cosine')", + new ParserErrorInfo(74, "SQL46010", "'cosine'")); + + // Incomplete WITH clause + ParserTestUtils.ErrorTest170("CREATE VECTOR INDEX IX_Test ON dbo.Documents (VectorData) WITH", + new ParserErrorInfo(63, "SQL46029", "WITH")); + } } } From 849df2937a133c23df137fc84a7f51a1be02bd69 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 4 Aug 2025 20:51:36 +0000 Subject: [PATCH 8/9] Remove invalid negative test case for vector index - CREATE INDEX syntax is valid for regular indexes Co-authored-by: zijchen <13544267+zijchen@users.noreply.github.com> --- Test/SqlDom/ParserErrorsTests.cs | 4 ---- 1 file changed, 4 deletions(-) diff --git a/Test/SqlDom/ParserErrorsTests.cs b/Test/SqlDom/ParserErrorsTests.cs index 24e78e4..f296064 100644 --- a/Test/SqlDom/ParserErrorsTests.cs +++ b/Test/SqlDom/ParserErrorsTests.cs @@ -6959,10 +6959,6 @@ Description NVARCHAR(200) [SqlStudioTestCategory(Category.UnitTest)] public void VectorIndexNegativeTests() { - // Missing VECTOR keyword - ParserTestUtils.ErrorTest170("CREATE INDEX IX_Test ON dbo.Documents (VectorData)", - new ParserErrorInfo(7, "SQL46010", "INDEX")); - // Missing INDEX keyword ParserTestUtils.ErrorTest170("CREATE VECTOR IX_Test ON dbo.Documents (VectorData)", new ParserErrorInfo(14, "SQL46010", "IX_Test")); From ab0b17bd60a4de624807e1e362e6156423418609 Mon Sep 17 00:00:00 2001 From: Leila Lali Date: Mon, 4 Aug 2025 14:02:55 -0700 Subject: [PATCH 9/9] fixing the test --- Test/SqlDom/ParserErrorsTests.cs | 42 +++++++++++++------------------- 1 file changed, 17 insertions(+), 25 deletions(-) diff --git a/Test/SqlDom/ParserErrorsTests.cs b/Test/SqlDom/ParserErrorsTests.cs index f296064..23d2be1 100644 --- a/Test/SqlDom/ParserErrorsTests.cs +++ b/Test/SqlDom/ParserErrorsTests.cs @@ -6961,15 +6961,15 @@ public void VectorIndexNegativeTests() { // Missing INDEX keyword ParserTestUtils.ErrorTest170("CREATE VECTOR IX_Test ON dbo.Documents (VectorData)", - new ParserErrorInfo(14, "SQL46010", "IX_Test")); + new ParserErrorInfo(7, "SQL46010", "VECTOR")); // Missing table name ParserTestUtils.ErrorTest170("CREATE VECTOR INDEX IX_Test ON (VectorData)", - new ParserErrorInfo(35, "SQL46010", "(")); + new ParserErrorInfo(31, "SQL46010", "(")); // Missing column specification ParserTestUtils.ErrorTest170("CREATE VECTOR INDEX IX_Test ON dbo.Documents", - new ParserErrorInfo(45, "SQL46029", "")); + new ParserErrorInfo(44, "SQL46029", "")); // Empty column list ParserTestUtils.ErrorTest170("CREATE VECTOR INDEX IX_Test ON dbo.Documents ()", @@ -6977,67 +6977,59 @@ public void VectorIndexNegativeTests() // Multiple columns (not supported for vector indexes) ParserTestUtils.ErrorTest170("CREATE VECTOR INDEX IX_Test ON dbo.Documents (VectorData, OtherColumn)", - new ParserErrorInfo(57, "SQL46010", ",")); + new ParserErrorInfo(56, "SQL46010", ",")); // Invalid metric value ParserTestUtils.ErrorTest170("CREATE VECTOR INDEX IX_Test ON dbo.Documents (VectorData) WITH (METRIC = 'invalid')", - new ParserErrorInfo(76, "SQL46010", "'invalid'")); + new ParserErrorInfo(73, "SQL46010", "'invalid'")); // Invalid type value ParserTestUtils.ErrorTest170("CREATE VECTOR INDEX IX_Test ON dbo.Documents (VectorData) WITH (TYPE = 'invalid')", - new ParserErrorInfo(74, "SQL46010", "'invalid'")); + new ParserErrorInfo(71, "SQL46010", "'invalid'")); // Missing option value ParserTestUtils.ErrorTest170("CREATE VECTOR INDEX IX_Test ON dbo.Documents (VectorData) WITH (METRIC = )", - new ParserErrorInfo(76, "SQL46010", ")")); + new ParserErrorInfo(73, "SQL46010", ")")); // Empty option value ParserTestUtils.ErrorTest170("CREATE VECTOR INDEX IX_Test ON dbo.Documents (VectorData) WITH (METRIC = '')", - new ParserErrorInfo(76, "SQL46063", "METRIC")); - - // Duplicate METRIC options - ParserTestUtils.ErrorTest170("CREATE VECTOR INDEX IX_Test ON dbo.Documents (VectorData) WITH (METRIC = 'cosine', METRIC = 'dot')", - new ParserErrorInfo(80, "SQL46049", "METRIC")); - - // Duplicate TYPE options - ParserTestUtils.ErrorTest170("CREATE VECTOR INDEX IX_Test ON dbo.Documents (VectorData) WITH (TYPE = 'DiskANN', TYPE = 'DiskANN')", - new ParserErrorInfo(78, "SQL46049", "TYPE")); + new ParserErrorInfo(73, "SQL46010", "''")); // Missing WITH keyword when options are present ParserTestUtils.ErrorTest170("CREATE VECTOR INDEX IX_Test ON dbo.Documents (VectorData) (METRIC = 'cosine')", - new ParserErrorInfo(63, "SQL46010", "(")); + new ParserErrorInfo(59, "SQL46010", "METRIC")); // Missing parentheses around options ParserTestUtils.ErrorTest170("CREATE VECTOR INDEX IX_Test ON dbo.Documents (VectorData) WITH METRIC = 'cosine'", - new ParserErrorInfo(68, "SQL46010", "METRIC")); + new ParserErrorInfo(63, "SQL46010", "METRIC")); // Invalid option name ParserTestUtils.ErrorTest170("CREATE VECTOR INDEX IX_Test ON dbo.Documents (VectorData) WITH (INVALID_OPTION = 'value')", - new ParserErrorInfo(68, "SQL46010", "INVALID_OPTION")); + new ParserErrorInfo(64, "SQL46010", "INVALID_OPTION")); // Metric value without quotes ParserTestUtils.ErrorTest170("CREATE VECTOR INDEX IX_Test ON dbo.Documents (VectorData) WITH (METRIC = cosine)", - new ParserErrorInfo(76, "SQL46010", "cosine")); + new ParserErrorInfo(73, "SQL46010", "cosine")); // Type value without quotes ParserTestUtils.ErrorTest170("CREATE VECTOR INDEX IX_Test ON dbo.Documents (VectorData) WITH (TYPE = DiskANN)", - new ParserErrorInfo(74, "SQL46010", "DiskANN")); + new ParserErrorInfo(71, "SQL46010", "DiskANN")); // MAXDOP with invalid value ParserTestUtils.ErrorTest170("CREATE VECTOR INDEX IX_Test ON dbo.Documents (VectorData) WITH (MAXDOP = 'invalid')", - new ParserErrorInfo(76, "SQL46010", "'invalid'")); + new ParserErrorInfo(73, "SQL46010", "'invalid'")); // MAXDOP with negative value ParserTestUtils.ErrorTest170("CREATE VECTOR INDEX IX_Test ON dbo.Documents (VectorData) WITH (MAXDOP = -1)", - new ParserErrorInfo(76, "SQL46010", "-")); + new ParserErrorInfo(73, "SQL46010", "-")); // Missing equals sign in option ParserTestUtils.ErrorTest170("CREATE VECTOR INDEX IX_Test ON dbo.Documents (VectorData) WITH (METRIC 'cosine')", - new ParserErrorInfo(74, "SQL46010", "'cosine'")); + new ParserErrorInfo(64, "SQL46010", "METRIC")); // Incomplete WITH clause ParserTestUtils.ErrorTest170("CREATE VECTOR INDEX IX_Test ON dbo.Documents (VectorData) WITH", - new ParserErrorInfo(63, "SQL46029", "WITH")); + new ParserErrorInfo(58, "SQL46010", "WITH")); } } }