Add support for Vector index syntax in T-SQL parser with separate index option classes

SqlScriptDom/Parser/TSql/Ast.xml

@@ -4607,6 +4607,13 @@
     <Member Name="ForJsonPaths" Type="StringLiteral" Collection="true" Summary="The JSON paths specified in the FOR clause. Optional may have zero elements."/>
     <InheritedMember Name="IndexOptions" ContainerClass="IndexStatement" />
   </Class>
+  <Class Name="CreateVectorIndexStatement" Base="IndexStatement" Summary="Represents the create vector index statement.">
+    <InheritedMember Name="Name" ContainerClass="IndexStatement" />
+    <InheritedMember Name="OnName" ContainerClass="IndexStatement" />
+    <Member Name="VectorColumn" Type="Identifier" Summary="The vector column for the index."/>
+    <InheritedMember Name="IndexOptions" ContainerClass="IndexStatement" />
+    <Member Name="OnFileGroupOrPartitionScheme" Type="FileGroupOrPartitionScheme" Summary="The filegroup or partition scheme. Might be null."/>
+  </Class>
   <Class Name="WindowFrameClause" Summary="Represents the specification of window bounds for windowing aggregates.">
     <Member Name="Top" Type="WindowDelimiter" Summary="Top boundary of the window."/>
     <Member Name="Bottom" Type="WindowDelimiter" Summary="Bottom boundary of the window. Optional may be null."/>
@@ -4639,6 +4646,14 @@
     <Member Name="Expression" Type="ScalarExpression" Summary="Can be an integer in the range 0 - 10080."/>
     <Member Name="TimeUnit" Type="CompressionDelayTimeUnit" GenerateUpdatePositionInfoCall="false" Summary="The optional unit of measure for the compression delay (it can be MINUTE or MINUTES)."/>
   </Class>
+  <Class Name="VectorMetricIndexOption" Base="IndexOption" Summary="Represents the vector metric index option.">
+    <InheritedClass Name="IndexOption" />
+    <Member Name="MetricType" Type="VectorMetricType" GenerateUpdatePositionInfoCall="false" Summary="The vector metric type (cosine, dot, euclidean)."/>
+  </Class>
+  <Class Name="VectorTypeIndexOption" Base="IndexOption" Summary="Represents the vector type index option.">
+    <InheritedClass Name="IndexOption" />
+    <Member Name="VectorType" Type="VectorIndexType" GenerateUpdatePositionInfoCall="false" Summary="The vector index type (DiskANN)."/>
+  </Class>
   <Class Name="ExternalLibraryStatement" Abstract="true" Base="TSqlStatement" Summary="Represents common parts in create and alter external library statements.">
     <Member Name="Name" Type="Identifier" Summary="The name of the external library."/>
     <Member Name="Language" Type="StringLiteral" Summary="The language of this external library."/>

SqlScriptDom/Parser/TSql/CodeGenerationSupporter.cs

@@ -225,6 +225,7 @@ internal static class CodeGenerationSupporter
         internal const string CompressionDelay = "COMPRESSION_DELAY";
         internal const string CompressAllRowGroups = "COMPRESS_ALL_ROW_GROUPS";
         internal const string Concat = "CONCAT";
+        internal const string Cosine = "COSINE";
         internal const string Configuration = "CONFIGURATION";
         internal const string ConnectionOptions = "CONNECTION_OPTIONS";
         internal const string Contained = "CONTAINED";
@@ -299,6 +300,8 @@ internal static class CodeGenerationSupporter
         internal const string Dependents = "DEPENDENTS";
         internal const string Description = "DESCRIPTION";
         internal const string DesiredState = "DESIRED_STATE";
+        internal const string DiskANN = "DISKANN";
+        internal const string Dot = "DOT";
         internal const string Delay = "DELAY";
         internal const string DelayedDurability = "DELAYED_DURABILITY";
         internal const string DelimitedText = "DELIMITEDTEXT";
@@ -347,6 +350,7 @@ internal static class CodeGenerationSupporter
         internal const string EnvironmentVariables = "ENVIRONMENT_VARIABLES";
         internal const string Equal = "=";
         internal const string Error = "ERROR";
+        internal const string Euclidean = "EUCLIDEAN";
         internal const string ErrorBrokerConversations = "ERROR_BROKER_CONVERSATIONS";
         internal const string ErrorDataSource = "ERRORFILE_DATA_SOURCE";
         internal const string ErrorFile = "ERRORFILE";
@@ -612,6 +616,7 @@ internal static class CodeGenerationSupporter
         internal const string MessageForwarding = "MESSAGE_FORWARDING";
         internal const string MessageForwardSize = "MESSAGE_FORWARD_SIZE";
         internal const string MigrationState = "MIGRATION_STATE";
+        internal const string Metric = "METRIC";
         internal const string Min = "MIN";
         internal const string MinGrantPercent = "MIN_GRANT_PERCENT";
         internal const string MinCpuPercent = "MIN_CPU_PERCENT";

SqlScriptDom/Parser/TSql/IndexOptionHelper.cs

@@ -43,6 +43,9 @@ private IndexOptionHelper()
 
             AddOptionMapping(IndexOptionKind.XmlCompression, CodeGenerationSupporter.XmlCompression, SqlVersionFlags.TSql160AndAbove);
 
+            AddOptionMapping(IndexOptionKind.VectorMetric, CodeGenerationSupporter.Metric, SqlVersionFlags.TSql170AndAbove);
+            AddOptionMapping(IndexOptionKind.VectorType, CodeGenerationSupporter.Type, SqlVersionFlags.TSql170AndAbove);
+
         }
 
         internal static readonly IndexOptionHelper Instance = new IndexOptionHelper();

SqlScriptDom/Parser/TSql/IndexOptionKind.cs

@@ -38,6 +38,8 @@ public enum IndexOptionKind
         WaitAtLowPriority   = 21,
         OptimizeForSequentialKey = 22,
         XmlCompression      = 23,
+        VectorMetric        = 24,
+        VectorType          = 25,
     }
 
 #pragma warning restore 1591

SqlScriptDom/Parser/TSql/OptionsHelper.cs

@@ -135,7 +135,7 @@ internal SqlVersionFlags MapSqlVersionToSqlVersionFlags(SqlVersion sqlVersion)
             }
         }
 
-        internal OptionType ParseOption(antlr.IToken token, SqlVersionFlags version)
+        internal virtual OptionType ParseOption(antlr.IToken token, SqlVersionFlags version)
         {
             OptionInfo optionInfo;
             if (_stringToOptionInfo.TryGetValue(token.getText(), out optionInfo) &&

SqlScriptDom/Parser/TSql/TSql170.g

@@ -886,6 +886,9 @@ create2005Statements returns [TSqlStatement vResult = null]
         |
             {NextTokenMatches(CodeGenerationSupporter.Json)}?
             vResult=createJsonIndexStatement[null, null]
+        |
+            {NextTokenMatches(CodeGenerationSupporter.Vector)}?
+            vResult=createVectorIndexStatement[null, null]
         |
             {NextTokenMatches(CodeGenerationSupporter.Contract)}?
             vResult=createContractStatement
@@ -16848,6 +16851,7 @@ createIndexStatement returns [TSqlStatement vResult = null]
                     vResult=createRelationalIndexStatement[tUnique, isClustered]
                     | vResult=createColumnStoreIndexStatement[tUnique, isClustered]
                     | vResult=createJsonIndexStatement[tUnique, isClustered]
+                    | vResult=createVectorIndexStatement[tUnique, isClustered]
                 )
             )
         |
@@ -17036,6 +17040,50 @@ createJsonIndexStatement [IToken tUnique, bool? isClustered] returns [CreateJson
     )?
     ;
 
+createVectorIndexStatement [IToken tUnique, bool? isClustered] returns [CreateVectorIndexStatement vResult = FragmentFactory.CreateFragment<CreateVectorIndexStatement>()]
+{
+    Identifier vIdentifier;
+    SchemaObjectName vSchemaObjectName;
+    Identifier vVectorColumn;
+    FileGroupOrPartitionScheme vFileGroupOrPartitionScheme;
+
+    if (tUnique != null)
+    {
+        ThrowIncorrectSyntaxErrorException(tUnique);
+    }
+    if (isClustered.HasValue)
+    {
+        ThrowIncorrectSyntaxErrorException(LT(1));
+    }
+}
+    : tVector:Identifier tIndex:Index vIdentifier=identifier
+    {
+        Match(tVector, CodeGenerationSupporter.Vector);
+        vResult.Name = vIdentifier;
+    }
+    tOn:On vSchemaObjectName=schemaObjectThreePartName
+    {
+        vResult.OnName = vSchemaObjectName;
+    }
+    LeftParenthesis vVectorColumn=identifier tRParen:RightParenthesis
+    {
+        vResult.VectorColumn = vVectorColumn;
+        UpdateTokenInfo(vResult, tRParen);
+    }
+    (
+        // Greedy due to conflict with withCommonTableExpressionsAndXmlNamespaces
+        options {greedy = true; } :
+        With
+        indexOptionList[IndexAffectingStatement.CreateIndex, vResult.IndexOptions, vResult]
+    )?
+    (
+        On vFileGroupOrPartitionScheme=filegroupOrPartitionScheme
+        {
+            vResult.OnFileGroupOrPartitionScheme = vFileGroupOrPartitionScheme;
+        }
+    )?
+    ;
+
 indexKeyColumnList[CreateIndexStatement vParent]
 {
     ColumnWithSortOrder vColumnWithSortOrder;
@@ -17680,6 +17728,12 @@ indexOption returns [IndexOption vResult = null]
     |
         {NextTokenMatches(CodeGenerationSupporter.WaitAtLowPriority)}?
         vResult=waitAtLowPriorityOption
+    |
+        {NextTokenMatches(CodeGenerationSupporter.Metric)}?
+        vResult=vectorMetricOption
+    |
+        {NextTokenMatches(CodeGenerationSupporter.Type)}?
+        vResult=vectorTypeOption
     |
         vResult=indexStateOption
     ;
@@ -27669,6 +27723,27 @@ xmlCompressionOption returns [XmlCompressionOption vResult = FragmentFactory.Cre
         )?
     ;
 
+vectorMetricOption returns [VectorMetricIndexOption vResult = FragmentFactory.CreateFragment<VectorMetricIndexOption>()]
+    : tMetric:Identifier EqualsSign tMetricValue:AsciiStringLiteral
+        {
+            Match(tMetric, CodeGenerationSupporter.Metric);
+            vResult.OptionKind = IndexOptionKind.VectorMetric;
+            vResult.MetricType = VectorMetricTypeHelper.Instance.ParseOption(tMetricValue);
+
+            UpdateTokenInfo(vResult, tMetric);
+        }
+    ;
+
+vectorTypeOption returns [VectorTypeIndexOption vResult = FragmentFactory.CreateFragment<VectorTypeIndexOption>()]
+    : tType:Identifier EqualsSign tTypeValue:AsciiStringLiteral
+        {
+            Match(tType, CodeGenerationSupporter.Type);
+            vResult.OptionKind = IndexOptionKind.VectorType;
+            vResult.VectorType = VectorIndexTypeHelper.Instance.ParseOption(tTypeValue);
+            UpdateTokenInfo(vResult, tType);
+        }
+    ;
+
 compressionPartitionRange returns [CompressionPartitionRange vResult = FragmentFactory.CreateFragment<CompressionPartitionRange>()]
 {
     ScalarExpression vExpression;

SqlScriptDom/Parser/TSql/VectorIndexType.cs

@@ -0,0 +1,23 @@
+//------------------------------------------------------------------------------
+// <copyright file="VectorIndexType.cs" company="Microsoft">
+//         Copyright (c) Microsoft Corporation.  All rights reserved.
+// </copyright>
+//------------------------------------------------------------------------------
+
+
+
+namespace Microsoft.SqlServer.TransactSql.ScriptDom
+{
+#pragma warning disable 1591
+
+    /// <summary>
+    /// The possible values for vector index type
+    /// </summary>              
+    public enum VectorIndexType
+    {
+        DiskANN = 0
+    }
+
+#pragma warning restore 1591
+}
+

SqlScriptDom/Parser/TSql/VectorIndexTypeHelper.cs

@@ -0,0 +1,21 @@
+//------------------------------------------------------------------------------
+// <copyright file="VectorIndexTypeHelper.cs" company="Microsoft">
+//         Copyright (c) Microsoft Corporation.  All rights reserved.
+// </copyright>
+//------------------------------------------------------------------------------
+
+
+
+namespace Microsoft.SqlServer.TransactSql.ScriptDom
+{
+
+    internal class VectorIndexTypeHelper : OptionsHelper<VectorIndexType>
+    {
+        private VectorIndexTypeHelper()
+        {
+            AddOptionMapping(VectorIndexType.DiskANN, "'" + CodeGenerationSupporter.DiskANN + "'");
+        }
+
+        public static readonly VectorIndexTypeHelper Instance = new VectorIndexTypeHelper();
+    }
+}

SqlScriptDom/Parser/TSql/VectorMetricType.cs

@@ -0,0 +1,25 @@
+//------------------------------------------------------------------------------
+// <copyright file="VectorMetricType.cs" company="Microsoft">
+//         Copyright (c) Microsoft Corporation.  All rights reserved.
+// </copyright>
+//------------------------------------------------------------------------------
+
+
+
+namespace Microsoft.SqlServer.TransactSql.ScriptDom
+{
+#pragma warning disable 1591
+
+    /// <summary>
+    /// The possible values for vector index metric
+    /// </summary>              
+    public enum VectorMetricType
+    {
+        Cosine = 0,
+        Dot = 1,
+        Euclidean = 2
+    }
+
+#pragma warning restore 1591
+}
+

SqlScriptDom/Parser/TSql/VectorMetricTypeHelper.cs

@@ -0,0 +1,25 @@
+//------------------------------------------------------------------------------
+// <copyright file="VectorMetricTypeHelper.cs" company="Microsoft">
+//         Copyright (c) Microsoft Corporation.  All rights reserved.
+// </copyright>
+//------------------------------------------------------------------------------
+
+
+
+using static Microsoft.SqlServer.TransactSql.ScriptDom.SensitivityClassification;
+
+namespace Microsoft.SqlServer.TransactSql.ScriptDom
+{
+
+    internal class VectorMetricTypeHelper : OptionsHelper<VectorMetricType>
+    {
+        private VectorMetricTypeHelper()
+        {
+            AddOptionMapping(VectorMetricType.Cosine, "'" + CodeGenerationSupporter.Cosine + "'");
+            AddOptionMapping(VectorMetricType.Dot, "'" + CodeGenerationSupporter.Dot + "'");
+            AddOptionMapping(VectorMetricType.Euclidean, "'" + CodeGenerationSupporter.Euclidean + "'");
+        }
+
+        public static readonly VectorMetricTypeHelper Instance = new VectorMetricTypeHelper();
+    }
+}

SqlScriptDom/ScriptDom/SqlServer/ScriptGenerator/SqlScriptGeneratorVisitor.CreateVectorIndexStatement.cs

@@ -0,0 +1,48 @@
+//------------------------------------------------------------------------------
+// <copyright file="SqlScriptGeneratorVisitor.CreateVectorIndexStatement.cs" company="Microsoft">
+//         Copyright (c) Microsoft Corporation.  All rights reserved.
+// </copyright>
+//------------------------------------------------------------------------------
+using System.Collections.Generic;
+using Microsoft.SqlServer.TransactSql.ScriptDom;
+
+namespace Microsoft.SqlServer.TransactSql.ScriptDom.ScriptGenerator
+{
+    partial class SqlScriptGeneratorVisitor
+    {
+        public override void ExplicitVisit(CreateVectorIndexStatement node)
+        {
+            GenerateKeyword(TSqlTokenType.Create);
+
+            GenerateSpaceAndIdentifier(CodeGenerationSupporter.Vector);
+
+            GenerateSpaceAndKeyword(TSqlTokenType.Index);
+
+            // name
+            GenerateSpaceAndFragmentIfNotNull(node.Name);
+
+            NewLineAndIndent();
+            GenerateKeyword(TSqlTokenType.On);
+            GenerateSpaceAndFragmentIfNotNull(node.OnName);
+
+            // Vector column
+            if (node.VectorColumn != null)
+            {
+                GenerateSpace();
+                GenerateSymbol(TSqlTokenType.LeftParenthesis);
+                GenerateFragmentIfNotNull(node.VectorColumn);
+                GenerateSymbol(TSqlTokenType.RightParenthesis);
+            }
+
+            GenerateIndexOptions(node.IndexOptions);
+
+            if (node.OnFileGroupOrPartitionScheme != null)
+            {
+                NewLineAndIndent();
+                GenerateKeyword(TSqlTokenType.On);
+
+                GenerateSpaceAndFragmentIfNotNull(node.OnFileGroupOrPartitionScheme);
+            }
+        }
+    }
+}

SqlScriptDom/ScriptDom/SqlServer/ScriptGenerator/SqlScriptGeneratorVisitor.VectorMetricIndexOption.cs

@@ -0,0 +1,19 @@
+//------------------------------------------------------------------------------
+// <copyright file="SqlScriptGeneratorVisitor.VectorMetricIndexOption.cs" company="Microsoft">
+//         Copyright (c) Microsoft Corporation.  All rights reserved.
+// </copyright>
+//------------------------------------------------------------------------------
+
+namespace Microsoft.SqlServer.TransactSql.ScriptDom.ScriptGenerator
+{
+    partial class SqlScriptGeneratorVisitor
+    {
+        public override void ExplicitVisit(VectorMetricIndexOption node)
+        {
+            IndexOptionHelper.Instance.GenerateSourceForOption(_writer, node.OptionKind);
+            GenerateSpaceAndSymbol(TSqlTokenType.EqualsSign);
+            GenerateSpace();
+            VectorMetricTypeHelper.Instance.GenerateSourceForOption(_writer, node.MetricType);
+        }
+    }
+}

SqlScriptDom/ScriptDom/SqlServer/ScriptGenerator/SqlScriptGeneratorVisitor.VectorTypeIndexOption.cs

@@ -0,0 +1,19 @@
+//------------------------------------------------------------------------------
+// <copyright file="SqlScriptGeneratorVisitor.VectorTypeIndexOption.cs" company="Microsoft">
+//         Copyright (c) Microsoft Corporation.  All rights reserved.
+// </copyright>
+//------------------------------------------------------------------------------
+
+namespace Microsoft.SqlServer.TransactSql.ScriptDom.ScriptGenerator
+{
+    partial class SqlScriptGeneratorVisitor
+    {
+        public override void ExplicitVisit(VectorTypeIndexOption node)
+        {
+            IndexOptionHelper.Instance.GenerateSourceForOption(_writer, node.OptionKind);
+            GenerateSpaceAndSymbol(TSqlTokenType.EqualsSign);
+            GenerateSpace();
+            VectorIndexTypeHelper.Instance.GenerateSourceForOption(_writer, node.VectorType);
+        }
+    }
+}