SOLR-18267: Add flat vector index with no HNSW

changelog/unreleased/SOLR-18267-add-flat-vector-index.yml

@@ -0,0 +1,7 @@
+title: Add knnAlgorithm="flat" option to DenseVectorField to skip HNSW graph construction for exact vector search
+type: added
+authors:
+- name: Adam Quigley
+links:
+- name: SOLR-18267
+  url: https://issues.apache.org/jira/browse/SOLR-18267

solr/core/src/java/org/apache/solr/core/SchemaCodecFactory.java

@@ -128,7 +128,8 @@ public KnnVectorsFormat getKnnVectorsFormatForField(String field) {
             FieldType fieldType = (schemaField == null ? null : schemaField.getType());
             if (fieldType instanceof DenseVectorField vectorField) {
               final String knnAlgorithm = vectorField.getKnnAlgorithm();
-              if (!DenseVectorField.HNSW_ALGORITHM.equals(knnAlgorithm)) {
+              if (!DenseVectorField.HNSW_ALGORITHM.equals(knnAlgorithm)
+                  && !DenseVectorField.FLAT_ALGORITHM.equals(knnAlgorithm)) {
                 throw new SolrException(
                     ErrorCode.SERVER_ERROR, knnAlgorithm + " KNN algorithm is not supported");
               }

solr/core/src/java/org/apache/solr/core/Solr101FlatVectorFormat.java

@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.core;
+
+import java.io.IOException;
+import org.apache.lucene.codecs.KnnVectorsFormat;
+import org.apache.lucene.codecs.KnnVectorsReader;
+import org.apache.lucene.codecs.KnnVectorsWriter;
+import org.apache.lucene.codecs.hnsw.FlatVectorScorerUtil;
+import org.apache.lucene.codecs.lucene99.Lucene99FlatVectorsFormat;
+import org.apache.lucene.index.SegmentReadState;
+import org.apache.lucene.index.SegmentWriteState;
+
+/**
+ * SPI-registered wrapper for {@link Lucene99FlatVectorsFormat}, which Lucene does not register as a
+ * {@link KnnVectorsFormat} in SPI.
+ *
+ * @lucene.spi {@value #NAME}
+ * @since 10.1
+ */
+public final class Solr101FlatVectorFormat extends KnnVectorsFormat {
+
+  static final String NAME = "Solr101FlatVectorFormat";
+
+  private final Lucene99FlatVectorsFormat delegate;
+
+  public Solr101FlatVectorFormat() {
+    super(NAME);
+    this.delegate =
+        new Lucene99FlatVectorsFormat(FlatVectorScorerUtil.getLucene99FlatVectorsScorer());
+  }
+
+  @Override
+  public KnnVectorsWriter fieldsWriter(SegmentWriteState state) throws IOException {
+    return delegate.fieldsWriter(state);
+  }
+
+  @Override
+  public KnnVectorsReader fieldsReader(SegmentReadState state) throws IOException {
+    return delegate.fieldsReader(state);
+  }
+
+  @Override
+  public int getMaxDimensions(String fieldName) {
+    return delegate.getMaxDimensions(fieldName);
+  }
+
+  @Override
+  public String toString() {
+    return NAME;
+  }
+}

solr/core/src/java/org/apache/solr/schema/BinaryQuantizedDenseVectorField.java

@@ -16,12 +16,25 @@
  */
 package org.apache.solr.schema;
 
+import java.util.Map;
 import org.apache.lucene.codecs.KnnVectorsFormat;
 import org.apache.lucene.codecs.lucene104.Lucene104HnswScalarQuantizedVectorsFormat;
 import org.apache.lucene.codecs.lucene104.Lucene104ScalarQuantizedVectorsFormat.ScalarEncoding;
+import org.apache.solr.common.SolrException;
 
 public class BinaryQuantizedDenseVectorField extends DenseVectorField {
 
+  @Override
+  public void init(IndexSchema schema, Map<String, String> args) {
+    super.init(schema, args);
+
+    if (FLAT_ALGORITHM.equals(getKnnAlgorithm())) {
+      throw new SolrException(
+          SolrException.ErrorCode.BAD_REQUEST,
+          "knnAlgorithm 'flat' is not supported for BinaryQuantizedDenseVectorField");
+    }
+  }
+
   @Override
   public KnnVectorsFormat buildKnnVectorsFormat() {
     return new Lucene104HnswScalarQuantizedVectorsFormat(

solr/core/src/java/org/apache/solr/schema/DenseVectorField.java

@@ -45,6 +45,7 @@
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.hnsw.HnswGraph;
 import org.apache.solr.common.SolrException;
+import org.apache.solr.core.Solr101FlatVectorFormat;
 import org.apache.solr.search.QParser;
 import org.apache.solr.search.vector.KnnQParser.EarlyTerminationParams;
 import org.apache.solr.search.vector.SolrKnnByteVectorQuery;
@@ -69,6 +70,7 @@
 public class DenseVectorField extends FloatPointField {
   private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
   public static final String HNSW_ALGORITHM = "hnsw";
+  public static final String FLAT_ALGORITHM = "flat";
   public static final String CAGRA_HNSW_ALGORITHM = "cagra_hnsw";
   public static final String DEFAULT_KNN_ALGORITHM = HNSW_ALGORITHM;
   static final String KNN_VECTOR_DIMENSION = "vectorDimension";
@@ -471,7 +473,11 @@ public DenseVectorParser getVectorBuilder(
   }
 
   public KnnVectorsFormat buildKnnVectorsFormat() {
-    return new Lucene99HnswVectorsFormat(hnswM, hnswEfConstruction);
+    if (FLAT_ALGORITHM.equals(knnAlgorithm)) {
+      return new Solr101FlatVectorFormat();
+    } else {
+      return new Lucene99HnswVectorsFormat(hnswM, hnswEfConstruction);
+    }
   }
 
   @Override
@@ -503,6 +509,13 @@ public Query getKnnVectorQuery(
       EarlyTerminationParams earlyTermination,
       Integer filteredSearchThreshold) {
 
+    if (FLAT_ALGORITHM.equals(knnAlgorithm)) {
+      throw new SolrException(
+          SolrException.ErrorCode.BAD_REQUEST,
+          "KNN vector queries are not supported for fields using knnAlgorithm=\"flat\". "
+              + "Use vectorSimilarity() function queries instead.");
+    }
+
     DenseVectorParser vectorBuilder =
         getVectorBuilder(vectorToSearch, DenseVectorParser.BuilderPhase.QUERY);
 

solr/core/src/java/org/apache/solr/schema/ScalarQuantizedDenseVectorField.java

@@ -122,6 +122,12 @@ public void init(IndexSchema schema, Map<String, String> args) {
     }
 
     super.init(schema, args);
+
+    if (FLAT_ALGORITHM.equals(getKnnAlgorithm())) {
+      throw new SolrException(
+          SolrException.ErrorCode.BAD_REQUEST,
+          "knnAlgorithm 'flat' is not supported for ScalarQuantizedDenseVectorField");
+    }
   }
 
   @Override

solr/core/src/java/org/apache/solr/search/vector/KnnQParser.java

@@ -119,6 +119,14 @@ public Query parse() throws SyntaxError {
     final String vectorField = getFieldName();
     final SchemaField schemaField = req.getCore().getLatestSchema().getField(getFieldName());
     final DenseVectorField denseVectorType = getCheckedFieldType(schemaField);
+
+    if (DenseVectorField.FLAT_ALGORITHM.equals(denseVectorType.getKnnAlgorithm())) {
+      throw new SolrException(
+          SolrException.ErrorCode.BAD_REQUEST,
+          "The {!knn} query parser is not supported for fields using knnAlgorithm=\"flat\". "
+              + "Use vectorSimilarity() function queries instead.");
+    }
+
     final String vectorToSearch = getVectorToSearch();
     final int topK = localParams.getInt(TOP_K, DEFAULT_TOP_K);
 

solr/core/src/java/org/apache/solr/search/vector/VectorSimilarityQParser.java

@@ -46,6 +46,14 @@ public Query parse() throws SyntaxError {
     final String fieldName = getFieldName();
     final SchemaField schemaField = req.getCore().getLatestSchema().getField(fieldName);
     final DenseVectorField denseVectorType = getCheckedFieldType(schemaField);
+
+    if (DenseVectorField.FLAT_ALGORITHM.equals(denseVectorType.getKnnAlgorithm())) {
+      throw new SolrException(
+          SolrException.ErrorCode.BAD_REQUEST,
+          "The {!vectorSimilarity} query parser is not supported for fields using knnAlgorithm=\"flat\". "
+              + "Use vectorSimilarity() function queries instead.");
+    }
+
     final String vectorToSearch = getVectorToSearch();
     final float minTraverse = localParams.getFloat(MIN_TRAVERSE, DEFAULT_MIN_TRAVERSE);
     final Float minReturn = localParams.getFloat(MIN_RETURN);

solr/core/src/resources/META-INF/services/org.apache.lucene.codecs.KnnVectorsFormat

@@ -0,0 +1,16 @@
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+org.apache.solr.core.Solr101FlatVectorFormat

solr/core/src/test-files/solr/collection1/conf/bad-schema-densevector-flat-binaryQuantized.xml

@@ -0,0 +1,29 @@
+<?xml version="1.0" ?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements.  See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License.  You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<!-- Test schema file for DenseVectorField -->
+
+<schema name="bad-schema-densevector-flat-bq" version="1.7">
+  <fieldType name="string" class="solr.StrField" multiValued="true"/>
+  <fieldType name="knn_vector_flat_bq" class="solr.BinaryQuantizedDenseVectorField" vectorDimension="4" similarityFunction="cosine" knnAlgorithm="flat"/>
+
+  <field name="id" type="string" indexed="true" stored="true" multiValued="false" required="false"/>
+  <field name="vector" type="knn_vector_flat_bq" indexed="true" stored="true"/>
+
+  <uniqueKey>id</uniqueKey>
+</schema>

solr/core/src/test-files/solr/collection1/conf/bad-schema-densevector-flat-scalarQuantized.xml

@@ -0,0 +1,29 @@
+<?xml version="1.0" ?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements.  See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License.  You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<!-- Test schema file for DenseVectorField -->
+
+<schema name="bad-schema-densevector-flat-quantized" version="1.7">
+  <fieldType name="string" class="solr.StrField" multiValued="true"/>
+  <fieldType name="knn_vector_flat_sq" class="solr.ScalarQuantizedDenseVectorField" vectorDimension="4" similarityFunction="cosine" knnAlgorithm="flat"/>
+
+  <field name="id" type="string" indexed="true" stored="true" multiValued="false" required="false"/>
+  <field name="vector" type="knn_vector_flat_sq" indexed="true" stored="true"/>
+
+  <uniqueKey>id</uniqueKey>
+</schema>

solr/core/src/test-files/solr/collection1/conf/schema-densevector-flat.xml

@@ -0,0 +1,31 @@
+<?xml version="1.0" ?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements.  See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License.  You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<!-- Test schema file for DenseVectorField with flat (non-HNSW) algorithm -->
+
+<schema name="schema-densevector-flat" version="1.7">
+  <fieldType name="string" class="solr.StrField" multiValued="true"/>
+  <fieldType name="knn_vector_flat" class="solr.DenseVectorField" vectorDimension="4" similarityFunction="cosine" knnAlgorithm="flat"/>
+  <fieldType name="knn_vector_flat_byte" class="solr.DenseVectorField" vectorDimension="4" similarityFunction="cosine" knnAlgorithm="flat" vectorEncoding="BYTE"/>
+
+  <field name="id" type="string" indexed="true" stored="true" multiValued="false" required="false"/>
+  <field name="vector_flat" type="knn_vector_flat" indexed="true" stored="true"/>
+  <field name="vector_flat_byte" type="knn_vector_flat_byte" indexed="true" stored="true"/>
+
+  <uniqueKey>id</uniqueKey>
+</schema>

solr/core/src/test/org/apache/solr/schema/BinaryQuantizedDenseVectorFieldTest.java

@@ -38,6 +38,14 @@ public void fieldDefinition_correctConfiguration_shouldLoadSchemaField() throws
     }
   }
 
+  @Test
+  public void fieldDefinition_flatAlgorithm_shouldThrowException() throws Exception {
+    assertConfigs(
+        "solrconfig-basic.xml",
+        "bad-schema-densevector-flat-binaryQuantized.xml",
+        "knnAlgorithm 'flat' is not supported for BinaryQuantizedDenseVectorField");
+  }
+
   // there are no major interface differences between BinaryBitQuantizedDenseVectorField and
   // DenseVectorField
   // so we can rely on those tests for validation cases