feat: Add FlatBuffers schemas for dense index and bloom filter

Summary:
CONTEXT: Nimble needs a hash-based dense index for row-level point lookups.
The index metadata is serialized using FlatBuffers.

WHAT: Add FlatBuffers schema definitions for the dense index:
- BloomFilter.fbs: Extracted into a separate file for reuse by other index types
- DenseIndex.fbs: DenseIndexDirectory, DenseIndexSection, DenseIndex, and
  DenseIndexPartition tables for the hash-based index structure
- BUCK/CMakeLists.txt: Build rules for the new .fbs files

Reviewed By: tanjialiang

Differential Revision: D99569601

Author: xiaoxmeng

dwio/nimble/tablet/BloomFilter.fbs

@@ -0,0 +1,29 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+namespace facebook.nimble.serialization;
+
+/// Split block bloom filter for fast negative lookups.
+/// Uses 256-bit blocks with multiple hash probes per block,
+/// following the Parquet bloom filter design for cache efficiency.
+table BloomFilter {
+  /// Number of 256-bit blocks in the filter.
+  num_blocks:uint32;
+  /// Bits per key used during construction (for diagnostics/rebuild).
+  bits_per_key:float;
+  /// Raw filter data. Size = num_blocks * 32 bytes.
+  data:[uint8];
+}

dwio/nimble/tablet/CMakeLists.txt

@@ -64,6 +64,38 @@ target_include_directories(
 )
 add_dependencies(nimble_chunk_index_fb nimble_chunk_index_schema_fb)
 
+build_flatbuffers(
+  "${CMAKE_CURRENT_SOURCE_DIR}/BloomFilter.fbs"
+  ""
+  nimble_bloom_filter_schema_fb
+  ""
+  "${CMAKE_CURRENT_BINARY_DIR}"
+  ""
+  ""
+)
+add_library(nimble_bloom_filter_fb INTERFACE)
+target_include_directories(
+  nimble_bloom_filter_fb
+  INTERFACE ${PROJECT_BINARY_DIR} ${FLATBUFFERS_INCLUDE_DIR}
+)
+add_dependencies(nimble_bloom_filter_fb nimble_bloom_filter_schema_fb)
+
+build_flatbuffers(
+  "${CMAKE_CURRENT_SOURCE_DIR}/DenseIndex.fbs"
+  "${CMAKE_CURRENT_SOURCE_DIR}"
+  nimble_dense_index_schema_fb
+  ""
+  "${CMAKE_CURRENT_BINARY_DIR}"
+  ""
+  ""
+)
+add_library(nimble_dense_index_fb INTERFACE)
+target_include_directories(
+  nimble_dense_index_fb
+  INTERFACE ${PROJECT_BINARY_DIR} ${FLATBUFFERS_INCLUDE_DIR}
+)
+add_dependencies(nimble_dense_index_fb nimble_dense_index_schema_fb)
+
 add_library(nimble_tablet_common Compression.cpp MetadataBuffer.cpp)
 target_link_libraries(
   nimble_tablet_common

dwio/nimble/tablet/DenseIndex.fbs

@@ -0,0 +1,104 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+include "BloomFilter.fbs";
+include "Footer.fbs";
+
+namespace facebook.nimble.serialization;
+
+/// A single dense hash index on a set of composite key columns.
+/// Uses a two-level hash table: bucket directory + key arrays.
+///
+/// Lookup flow:
+///   1. Optional bloom filter check for fast negative result
+///   2. Hash(encodedKey) mod num_buckets → bucket
+///   3. Scan keys in bucket by exact key match
+///   4. Return matching row numbers
+table DenseIndex {
+  /// Names of columns forming the composite key.
+  index_columns:[string];
+
+  /// Total number of rows indexed.
+  row_count:uint64;
+
+  /// Number of buckets in the hash table (always power of 2).
+  num_buckets:uint32;
+  /// Total number of keys in the index. Diagnostic only; not used on the
+  /// read path. Equal to row_count when there is no deduplication.
+  num_keys:uint64;
+  /// Load factor used during construction (for diagnostics).
+  load_factor:float;
+
+  /// Accumulated row counts per stripe (prefix sum).
+  /// Enables translating global row number to (stripe, stripe-local row).
+  /// Size = stripe_count.
+  stripe_row_counts:[uint32];
+
+  /// Optional bloom filter for fast negative lookups.
+  bloom_filter:BloomFilter;
+
+  /// Partitions. The hash table is always split into one or more partitions,
+  /// each stored as a separate DenseIndexPartition section. Small indices
+  /// use a single partition; large indices are split by maxPartitionSizeBytes.
+  /// Starting bucket index in the global bucket space (inclusive).
+  /// Size = partition_count.
+  partition_start_buckets:[uint32];
+  /// Number of buckets in each partition.
+  /// Size = partition_count.
+  partition_bucket_counts:[uint32];
+  /// References to the serialized DenseIndexPartition for each partition.
+  /// Size = partition_count.
+  partition_sections:[MetadataSection];
+}
+
+/// Data for a single partition of a dense index.
+/// Stored as a separate section, loaded on-demand during lookup.
+table DenseIndexPartition {
+  /// Bucket directory for this partition. Indices are relative to the
+  /// partition's key arrays (encoded_keys and row_numbers).
+  bucket_offsets:[uint32];
+  /// Number of keys in each bucket.
+  bucket_counts:[uint16];
+  /// Binary-encoded composite keys for this partition. Aligned with
+  /// row_numbers: encoded_keys[i] corresponds to row_numbers[i].
+  encoded_keys:[string];
+  /// Global row numbers for this partition. Aligned with encoded_keys.
+  row_numbers:[uint32];
+}
+
+/// Lightweight descriptor for a dense index.
+/// Stores the index columns for lookup dispatch and a MetadataSection
+/// pointing to the serialized DenseIndex data.
+table DenseIndexSection {
+  /// Names of columns forming the composite key.
+  /// Duplicated here so the reader can find the right index
+  /// without loading the full DenseIndex data.
+  index_columns:[string];
+  /// Reference to the serialized DenseIndex stored as a
+  /// separate section in the file.
+  section:MetadataSection;
+}
+
+/// Root table for the dense index optional section.
+/// Supports multiple independent dense indices per file,
+/// each on a different set of composite key columns.
+/// Stored in optional section "columnar.dense.index".
+table DenseIndexDirectory {
+  /// One descriptor per dense index defined on the file.
+  indices:[DenseIndexSection];
+}
+
+root_type DenseIndexDirectory;