wip

giga wip (doesnt compile)

wip

Author: teh-cmc

Cargo.lock

@@ -4311,6 +4311,7 @@ dependencies = [
  "rand",
  "re_arrow2",
  "re_build_info",
+ "re_error",
  "re_format",
  "re_format_arrow",
  "re_log",
@@ -4526,6 +4527,7 @@ dependencies = [
  "re_log_encoding",
  "re_log_types",
  "re_query",
+ "re_query2",
  "re_smart_channel",
  "re_tracing",
  "re_types",
@@ -4714,6 +4716,39 @@ dependencies = [
  "thiserror",
 ]
 
+[[package]]
+name = "re_query2"
+version = "0.18.0-alpha.1+dev"
+dependencies = [
+ "ahash",
+ "anyhow",
+ "backtrace",
+ "criterion",
+ "indent",
+ "indexmap 2.1.0",
+ "itertools 0.13.0",
+ "mimalloc",
+ "nohash-hasher",
+ "parking_lot",
+ "paste",
+ "rand",
+ "re_arrow2",
+ "re_chunk",
+ "re_chunk_store",
+ "re_error",
+ "re_format",
+ "re_log",
+ "re_log_types",
+ "re_tracing",
+ "re_tuid",
+ "re_types",
+ "re_types_core",
+ "seq-macro",
+ "similar-asserts",
+ "static_assertions",
+ "thiserror",
+]
+
 [[package]]
 name = "re_renderer"
 version = "0.18.0-alpha.1+dev"
@@ -4891,6 +4926,7 @@ dependencies = [
  "re_log",
  "re_log_types",
  "re_query",
+ "re_query2",
  "re_tracing",
  "re_types_core",
  "re_ui",
@@ -5049,7 +5085,7 @@ dependencies = [
  "re_format",
  "re_log",
  "re_log_types",
- "re_query",
+ "re_query2",
  "re_renderer",
  "re_space_view",
  "re_tracing",
@@ -5278,6 +5314,7 @@ dependencies = [
  "re_log_types",
  "re_memory",
  "re_query",
+ "re_query2",
  "re_renderer",
  "re_sdk_comms",
  "re_selection_panel",
@@ -5349,6 +5386,7 @@ dependencies = [
  "re_log_types",
  "re_math",
  "re_query",
+ "re_query2",
  "re_renderer",
  "re_smart_channel",
  "re_string_interner",

Cargo.toml

@@ -49,6 +49,7 @@ re_format_arrow = { path = "crates/store/re_format_arrow", version = "=0.18.0-al
 re_log_encoding = { path = "crates/store/re_log_encoding", version = "=0.18.0-alpha.1", default-features = false }
 re_log_types = { path = "crates/store/re_log_types", version = "=0.18.0-alpha.1", default-features = false }
 re_query = { path = "crates/store/re_query", version = "=0.18.0-alpha.1", default-features = false }
+re_query2 = { path = "crates/store/re_query2", version = "=0.18.0-alpha.1", default-features = false }
 re_sdk_comms = { path = "crates/store/re_sdk_comms", version = "=0.18.0-alpha.1", default-features = false }
 re_types = { path = "crates/store/re_types", version = "=0.18.0-alpha.1", default-features = false }
 re_types_blueprint = { path = "crates/store/re_types_blueprint", version = "=0.18.0-alpha.1", default-features = false }

crates/store/re_chunk/Cargo.toml

@@ -37,6 +37,7 @@ serde = [
 
 # Rerun
 re_build_info.workspace = true
+re_error.workspace = true
 re_format.workspace = true
 re_format_arrow.workspace = true
 re_log.workspace = true

crates/store/re_chunk/src/chunk.rs

@@ -3,14 +3,19 @@ use std::{
     sync::atomic::{AtomicU64, Ordering},
 };
 
-use arrow2::array::{
-    Array as ArrowArray, ListArray as ArrowListArray, PrimitiveArray as ArrowPrimitiveArray,
-    StructArray as ArrowStructArray,
+use arrow2::{
+    array::{
+        Array as ArrowArray, ListArray as ArrowListArray, PrimitiveArray as ArrowPrimitiveArray,
+        StructArray as ArrowStructArray,
+    },
+    Either,
 };
 
 use itertools::{izip, Itertools};
 use re_log_types::{EntityPath, ResolvedTimeRange, Time, TimeInt, TimePoint, Timeline};
-use re_types_core::{ComponentName, Loggable, LoggableBatch, SerializationError, SizeBytes};
+use re_types_core::{
+    ComponentName, DeserializationError, Loggable, LoggableBatch, SerializationError, SizeBytes,
+};
 
 use crate::{ChunkId, RowId};
 
@@ -26,8 +31,18 @@ pub enum ChunkError {
     #[error(transparent)]
     Arrow(#[from] arrow2::error::Error),
 
+    #[error("{kind} index out of bounds: {index} (len={len})")]
+    IndexOutOfBounds {
+        kind: String,
+        len: usize,
+        index: usize,
+    },
+
     #[error(transparent)]
     Serialization(#[from] SerializationError),
+
+    #[error(transparent)]
+    Deserialization(#[from] DeserializationError),
 }
 
 pub type ChunkResult<T> = Result<T, ChunkError>;
@@ -822,6 +837,32 @@ impl Chunk {
             .map(|(&time, &counter)| RowId::from_u128((time as u128) << 64 | (counter as u128)))
     }
 
+    /// Returns an iterator over the [`RowId`]s of a [`Chunk`], for a given component.
+    ///
+    /// This is different than [`Self::row_ids`]: it will only yield `RowId`s for rows at which
+    /// there is data for the specified `component_name`.
+    #[inline]
+    pub fn component_row_ids(
+        &self,
+        component_name: &ComponentName,
+    ) -> impl Iterator<Item = RowId> + '_ {
+        let Some(list_array) = self.components.get(component_name) else {
+            return Either::Left(std::iter::empty());
+        };
+
+        let row_ids = self.row_ids();
+
+        if let Some(validity) = list_array.validity() {
+            Either::Right(Either::Left(
+                row_ids
+                    .enumerate()
+                    .filter_map(|(i, o)| validity.get_bit(i).then_some(o)),
+            ))
+        } else {
+            Either::Right(Either::Right(row_ids))
+        }
+    }
+
     /// Returns the [`RowId`]-range covered by this [`Chunk`].
     ///
     /// `None` if the chunk `is_empty`.