Merge branch 'optimize-offline-mft'

Author: githubrobbi

crates/uffs-cli/src/commands/output.rs

@@ -18,6 +18,18 @@ use uffs_mft::MftReader;
 
 use super::output::results_to_dataframe;
 
+/// Native offline query results for direct `--mft-file` output.
+pub(super) struct NativeOfflineQueryResults {
+    /// Loaded offline index used for record metadata lookups during output.
+    pub(super) index: uffs_mft::MftIndex,
+    /// Native search results collected from `IndexQuery`.
+    pub(super) results: Vec<uffs_core::SearchResult>,
+    /// Raw MFT load duration in milliseconds.
+    pub(super) load_ms: u128,
+    /// Query/filter duration in milliseconds.
+    pub(super) query_ms: u128,
+}
+
 /// Query filter options for the search command.
 pub(super) struct QueryFilters<'a> {
     /// Parsed search pattern (glob, regex, or literal).
@@ -146,6 +158,42 @@ pub(super) fn load_and_filter_from_mft_file(
     profile: bool,
     debug_tree: bool,
 ) -> Result<uffs_mft::DataFrame> {
+    let native = load_and_filter_native_from_mft_file(
+        mft_path,
+        drive_letter,
+        filters,
+        needs_paths,
+        debug_tree,
+    )?;
+    let matches = native.results.len();
+    let records = native.index.len();
+    let df = results_to_dataframe(&native.index, native.results, needs_paths)?;
+
+    if profile {
+        let total_ms = native.load_ms + native.query_ms;
+        eprintln!("=== RAW MFT FILE TIMING ===");
+        eprintln!(
+            "  Load from file:  {:>6} ms  ({} records)",
+            native.load_ms, records
+        );
+        eprintln!(
+            "  Query/filter:    {:>6} ms  ({} matches)",
+            native.query_ms, matches
+        );
+        eprintln!("  TOTAL:           {total_ms:>6} ms");
+    }
+
+    Ok(df)
+}
+
+/// Load, query, and return native results from a raw offline MFT file.
+pub(super) fn load_and_filter_native_from_mft_file(
+    mft_path: &Path,
+    drive_letter: Option<char>,
+    filters: &QueryFilters<'_>,
+    needs_paths: bool,
+    debug_tree: bool,
+) -> Result<NativeOfflineQueryResults> {
     use uffs_mft::LoadRawOptions;
 
     let volume = drive_letter.unwrap_or('X');
@@ -166,24 +214,15 @@ pub(super) fn load_and_filter_from_mft_file(
     let load_ms = t_load.elapsed().as_millis();
 
     let t_query = std::time::Instant::now();
-    let results = execute_index_query(&index, filters, needs_paths)?;
+    let results = execute_index_query_native(&index, filters, needs_paths)?;
     let query_ms = t_query.elapsed().as_millis();
 
-    if profile {
-        let total_ms = load_ms + query_ms;
-        eprintln!("=== RAW MFT FILE TIMING ===");
-        eprintln!(
-            "  Load from file:  {load_ms:>6} ms  ({} records)",
-            index.len()
-        );
-        eprintln!(
-            "  Query/filter:    {query_ms:>6} ms  ({} matches)",
-            results.height()
-        );
-        eprintln!("  TOTAL:           {total_ms:>6} ms");
-    }
-
-    Ok(results)
+    Ok(NativeOfflineQueryResults {
+        index,
+        results,
+        load_ms,
+        query_ms,
+    })
 }
 
 /// Load raw MFT with debug output for tree metrics.
@@ -744,13 +783,12 @@ fn execute_query(
 
 /// Execute query using fast `IndexQuery` path (no `DataFrame` conversion).
 ///
-/// This is the fast path for simple queries. Returns results as a `DataFrame`
-/// for compatibility with the output pipeline.
-fn execute_index_query(
+/// This is the fast path for simple queries.
+fn execute_index_query_native(
     index: &uffs_mft::MftIndex,
     filters: &QueryFilters<'_>,
     resolve_paths: bool,
-) -> Result<uffs_mft::DataFrame> {
+) -> Result<Vec<uffs_core::SearchResult>> {
     use uffs_core::{IndexQuery, TypeFilter, compile_parsed_pattern};
 
     let mut query = IndexQuery::new(index);
@@ -789,6 +827,5 @@ fn execute_index_query(
     query = query.case_sensitive(filters.parsed.is_case_sensitive());
     query = query.with_resolve_paths(resolve_paths);
 
-    let results = query.collect();
-    results_to_dataframe(index, &results, resolve_paths)
+    Ok(query.collect())
 }

crates/uffs-cli/src/commands/raw_io.rs

@@ -9,8 +9,11 @@ use uffs_core::output::OutputConfig;
 use uffs_core::pattern::ParsedPattern;
 use uffs_core::tree::add_tree_columns;
 
-use super::output::write_results;
-use super::raw_io::{QueryFilters, load_and_filter_data, load_and_filter_from_mft_file};
+use super::output::{can_write_native_results, write_native_results, write_results};
+use super::raw_io::{
+    QueryFilters, load_and_filter_data, load_and_filter_from_mft_file,
+    load_and_filter_native_from_mft_file,
+};
 #[cfg(windows)]
 use super::raw_io::{load_and_filter_data_index, load_and_filter_data_index_multi};
 
@@ -200,6 +203,64 @@ pub async fn search(
         .or_else(|| filters.parsed.drive().map(|drive| vec![drive]))
         .unwrap_or_default();
 
+    if let Some(mft_path) = mft_file.as_ref()
+        && !benchmark
+        && can_write_native_results(format, &output_config)
+    {
+        info!(
+            path = %mft_path.display(),
+            format,
+            "📂 Loading raw MFT file via native direct-output path"
+        );
+
+        let native = load_and_filter_native_from_mft_file(
+            mft_path,
+            single_drive,
+            &filters,
+            needs_paths,
+            debug_tree,
+        )?;
+
+        let t_output = std::time::Instant::now();
+        write_native_results(
+            &native.index,
+            &native.results,
+            format,
+            out,
+            &output_config,
+            &output_targets,
+        )?;
+        let output_ms = t_output.elapsed().as_millis();
+        let elapsed = start_time.elapsed();
+
+        if profile {
+            let raw_total_ms = native.load_ms + native.query_ms;
+            eprintln!("=== RAW MFT FILE TIMING ===");
+            eprintln!(
+                "  Load from file:  {:>6} ms  ({} records)",
+                native.load_ms,
+                native.index.len()
+            );
+            eprintln!(
+                "  Query/filter:    {:>6} ms  ({} matches)",
+                native.query_ms,
+                native.results.len()
+            );
+            eprintln!("  TOTAL:           {raw_total_ms:>6} ms");
+            eprintln!("=== PROFILE: Output ===");
+            eprintln!("  Tree columns:    {:>6} ms", 0_u128);
+            eprintln!(
+                "  Output/write:    {:>6} ms  ({} rows)",
+                output_ms,
+                native.results.len()
+            );
+            eprintln!("=== TOTAL: {} ms ===", elapsed.as_millis());
+        }
+
+        info!(count = native.results.len(), "Search complete");
+        return Ok(());
+    }
+
     let mut results = if let Some(mft_path) = mft_file.as_ref() {
         info!(path = %mft_path.display(), "📂 Loading from raw MFT file");
         load_and_filter_from_mft_file(

crates/uffs-cli/src/commands/search/mod.rs

@@ -5,7 +5,7 @@ use rayon::prelude::*;
 use super::IndexQuery;
 use super::expansion::RecordExpander;
 use super::filtering::RecordFilter;
-use super::planning::CollectPlan;
+use super::planning::{CollectPlan, ScanPlan};
 use crate::index_search::SearchResult;
 
 impl IndexQuery<'_> {
@@ -31,21 +31,42 @@ impl IndexQuery<'_> {
             self.min_size,
             self.max_size,
         );
-        let plan = CollectPlan::build(index, self.pattern.as_ref(), include_system_metafiles);
+        let CollectPlan {
+            path_cache,
+            scan_plan,
+        } = CollectPlan::build(index, self.pattern.as_ref(), include_system_metafiles);
+        let path_resolver = path_cache.resolver();
         let expander = RecordExpander::new(
             index,
-            &plan.path_cache,
+            &path_cache,
             expand_names,
             expand_streams,
             resolve_paths,
         );
+        let scan_limit = limit.unwrap_or(usize::MAX);
 
-        plan.records_to_scan
-            .par_iter()
-            .filter(|record| plan.path_cache.is_valid(record.frs) && filters.matches(record))
-            .take_any(limit.unwrap_or(usize::MAX))
-            .flat_map_iter(|record| expander.collect_results(record))
-            .collect()
+        match scan_plan {
+            ScanPlan::Full(records) => records
+                .par_iter()
+                .enumerate()
+                .filter(|(record_idx, record)| {
+                    path_resolver.is_valid_idx(*record_idx) && filters.matches(record)
+                })
+                .take_any(scan_limit)
+                .flat_map_iter(|(record_idx, record)| expander.collect_results(record_idx, record))
+                .collect(),
+            ScanPlan::Filtered { records, indices } => indices
+                .par_iter()
+                .filter_map(|&record_idx_u32| {
+                    let record_idx = usize::try_from(record_idx_u32).ok()?;
+                    let record = records.get(record_idx)?;
+                    (path_resolver.is_valid_idx(record_idx) && filters.matches(record))
+                        .then_some((record_idx, record))
+                })
+                .take_any(scan_limit)
+                .flat_map_iter(|(record_idx, record)| expander.collect_results(record_idx, record))
+                .collect(),
+        }
     }
 
     /// Count matching records without collecting results.

crates/uffs-core/src/index_search/query/execution.rs

@@ -43,7 +43,11 @@ impl<'a> RecordExpander<'a> {
 
     /// Collect all output search results for the record.
     #[must_use]
-    pub(super) fn collect_results(&self, record: &FileRecord) -> Vec<SearchResult> {
+    pub(super) fn collect_results(
+        &self,
+        record_idx: usize,
+        record: &FileRecord,
+    ) -> Vec<SearchResult> {
         let name_count = if self.expand_names {
             record.name_count.max(1)
         } else {
@@ -54,11 +58,15 @@ impl<'a> RecordExpander<'a> {
         } else {
             1
         };
-        let cached_path = if self.resolve_paths {
-            self.path_cache.get(record.frs)
-        } else {
-            None
-        };
+        let path_index = self.path_cache.index();
+        let path_resolver = self.path_cache.resolver();
+        let cached_path = self.resolve_paths.then(|| {
+            debug_assert!(
+                path_resolver.is_valid_idx(record_idx),
+                "collect_results only resolves paths for valid record indices"
+            );
+            path_resolver.materialize_path(path_index, record_idx)
+        });
 
         let mut results = Vec::with_capacity(usize::from(name_count) * usize::from(stream_count));
         for name_idx in 0..name_count {
@@ -76,6 +84,7 @@ impl<'a> RecordExpander<'a> {
                     self.resolve_result_path(
                         expanded_result,
                         record,
+                        record_idx,
                         name_idx,
                         stream_idx,
                         cached_path.clone(),
@@ -98,19 +107,22 @@ impl<'a> RecordExpander<'a> {
         &self,
         result: SearchResult,
         record: &FileRecord,
+        record_idx: usize,
         name_idx: u16,
         stream_idx: u16,
         cached_path: Option<String>,
     ) -> SearchResult {
+        let path_index = self.path_cache.index();
+        let path_resolver = self.path_cache.resolver();
         let stream = self
             .index
             .get_stream_at(record, stream_idx)
             .unwrap_or(&record.first_stream);
 
         let mut base_path = if name_idx == 0 {
-            cached_path.unwrap_or_else(|| self.index.build_path(record.frs))
+            cached_path.unwrap_or_else(|| path_resolver.materialize_path(path_index, record_idx))
         } else {
-            self.index.build_path_for_name(record, name_idx)
+            path_resolver.materialize_path_for_name(path_index, record_idx, name_idx)
         };
 
         let stream_name = self.index.stream_name(stream);

crates/uffs-core/src/index_search/query/expansion.rs

@@ -4,12 +4,25 @@ use uffs_mft::index::{FileRecord, MftIndex, PathCache};
 
 use crate::index_search::IndexPattern;
 
+/// Candidate-record scan plan for `IndexQuery::collect()`.
+pub(super) enum ScanPlan<'a> {
+    /// Scan the full record slice directly.
+    Full(&'a [FileRecord]),
+    /// Scan a narrowed set of record indices from the extension index.
+    Filtered {
+        /// Backing slice used for index lookup.
+        records: &'a [FileRecord],
+        /// Candidate indices from the extension index.
+        indices: Vec<u32>,
+    },
+}
+
 /// Precomputed inputs for `IndexQuery::collect()`.
 pub(super) struct CollectPlan<'a> {
     /// Shared path cache used for path validity checks and materialization.
     pub(super) path_cache: PathCache<'a>,
     /// Candidate records to scan after extension-index planning.
-    pub(super) records_to_scan: Vec<&'a FileRecord>,
+    pub(super) scan_plan: ScanPlan<'a>,
 }
 
 impl<'a> CollectPlan<'a> {
@@ -25,21 +38,15 @@ impl<'a> CollectPlan<'a> {
         include_system_metafiles: bool,
     ) -> Self {
         let path_cache = PathCache::build(index, include_system_metafiles);
-        let extension_filter_indices = Self::build_extension_filter_indices(pattern, index);
         let records = index.records();
-        let records_to_scan: Vec<&FileRecord> = extension_filter_indices.as_ref().map_or_else(
-            || records.iter().collect(),
-            |indices| {
-                indices
-                    .iter()
-                    .filter_map(|&idx| records.get(idx as usize))
-                    .collect()
-            },
+        let scan_plan = Self::build_extension_filter_indices(pattern, index).map_or(
+            ScanPlan::Full(records),
+            |indices| ScanPlan::Filtered { records, indices },
         );
 
         Self {
             path_cache,
-            records_to_scan,
+            scan_plan,
         }
     }