From 6085b944e262dbc0bb77f933ea10663b1aa94d3e Mon Sep 17 00:00:00 2001 From: Ted Willke Date: Wed, 18 Feb 2026 02:51:05 +0000 Subject: [PATCH 1/4] Add construction metrics to logging. --- .../github/jbellis/jvector/example/Grid.java | 39 ++++++++++++++++++- .../jvector/example/benchmarks/Metric.java | 6 ++- .../example/benchmarks/QueryTester.java | 16 +++++++- .../reporting/SearchReportingCatalog.java | 3 +- jvector-examples/yaml-configs/run.yml | 2 +- logging/2026-02-18_02-34-17/dataset_info.csv | 4 ++ logging/2026-02-18_02-34-17/experiments.csv | 13 +++++++ logging/2026-02-18_02-34-17/sys_info.json | 1 + 8 files changed, 77 insertions(+), 7 deletions(-) create mode 100644 logging/2026-02-18_02-34-17/dataset_info.csv create mode 100644 logging/2026-02-18_02-34-17/experiments.csv create mode 100644 logging/2026-02-18_02-34-17/sys_info.json diff --git a/jvector-examples/src/main/java/io/github/jbellis/jvector/example/Grid.java b/jvector-examples/src/main/java/io/github/jbellis/jvector/example/Grid.java index d5a1e9bb2..187980b08 100644 --- a/jvector-examples/src/main/java/io/github/jbellis/jvector/example/Grid.java +++ b/jvector-examples/src/main/java/io/github/jbellis/jvector/example/Grid.java @@ -225,6 +225,8 @@ static void runOneGraph(OnDiskGraphIndexCache cache, DataSet ds, Path workDirectory) throws IOException { + // Prepare to collect index construction metrics for reporting.... + var constructionMetrics = new ConstructionMetrics(); // TODO this does not capture disk usage for cached indexes. Need to update // Capture initial memory and disk state try (var diagnostics = new BenchmarkDiagnostics(getDiagnosticLevel())) { @@ -279,6 +281,7 @@ static void runOneGraph(OnDiskGraphIndexCache cache, diagnostics.printDiskStatistics("Graph Index Build"); System.out.printf("Index build time: %f seconds%n", Grid.getIndexBuildTimeSeconds(ds.getName())); + constructionMetrics.indexBuildTimeS = Grid.getIndexBuildTimeSeconds(ds.getName()); try { for (var cpSupplier : compressionGrid) { @@ -298,13 +301,15 @@ static void runOneGraph(OnDiskGraphIndexCache cache, } else { long start = System.nanoTime(); cv = compressor.encodeAll(ds.getBaseRavv()); - System.out.format("%s encoded %d vectors [%.2f MB] in %.2fs%n", compressor, ds.getBaseVectors().size(), (cv.ramBytesUsed() / 1024f / 1024f), (System.nanoTime() - start) / 1_000_000_000.0); + double encodingTimeS = (System.nanoTime() - start) / 1_000_000_000.0; + System.out.format("%s encoded %d vectors [%.2f MB] in %.2fs%n", compressor, ds.getBaseVectors().size(), (cv.ramBytesUsed() / 1024f / 1024f), encodingTimeS); + constructionMetrics.encodingTimeS = encodingTimeS; // Report encoding time } } try (var cs = new ConfiguredSystem(ds, index, cv, featureSetForIndex)) { testConfiguration(cs, topKGrid, usePruningGrid, M, efConstruction, neighborOverflow, addHierarchy, refineFinalGraph, featureSetForIndex, - artifacts, workDirectory); + artifacts, constructionMetrics, workDirectory); } catch (Exception e) { throw new RuntimeException(e); } @@ -582,6 +587,7 @@ private static void testConfiguration(ConfiguredSystem cs, boolean refineFinalGraph, Set featureSetForIndex, RunArtifacts artifacts, + ConstructionMetrics constructionMetrics, Path testDirectory) { int queryRuns = 2; System.out.format("Using %s:%n", cs.index); @@ -625,6 +631,8 @@ private static void testConfiguration(ConfiguredSystem cs, int rerankK = (int) (topK * overquery); var results = tester.run(cs, topK, rerankK, usePruning, queryRuns); + // Merge construction-phase metrics into the runtime results so selections can see them. + constructionMetrics.appendTo(results); // Best-effort runtime availability warnings (warn once per key) consoleSel.warnMissing(results, consoleResolved); @@ -1000,4 +1008,31 @@ public void close() throws Exception { searchers.close(); } } + + /** + * Construction-phase metrics captured while building (and optionally encoding and writing) an index. + *

+ * Values may be {@code null} when not measured or not applicable (e.g., cache hit). Call {@link #appendTo(List)} + * to emit these as {@link Metric} entries so they can be selected for console/CSV reporting. + */ + static final class ConstructionMetrics { + // null means “not applicable / not measured” + public Double encodingTimeS; + public Double indexBuildTimeS; + + ConstructionMetrics() {} + + /** Adds construction metrics as Metric entries (keys always present; null => blank in CSV). */ + void appendTo(List out) { + if (encodingTimeS != null) { + out.add(Metric.of("construction.encoding_time_s", + "Encoding time (s)", ".2f", encodingTimeS)); + } + + if (indexBuildTimeS != null) { + out.add(Metric.of("construction.index_build_time_s", + "Index build time (s)", ".2f", indexBuildTimeS)); + } + } + } } diff --git a/jvector-examples/src/main/java/io/github/jbellis/jvector/example/benchmarks/Metric.java b/jvector-examples/src/main/java/io/github/jbellis/jvector/example/benchmarks/Metric.java index e31c2086c..f62f6d9eb 100644 --- a/jvector-examples/src/main/java/io/github/jbellis/jvector/example/benchmarks/Metric.java +++ b/jvector-examples/src/main/java/io/github/jbellis/jvector/example/benchmarks/Metric.java @@ -50,6 +50,10 @@ public static Metric of(String key, String header, String fmtSpec, double value) @Override public String toString() { - return String.format(header + " = " + fmtSpec, value); + // Create a safe template: "%s = %" + fmtSpec + String template = "%s = %" + fmtSpec; + + // Pass 'header' as the first argument (%s) and 'value' as the second argument (%.1f) + return String.format(template, header, value); } } diff --git a/jvector-examples/src/main/java/io/github/jbellis/jvector/example/benchmarks/QueryTester.java b/jvector-examples/src/main/java/io/github/jbellis/jvector/example/benchmarks/QueryTester.java index 75b451243..34fb21968 100644 --- a/jvector-examples/src/main/java/io/github/jbellis/jvector/example/benchmarks/QueryTester.java +++ b/jvector-examples/src/main/java/io/github/jbellis/jvector/example/benchmarks/QueryTester.java @@ -102,13 +102,25 @@ public List run( var diskSnapshot = diagnostics.getLatestDiskSnapshot(); if (systemSnapshot != null) { - results.add(Metric.of("search.system.max_heap_mb", "Max heap usage", ".1f", + // Max heap usage in MB + results.add(Metric.of("search.system.max_heap_mb", "Max heap usage (MB)", ".1f", systemSnapshot.memoryStats.heapUsed / (1024.0 * 1024.0))); - results.add(Metric.of("search.system.max_offheap_mb", "Max offheap usage", ".1f", + // Max off-heap usage (direct + mapped) in MB + results.add(Metric.of("search.system.max_offheap_mb", "Max offheap usage (MB)", ".1f", systemSnapshot.memoryStats.getTotalOffHeapMemory() / (1024.0 * 1024.0))); } + if (diskSnapshot != null) { + // Number of index files created + results.add(Metric.of("search.disk.file_count", "File count", ".0f", + diskSnapshot.fileCount)); + + // Total size of index files created + results.add(Metric.of("search.disk.total_file_size_mb", "Total file size (MB)", ".1f", + diskSnapshot.totalBytes / (1024.0 * 1024.0))); + } + } catch (IOException e) { throw new RuntimeException(e); } diff --git a/jvector-examples/src/main/java/io/github/jbellis/jvector/example/reporting/SearchReportingCatalog.java b/jvector-examples/src/main/java/io/github/jbellis/jvector/example/reporting/SearchReportingCatalog.java index eb1c74902..3b7bf3229 100644 --- a/jvector-examples/src/main/java/io/github/jbellis/jvector/example/reporting/SearchReportingCatalog.java +++ b/jvector-examples/src/main/java/io/github/jbellis/jvector/example/reporting/SearchReportingCatalog.java @@ -91,7 +91,8 @@ private static Map> namedMetricKeys() { "file_count", "search.disk.file_count" ), "construction", Map.of( - "index_build_time_s", "construction.index_build_time_s" + "index_build_time_s", "construction.index_build_time_s", + "encoding_time_s", "construction.encoding_time_s" ) ); } diff --git a/jvector-examples/yaml-configs/run.yml b/jvector-examples/yaml-configs/run.yml index 4853701bd..4859b6f8d 100644 --- a/jvector-examples/yaml-configs/run.yml +++ b/jvector-examples/yaml-configs/run.yml @@ -35,4 +35,4 @@ logging: # metrics: # system: [ max_heap_mb, max_offheap_mb ] # additional options: [ max_heap_mb, max_offheap_mb ] # disk: [ total_file_size_mb, file_count ] # additional options: [ total_file_size_mb, file_count ] -# construction: [ index_build_time_s ] +# construction: [ encoding_time_s, index_build_time_s ] diff --git a/logging/2026-02-18_02-34-17/dataset_info.csv b/logging/2026-02-18_02-34-17/dataset_info.csv new file mode 100644 index 000000000..55ec38f32 --- /dev/null +++ b/logging/2026-02-18_02-34-17/dataset_info.csv @@ -0,0 +1,4 @@ +schema_version,run_id,dataset_name,dataset_id,base_path,query_path,ground_truth_path,base_count,query_count,ground_truth_count,dimension,similarity_function +1,2026-02-18_02-34-17,cohere-english-v3-100k,3ad6f6e1dd04,/mnt/raid0/jvector-logging/fvec/wikipedia_squad/100k/cohere_embed-english-v3.0_1024_base_vectors_100000.fvec,/mnt/raid0/jvector-logging/fvec/wikipedia_squad/100k/cohere_embed-english-v3.0_1024_query_vectors_10000.fvec,/mnt/raid0/jvector-logging/fvec/wikipedia_squad/100k/cohere_embed-english-v3.0_1024_indices_b100000_q10000_k100.ivec,99740,10000,10000,1024,COSINE +1,2026-02-18_02-34-17,ada002-100k,f5cec7f5c804,/mnt/raid0/jvector-logging/fvec/wikipedia_squad/100k/ada_002_100000_base_vectors.fvec,/mnt/raid0/jvector-logging/fvec/wikipedia_squad/100k/ada_002_100000_query_vectors_10000.fvec,/mnt/raid0/jvector-logging/fvec/wikipedia_squad/100k/ada_002_100000_indices_query_10000.ivec,99562,9753,9753,1536,COSINE +1,2026-02-18_02-34-17,openai-v3-small-100k,b913d76d616a,/mnt/raid0/jvector-logging/fvec/wikipedia_squad/100k/text-embedding-3-small_1536_100000_base_vectors.fvec,/mnt/raid0/jvector-logging/fvec/wikipedia_squad/100k/text-embedding-3-small_1536_100000_query_vectors_10000.fvec,/mnt/raid0/jvector-logging/fvec/wikipedia_squad/100k/text-embedding-3-small_1536_100000_indices_query_10000.ivec,99353,9230,9230,1536,COSINE diff --git a/logging/2026-02-18_02-34-17/experiments.csv b/logging/2026-02-18_02-34-17/experiments.csv new file mode 100644 index 000000000..05e98ec8f --- /dev/null +++ b/logging/2026-02-18_02-34-17/experiments.csv @@ -0,0 +1,13 @@ +schema_version,run_id,run_uuid,system_id,dataset_name,M,efConstruction,neighborOverflow,addHierarchy,refineFinalGraph,feature_set,usePruning,topK,overquery,rerankK,search.throughput.avg_qps,search.throughput.stddev_qps,search.throughput.cv_pct,search.latency.mean_ms,search.count.avg_visited,search.accuracy.recall_at_10,search.accuracy.recall_at_100,search.system.max_heap_mb,search.system.max_offheap_mb,search.disk.total_file_size_mb,search.disk.file_count,construction.index_build_time_s,construction.encoding_time_s +1,2026-02-18_02-34-17,d2634675-0f00-4f54-bee1-19c769f94d7e,d628e0f3aa8e,cohere-english-v3-100k,32,100,1.2,true,true,[NVQ_VECTORS],true,10,1.0,10,137880.69043523297,6121.039498589511,4.439373982874536,0.5192119925500023,359.0251,0.75005,,4630.5832595825195,116.53974437713623,116.48505783081055,1.0,11.317464915,0.268308486 +1,2026-02-18_02-34-17,d2634675-0f00-4f54-bee1-19c769f94d7e,d628e0f3aa8e,cohere-english-v3-100k,32,100,1.2,true,true,[NVQ_VECTORS],true,10,2.0,20,110893.7026773079,16010.048737132667,14.437292966689617,0.5747843745000012,518.4289,0.88632,,1724.5005111694336,116.53974437713623,116.48505783081055,1.0,11.317464915,0.268308486 +1,2026-02-18_02-34-17,d2634675-0f00-4f54-bee1-19c769f94d7e,d628e0f3aa8e,cohere-english-v3-100k,32,100,1.2,true,true,[NVQ_VECTORS],true,100,1.0,100,35486.61925505186,877.8795385890394,2.4738325515865105,1.206743071749995,1536.7348,,0.84955,3727.08056640625,116.53974437713623,116.48505783081055,1.0,11.317464915,0.268308486 +1,2026-02-18_02-34-17,d2634675-0f00-4f54-bee1-19c769f94d7e,d628e0f3aa8e,ada002-100k,32,100,1.2,true,true,[NVQ_VECTORS],true,10,1.0,10,209418.16542447967,19545.29645568491,9.333142813121114,0.35784149277145494,298.0428586076079,0.68594278683482,,3074.8428649902344,1033.3042936325073,0.0,0.0,,0.171918062 +1,2026-02-18_02-34-17,d2634675-0f00-4f54-bee1-19c769f94d7e,d628e0f3aa8e,ada002-100k,32,100,1.2,true,true,[NVQ_VECTORS],true,10,2.0,20,117402.27804744054,36233.162911056694,30.862401917291077,0.44727318676304856,409.92525376807134,0.8619501691787143,,6226.269477844238,1033.3042936325073,0.0,0.0,,0.171918062 +1,2026-02-18_02-34-17,d2634675-0f00-4f54-bee1-19c769f94d7e,d628e0f3aa8e,ada002-100k,32,100,1.2,true,true,[NVQ_VECTORS],true,100,1.0,100,33161.86351213665,1784.4380838220109,5.380994596907795,1.1393342634061348,1092.7116784579105,,0.7818999282272121,3047.906478881836,1033.3042936325073,0.0,0.0,,0.171918062 +1,2026-02-18_02-34-17,d2634675-0f00-4f54-bee1-19c769f94d7e,d628e0f3aa8e,ada002-100k,32,100,1.2,true,true,"[NVQ_VECTORS, FUSED_PQ]",true,10,1.0,10,201357.95088594776,958.7784630781841,0.47615624754805486,0.3263765463447149,297.32010663385626,0.688434327899108,,1991.497703552246,1033.3042936325073,0.0,0.0,,0.171918062 +1,2026-02-18_02-34-17,d2634675-0f00-4f54-bee1-19c769f94d7e,d628e0f3aa8e,ada002-100k,32,100,1.2,true,true,"[NVQ_VECTORS, FUSED_PQ]",true,10,2.0,20,110222.43312696244,33917.45913993111,30.7718294522336,0.4401976440582375,409.4947195734646,0.8652824771865067,,4536.834716796875,1033.3121061325073,0.0,0.0,,0.171918062 +1,2026-02-18_02-34-17,d2634675-0f00-4f54-bee1-19c769f94d7e,d628e0f3aa8e,ada002-100k,32,100,1.2,true,true,"[NVQ_VECTORS, FUSED_PQ]",true,100,1.0,100,31112.193443535554,693.7756262823608,2.2299155073764574,1.098944888136981,1092.217266482108,,0.7821296011483646,5466.7703857421875,1033.3121061325073,0.0,0.0,,0.171918062 +1,2026-02-18_02-34-17,d2634675-0f00-4f54-bee1-19c769f94d7e,d628e0f3aa8e,openai-v3-small-100k,32,100,1.2,true,true,[NVQ_VECTORS],true,10,1.0,10,19764.324524480086,355.26253830730155,1.7974939536500398,1.3911533892199333,334.8011917659805,0.7782881906825568,,1823.7971954345703,1197.8609342575073,164.548828125,1.0,9.754568967,0.155282791 +1,2026-02-18_02-34-17,d2634675-0f00-4f54-bee1-19c769f94d7e,d628e0f3aa8e,openai-v3-small-100k,32,100,1.2,true,true,[NVQ_VECTORS],true,10,2.0,20,18607.034544821472,511.54493291888696,2.7492018230344777,1.4860113169014155,473.6457204767064,0.9313001083423619,,8801.474113464355,1197.8531217575073,164.548828125,1.0,9.754568967,0.155282791 +1,2026-02-18_02-34-17,d2634675-0f00-4f54-bee1-19c769f94d7e,d628e0f3aa8e,openai-v3-small-100k,32,100,1.2,true,true,[NVQ_VECTORS],true,100,1.0,100,12906.165371531191,124.56405961131571,0.9651515847308362,2.20463880509208,1314.7923076923078,,0.8437638136511376,1513.2001876831055,1197.8531217575073,164.548828125,1.0,9.754568967,0.155282791 diff --git a/logging/2026-02-18_02-34-17/sys_info.json b/logging/2026-02-18_02-34-17/sys_info.json new file mode 100644 index 000000000..1df1a7b7a --- /dev/null +++ b/logging/2026-02-18_02-34-17/sys_info.json @@ -0,0 +1 @@ +{"schema_version":1,"run_id":"2026-02-18_02-34-17","run_uuid":"d2634675-0f00-4f54-bee1-19c769f94d7e","created_at":"2026-02-18T02:34:17.686903351Z","jvector_ref":"4.0.0-rc.7-SNAPSHOT","host":{"name":"jvector-benchmark-1b-max","user":"ted_willke"},"os":{"name":"Linux","version":"6.1.0-41-cloud-amd64","arch":"amd64"},"jvm":{"vendor":"Oracle Corporation","version":"23.0.2","vm_name":"OpenJDK 64-Bit Server VM","runtime":"OpenJDK Runtime Environment"},"threads":{"build_executor_parallelism":90,"parallel_stream_parallelism":179,"query_throughput_parallelism":179,"query_latency_threads":1,"fjp_common_parallelism":179},"memory":{"max_heap_bytes":31675383808,"total_physical_bytes":3043900555264},"simd":{"vectorization_provider":"PanamaVectorizationProvider","simd_config_present":true,"configured_vector_bit_size":"512","configured_float_lanes":16},"cpu":{"available_processors":180,"model":"AMD EPYC 9B14","flags":["3dnowprefetch","abm","adx","aes","apic","arat","avx","avx2","avx512_bf16","avx512_bitalg","avx512_vbmi2","avx512_vnni","avx512_vpopcntdq","avx512bw","avx512cd","avx512dq","avx512f","avx512ifma","avx512vbmi","avx512vl","bmi1","bmi2","clflush","clflushopt","clwb","clzero","cmov","cmp_legacy","constant_tsc","cpuid","cr8_legacy","cx16","cx8","de","erms","extd_apicid","f16c","fma","fpu","fsgsbase","fsrm","fxsr","fxsr_opt","gfni","ht","hypervisor","ibpb","ibrs","invpcid","invpcid_single","lahf_lm","lm","mca","mce","misalignsse","mmx","mmxext","movbe","msr","mtrr","nonstop_tsc","nopl","nx","osvw","pae","pat","pcid","pclmulqdq","pdpe1gb","pge","pni","popcnt","pse","pse36","rdpid","rdrand","rdseed","rdtscp","rep_good","sep","sha_ni","smap","smep","ssbd","sse","sse2","sse4_1","sse4_2","sse4a","ssse3","stibp","syscall","topoext","tsc","tsc_adjust","tsc_known_freq","umip","vaes","vme","vmmcall","vpclmulqdq","wbnoinvd","x2apic","xgetbv1","xsave","xsavec","xsaveerptr","xsaveopt","xsaves"]},"system_id":"d628e0f3aa8e"} From 1e779523c48be2056c4fab0306c9b71701a78b64 Mon Sep 17 00:00:00 2001 From: Ted Willke Date: Thu, 19 Feb 2026 03:45:22 +0000 Subject: [PATCH 2/4] Add quantization timing metrics, prefix selection, and logging schema support. Denoise console output. --- .../github/jbellis/jvector/example/Grid.java | 301 +++++++++++++++--- .../benchmarks/BenchmarkTablePrinter.java | 109 +++++-- .../benchmarks/ThroughputBenchmark.java | 8 +- .../reporting/LoggingSchemaPlanner.java | 58 ++++ .../reporting/ReportingSelectionResolver.java | 54 +++- .../reporting/SearchReportingCatalog.java | 5 +- .../example/reporting/SearchSelection.java | 65 +++- .../jvector/example/yaml/MultiConfig.java | 3 - jvector-examples/yaml-configs/run.yml | 34 +- logging/2026-02-18_02-34-17/dataset_info.csv | 4 - logging/2026-02-18_02-34-17/experiments.csv | 13 - logging/2026-02-18_02-34-17/sys_info.json | 1 - 12 files changed, 529 insertions(+), 126 deletions(-) delete mode 100644 logging/2026-02-18_02-34-17/dataset_info.csv delete mode 100644 logging/2026-02-18_02-34-17/experiments.csv delete mode 100644 logging/2026-02-18_02-34-17/sys_info.json diff --git a/jvector-examples/src/main/java/io/github/jbellis/jvector/example/Grid.java b/jvector-examples/src/main/java/io/github/jbellis/jvector/example/Grid.java index 187980b08..a690409e0 100644 --- a/jvector-examples/src/main/java/io/github/jbellis/jvector/example/Grid.java +++ b/jvector-examples/src/main/java/io/github/jbellis/jvector/example/Grid.java @@ -81,6 +81,8 @@ public class Grid { private static final String dirPrefix = "BenchGraphDir"; + private enum Phase { INDEX, SEARCH } + private static final Map indexBuildTimes = new HashMap<>(); private static int diagnostic_level; @@ -112,7 +114,7 @@ static void runAll(DataSet ds, // Always use a fresh temp directory for per-run artifacts final Path workDir = Files.createTempDirectory(dirPrefix); - // Initialize cache (creates stable directory when enabled, cleans up stale temp files; no-op otherwise) + // Initialize index cache (creates stable directory when enabled, cleans up stale temp files; no-op otherwise) final OnDiskGraphIndexCache cache = enableIndexCache ? OnDiskGraphIndexCache.initialize(Paths.get(indexCacheDir)) @@ -125,9 +127,8 @@ static void runAll(DataSet ds, for (float neighborOverflow : neighborOverflowGrid) { for (int efC : efConstructionGrid) { for (var bc : buildCompressors) { - var compressor = getCompressor(bc, ds); runOneGraph(cache, featureSets, M, efC, neighborOverflow, addHierarchy, refineFinalGraph, - compressor, compressionGrid, topKGrid, usePruningGrid, artifacts, ds, workDir); + bc, compressionGrid, topKGrid, usePruningGrid, artifacts, ds, workDir); } } } @@ -217,7 +218,7 @@ static void runOneGraph(OnDiskGraphIndexCache cache, float neighborOverflow, boolean addHierarchy, boolean refineFinalGraph, - VectorCompressor buildCompressor, + Function buildCompressor, List> compressionGrid, Map> topKGrid, List usePruningGrid, @@ -227,15 +228,25 @@ static void runOneGraph(OnDiskGraphIndexCache cache, { // Prepare to collect index construction metrics for reporting.... var constructionMetrics = new ConstructionMetrics(); + // TODO this does not capture disk usage for cached indexes. Need to update // Capture initial memory and disk state try (var diagnostics = new BenchmarkDiagnostics(getDiagnosticLevel())) { diagnostics.setMonitoredDirectory(workDirectory); diagnostics.capturePrePhaseSnapshot("Graph Build"); + // Resolve build compressor (and label quant type) so we can record compute time + VectorCompressor buildCompressorObj = null; + String buildQuantType = null; + + if (buildCompressor != null) { + var buildParams = buildCompressor.apply(ds); + buildQuantType = quantTypeOf(buildParams); // "PQ", "BQ", or null + buildCompressorObj = getCompressor(buildCompressor, ds, constructionMetrics, Phase.INDEX, buildQuantType); + } Map, ImmutableGraphIndex> indexes = new HashMap<>(); - if (buildCompressor == null) { + if (buildCompressorObj == null) { indexes = buildInMemory(featureSets, M, efConstruction, neighborOverflow, addHierarchy, refineFinalGraph, ds, workDirectory); } else { // If cache is disabled, we use the (tmp) workDirectory as the output @@ -246,7 +257,7 @@ static void runOneGraph(OnDiskGraphIndexCache cache, Map, OnDiskGraphIndexCache.WriteHandle> handles = new HashMap<>(); for (Set fs : featureSets) { - var key = cache.key(ds.getName(), fs, M, efConstruction, neighborOverflow, 1.2f, addHierarchy, refineFinalGraph, buildCompressor); + var key = cache.key(ds.getName(), fs, M, efConstruction, neighborOverflow, 1.2f, addHierarchy, refineFinalGraph, buildCompressorObj); var cached = cache.tryLoad(key); if (cached.isPresent()) { @@ -257,8 +268,7 @@ static void runOneGraph(OnDiskGraphIndexCache cache, if (cache.isEnabled()) { var handle = cache.beginWrite(key, OnDiskGraphIndexCache.Overwrite.ALLOW); // Log cache miss / build start - System.out.printf("%s: Building graph index (cached enabled) for %s%n", - key.datasetName, fs); + System.out.printf("%s: Building graph index (cached enabled) for %s%n", key.datasetName, fs); // Prepare the atomic write handle immediately handles.put(fs, handle); } else { @@ -271,7 +281,7 @@ static void runOneGraph(OnDiskGraphIndexCache cache, // At least one index needs to be built (b/c not in cache or cache is disabled) // We pass the handles map so buildOnDisk knows exactly where to write var newIndexes = buildOnDisk(missing, M, efConstruction, neighborOverflow, addHierarchy, refineFinalGraph, - ds, outputDir, buildCompressor, handles); + ds, outputDir, buildCompressorObj, handles, constructionMetrics); indexes.putAll(newIndexes); } } @@ -280,7 +290,7 @@ static void runOneGraph(OnDiskGraphIndexCache cache, diagnostics.capturePostPhaseSnapshot("Graph Build"); diagnostics.printDiskStatistics("Graph Index Build"); - System.out.printf("Index build time: %f seconds%n", Grid.getIndexBuildTimeSeconds(ds.getName())); + System.out.printf("Index build time: %f seconds%n%n", Grid.getIndexBuildTimeSeconds(ds.getName())); constructionMetrics.indexBuildTimeS = Grid.getIndexBuildTimeSeconds(ds.getName()); try { @@ -291,19 +301,21 @@ static void runOneGraph(OnDiskGraphIndexCache cache, CompressedVectors cv; if (featureSetForIndex.contains(FeatureId.FUSED_PQ)) { cv = null; - System.out.format("Fused graph index%n"); } else { - var compressor = getCompressor(cpSupplier, ds); + constructionMetrics.resetSearch(); // per (index, cpSupplier) config + + var searchParams = cpSupplier.apply(ds); + String searchQuantType = quantTypeOf(searchParams); // "PQ", "BQ", or null + var compressor = getCompressor(cpSupplier, ds, constructionMetrics, Phase.SEARCH, searchQuantType); if (compressor == null) { cv = null; - System.out.format("Uncompressed vectors%n"); } else { long start = System.nanoTime(); - cv = compressor.encodeAll(ds.getBaseRavv()); + cv = constructionMetrics.search(searchQuantType) + .timeEncode(() -> compressor.encodeAll(ds.getBaseRavv())); double encodingTimeS = (System.nanoTime() - start) / 1_000_000_000.0; - System.out.format("%s encoded %d vectors [%.2f MB] in %.2fs%n", compressor, ds.getBaseVectors().size(), (cv.ramBytesUsed() / 1024f / 1024f), encodingTimeS); - constructionMetrics.encodingTimeS = encodingTimeS; // Report encoding time + System.out.format("%s: %s encoded %d vectors [%.2f MB] in %.2fs%n", ds.getName(), compressor, ds.getBaseVectors().size(), (cv.ramBytesUsed() / 1024f / 1024f), encodingTimeS); } } @@ -347,14 +359,17 @@ private static Map, ImmutableGraphIndex> buildOnDisk(List buildCompressor, - Map, OnDiskGraphIndexCache.WriteHandle> handles) - throws IOException + Map, OnDiskGraphIndexCache.WriteHandle> handles, + ConstructionMetrics constructionMetrics) throws IOException { Files.createDirectories(outputDir); var floatVectors = ds.getBaseRavv(); - var pq = (PQVectors) buildCompressor.encodeAll(floatVectors); + // Record the encoding time if asked for.... + PQVectors pq = (PQVectors) ((constructionMetrics != null) + ? constructionMetrics.index("PQ").timeEncode(() -> buildCompressor.encodeAll(floatVectors)) + : buildCompressor.encodeAll(floatVectors)); var bsp = BuildScoreProvider.pqBuildScoreProvider(ds.getSimilarityFunction(), pq); GraphIndexBuilder builder = new GraphIndexBuilder(bsp, floatVectors.dimension(), M, efConstruction, neighborOverflow, 1.2f, addHierarchy, refineFinalGraph); @@ -371,7 +386,7 @@ private static Map, ImmutableGraphIndex> buildOnDisk(List features ImmutableGraphIndex onHeapGraph, Path outPath, RandomAccessVectorValues floatVectors, - ProductQuantization pq) + ProductQuantization pq, + ConstructionMetrics constructionMetrics) throws FileNotFoundException { var identityMapper = new OrdinalMapper.IdentityMapper(floatVectors.size() - 1); @@ -477,7 +493,9 @@ private static BuilderWithSuppliers builderWithSuppliers(Set features break; case NVQ_VECTORS: int nSubVectors = floatVectors.dimension() == 2 ? 1 : 2; - var nvq = NVQuantization.compute(floatVectors, nSubVectors); + var nvq = (constructionMetrics != null) + ? constructionMetrics.index("NVQ").timeCompute(() -> NVQuantization.compute(floatVectors, nSubVectors)) + : NVQuantization.compute(floatVectors, nSubVectors); builder.with(new NVQ(nvq)); suppliers.put(FeatureId.NVQ_VECTORS, ordinal -> new NVQ.State(nvq.encode(floatVectors.getVector(ordinal)))); break; @@ -561,7 +579,7 @@ private static Map, ImmutableGraphIndex> buildInMemory(List> benchmarksToCompute = artifacts.benchmarksToCompute(); Map> benchmarksToDisplay = artifacts.benchmarksToDisplay(); @@ -612,20 +630,31 @@ private static void testConfiguration(ConfiguredSystem cs, QueryTester tester = new QueryTester(benchmarks, testDirectory, cs.ds.getName()); // 3) Setup benchmark table for printing - for (var topK : topKGrid.keySet()) { - // Resolving selections depends on topK - var consoleResolved = consoleSel.resolveForTopK(topK); - var logResolved = logSel.resolveForTopK(topK); - - for (var usePruning : usePruningGrid) { - BenchmarkTablePrinter printer = new BenchmarkTablePrinter(); - printer.printConfig(Map.of( + for (var usePruning : usePruningGrid) { + BenchmarkTablePrinter printer = new BenchmarkTablePrinter(); + + // Print the config once per index and query configuration (not per topK) + printer.printConfig( + ordered( // Index configuration + "featureSetForIndex", featureSetForIndex, "M", M, "efConstruction", efConstruction, "neighborOverflow", neighborOverflow, "addHierarchy", addHierarchy, - "usePruning", usePruning - )); + "refineFinalGraph", refineFinalGraph + ), + ordered( // Query configuration + "usePruning", usePruning + ) + ); + + for (var topK : topKGrid.keySet()) { + // Resolving selections depends on topK + var consoleResolved = consoleSel.resolveForTopK(topK); + var logResolved = logSel.resolveForTopK(topK); + + // Force header to re-print for this topK (columns change) + printer.resetTable(); for (var overquery : topKGrid.get(topK)) { int rerankK = (int) (topK * overquery); @@ -748,6 +777,18 @@ private static List setupBenchmarks(Map> be return benchmarks; } + // Helper for printConfig (preserves insertion order) + private static Map ordered(Object... kv) { + if ((kv.length & 1) != 0) + throw new IllegalArgumentException("Need even number of args (k,v)*"); + + var m = new LinkedHashMap(kv.length / 2); + for (int i = 0; i < kv.length; i += 2) { + m.put((String) kv[i], kv[i + 1]); + } + return m; // insertion-ordered + } + public static List runAllAndCollectResults( DataSet ds, boolean enableIndexCache, @@ -821,7 +862,7 @@ public static List runAllAndCollectResults( // At least one index needs to be built (b/c not in cache or cache is disabled) // We pass the handles map so buildOnDisk knows exactly where to write var newIndexes = buildOnDisk(missing, m, ef, neighborOverflow, addHierarchy, refineFinalGraph, - ds, outputDir, compressor, handles); + ds, outputDir, compressor, handles, null); indexes.putAll(newIndexes); } @@ -935,7 +976,7 @@ private static VectorCompressor getCompressor(Function getCompressor(Function getCompressor(FunctionThis overload exists for reporting: when the compressor is constructed (i.e., cache miss), it records + * the compressor build time as a quantization compute metric under either + * {@code construction.index_quant_time_s.*} or {@code search.search_quant_time_s.*}, keyed by {@code quantType} + * (e.g., {@code PQ}, {@code BQ}).

+ * + *

We do not record load times. If a cached compressor is loaded, no compute metric is emitted + * (value remains {@code null} and will be blank in CSV/console).

+ * + * @param cpSupplier supplies dataset-specific compressor parameters (YAML-derived) + * @param ds dataset being benchmarked + * @param metrics per-run construction metrics sink (may be null) + * @param phase which phase to attribute compute time to (index construction vs search-time setup) + * @param quantType YAML quantization type label (e.g., PQ/BQ); if null, no quant metric is recorded + */ + private static VectorCompressor getCompressor(Function cpSupplier, + DataSet ds, + ConstructionMetrics metrics, + Phase phase, + String quantType) { + var cp = cpSupplier.apply(ds); + if (!cp.supportsCaching()) { + // treat this as "compute" if we want to capture it + if (metrics != null && quantType != null) { + var h = (phase == Phase.INDEX) ? metrics.index(quantType) : metrics.search(quantType); + return h.timeCompute(() -> cp.computeCompressor(ds)); + } + return cp.computeCompressor(ds); + } + + var fname = cp.idStringFor(ds); + return cachedCompressors.computeIfAbsent(fname, __ -> { + var path = Paths.get(pqCacheDir).resolve(fname); + if (path.toFile().exists()) { + try (var readerSupplier = ReaderSupplierFactory.open(path); + var rar = readerSupplier.get()) { + // Cache hit: intentionally do not record any timing (no load_time_s); compute_time_s stays null/blank. + var pq = ProductQuantization.load(rar); + System.out.format("%s: %s codebooks loaded from %s%n", ds.getName(), pq, fname); + return pq; + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + // cache miss -> compute (record) + java.util.function.Supplier> compute = () -> { + var start = System.nanoTime(); + var compressor = cp.computeCompressor(ds); + System.out.format("%s: %s computed codebooks in %.2fs,%n", ds.getName(), compressor, (System.nanoTime() - start) / 1_000_000_000.0); + if (cp.supportsCaching()) { + try { + Files.createDirectories(path.getParent()); + try (var writer = new BufferedRandomAccessWriter(path)) { + compressor.write(writer, OnDiskGraphIndex.CURRENT_VERSION); + } + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + return compressor; + }; + + if (metrics != null && quantType != null) { + var h = (phase == Phase.INDEX) ? metrics.index(quantType) : metrics.search(quantType); + return h.timeCompute(compute); + } + return compute.get(); + }); + } + public static class ConfiguredSystem implements AutoCloseable { DataSet ds; ImmutableGraphIndex index; @@ -1009,30 +1123,121 @@ public void close() throws Exception { } } + // Helper that maps YAML-derived CompressorParameters to a stable label + private static String quantTypeOf(CompressorParameters cp) { + if (cp == null || cp == CompressorParameters.NONE) return null; + if (cp instanceof CompressorParameters.PQParameters) return "PQ"; + if (cp instanceof CompressorParameters.BQParameters) return "BQ"; + // Unknown/unsupported type for quant timing purposes + return null; + } + /** - * Construction-phase metrics captured while building (and optionally encoding and writing) an index. - *

- * Values may be {@code null} when not measured or not applicable (e.g., cache hit). Call {@link #appendTo(List)} - * to emit these as {@link Metric} entries so they can be selected for console/CSV reporting. + * Construction/search quantization timing metrics captured while building an index and while encoding + * vectors for search-time evaluation. + * + *

Values may be {@code null} when not measured or not applicable (e.g., cache hit).

+ * + *

Call {@link #appendTo(List)} to emit these as {@link Metric} entries so they can be selected for + * console/CSV reporting.

*/ static final class ConstructionMetrics { // null means “not applicable / not measured” - public Double encodingTimeS; public Double indexBuildTimeS; + // Index-construction phase (single pass per run) + private final Map indexByType = new HashMap<>(); + + // Search phase (reset per evaluated config) + private final Map searchByType = new HashMap<>(); + ConstructionMetrics() {} - /** Adds construction metrics as Metric entries (keys always present; null => blank in CSV). */ - void appendTo(List out) { - if (encodingTimeS != null) { - out.add(Metric.of("construction.encoding_time_s", - "Encoding time (s)", ".2f", encodingTimeS)); - } + /** Clear search-phase quant timings for the next evaluated (index, compression) configuration. */ + void resetSearch() { + searchByType.clear(); + } + + QuantHandle index(String quantType) { return handle(indexByType, quantType); } + QuantHandle search(String quantType) { return handle(searchByType, quantType); } + private static QuantHandle handle(Map map, String quantType) { + if (quantType == null) return QuantHandle.NOOP; + QuantStats qs = map.computeIfAbsent(quantType, __ -> new QuantStats()); + return new QuantHandle(qs); + } + + /** Adds construction + quant metrics as Metric entries (null => blank in CSV). */ + void appendTo(List out) { if (indexBuildTimeS != null) { out.add(Metric.of("construction.index_build_time_s", "Index build time (s)", ".2f", indexBuildTimeS)); } + + // Index-construction quant timings + appendQuant(out, "construction.index_quant_time_s", "Idx", indexByType); + + // Search-phase quant timings (note: search.* namespace matches other query-time metrics) + appendQuant(out, "search.search_quant_time_s", "Search", searchByType); + } + + private static void appendQuant(List out, + String prefix, + String phaseLabel, + Map byType) { + final String hPrefix = (phaseLabel == null || phaseLabel.isBlank()) ? "" : (phaseLabel + " "); + + for (var e : byType.entrySet()) { + String qt = e.getKey(); + QuantStats qs = e.getValue(); + + // Compute + if (qs.computeTimeS != null) { + out.add(Metric.of(prefix + "." + qt + ".compute_time_s", + hPrefix + qt + " compute (s)", ".2f", qs.computeTimeS)); + } + + // Encoding (usually single; suffix only if multiple) + int n = qs.encodeTimesS.size(); + if (n == 1) { + out.add(Metric.of(prefix + "." + qt + ".encoding_time_s", + hPrefix + qt + " encoding (s)", ".2f", qs.encodeTimesS.get(0))); + } else if (n > 1) { + for (int i = 0; i < n; i++) { + out.add(Metric.of(prefix + "." + qt + ".encoding_time_s." + i, + hPrefix + qt + " encoding (s) #" + i, ".2f", qs.encodeTimesS.get(i))); + } + } + } + } + + private static final class QuantStats { + Double computeTimeS; + final List encodeTimesS = new ArrayList<>(); + } + + static final class QuantHandle { + static final QuantHandle NOOP = new QuantHandle(null); + + private final QuantStats qs; + + private QuantHandle(QuantStats qs) { + this.qs = qs; + } + + T timeCompute(java.util.function.Supplier f) { + if (qs == null) return f.get(); + long t0 = System.nanoTime(); + try { return f.get(); } + finally { qs.computeTimeS = (System.nanoTime() - t0) / 1_000_000_000.0; } + } + + T timeEncode(java.util.function.Supplier f) { + if (qs == null) return f.get(); + long t0 = System.nanoTime(); + try { return f.get(); } + finally { qs.encodeTimesS.add((System.nanoTime() - t0) / 1_000_000_000.0); } + } } } } diff --git a/jvector-examples/src/main/java/io/github/jbellis/jvector/example/benchmarks/BenchmarkTablePrinter.java b/jvector-examples/src/main/java/io/github/jbellis/jvector/example/benchmarks/BenchmarkTablePrinter.java index 7b66fa5b8..494644758 100644 --- a/jvector-examples/src/main/java/io/github/jbellis/jvector/example/benchmarks/BenchmarkTablePrinter.java +++ b/jvector-examples/src/main/java/io/github/jbellis/jvector/example/benchmarks/BenchmarkTablePrinter.java @@ -29,39 +29,59 @@ public class BenchmarkTablePrinter { private static final int MIN_COLUMN_WIDTH = 11; private static final int MIN_HEADER_PADDING = 3; + private static final int MAX_COLUMN_WIDTH = 15; // tune as desired private String headerFmt; private String rowFmt; + private int[] colWidths; public BenchmarkTablePrinter() { headerFmt = null; rowFmt = null; } + /** + * Clears header/row formats so the next printed row will emit a fresh header. + * Call this when the set of columns may change (e.g., when topK changes). + */ + public void resetTable() { + headerFmt = null; + rowFmt = null; + colWidths = null; + } private void initializeHeader(List cols) { if (headerFmt != null) { return; } + this.colWidths = new int[cols.size() + 1]; // Build the format strings for header & rows StringBuilder hsb = new StringBuilder(); StringBuilder rsb = new StringBuilder(); // 1) Overquery column width + // Overquery column width hsb.append("%-12s"); rsb.append("%-12.2f"); + colWidths[0] = 12; // 2) One column per Metric + int i = 0; for (Metric m : cols) { String hdr = m.getHeader(); String spec = m.getFmtSpec(); int width = Math.max(MIN_COLUMN_WIDTH, hdr.length() + MIN_HEADER_PADDING); + width = Math.min(width, MAX_COLUMN_WIDTH); + + colWidths[i + 1] = width; // Header: Always a string hsb.append(" %-").append(width).append("s"); // Row: Use the Metric’s fmtSpec (e.g. ".2f", ".3f") rsb.append(" %-").append(width).append(spec); + + i++; } this.headerFmt = hsb.toString(); @@ -72,33 +92,55 @@ private void initializeHeader(List cols) { } /** - * Call this once to print all the global parameters before the table. + * Prints the run-wide configuration header (index settings followed by query settings) + * once per run, before any results table output. * - * @param params a map from parameter name (e.g. "mGrid") to its List value + * Iteration order is preserved (use an insertion-ordered map such as {@link java.util.LinkedHashMap}). + * + * @param indexParams ordered map of index-construction parameters to print + * @param queryParams ordered map of query/search parameters to print */ - public void printConfig(Map params) { - System.out.println(); - System.out.println("Configuration:"); - params.forEach((name, values) -> - System.out.printf(Locale.US, " %-22s: %s%n", name, values) - ); + public void printConfig(Map indexParams, Map queryParams) { + printSection("\nIndex configuration", indexParams); + printSection("\nQuery configuration", queryParams); + } + + private void printSection(String title, Map params) { + System.out.println(title + ":"); + for (var e : params.entrySet()) { + System.out.printf(" %-20s %s%n", e.getKey(), String.valueOf(e.getValue())); + } } private void printHeader(List cols) { - // Prepare array: First "Overquery", then each Metric header - Object[] hdrs = new Object[cols.size() + 1]; - hdrs[0] = "Overquery"; + // Two header lines: split long headers onto a second line when possible + Object[] hdrs1 = new Object[cols.size() + 1]; + Object[] hdrs2 = new Object[cols.size() + 1]; + + hdrs1[0] = "Overquery"; + hdrs2[0] = ""; + + boolean anySecondLine = false; + for (int i = 0; i < cols.size(); i++) { - hdrs[i + 1] = cols.get(i).getHeader(); + String hdr = cols.get(i).getHeader(); + int width = colWidths[i + 1]; + + String[] parts = splitHeader2(hdr, width); + hdrs1[i + 1] = parts[0]; + hdrs2[i + 1] = parts[1]; + if (!parts[1].isEmpty()) anySecondLine = true; } - // Print header line - String line = String.format(Locale.US, headerFmt, hdrs); - System.out.println(line); - // Underline of same length - System.out.println(String.join("", - Collections.nCopies(line.length(), "-") - )); + String line1 = String.format(Locale.US, headerFmt, hdrs1); + System.out.println(line1); + + if (anySecondLine) { + String line2 = String.format(Locale.US, headerFmt, hdrs2); + System.out.println(line2); + } + + System.out.println(String.join("", Collections.nCopies(line1.length(), "-"))); } /** @@ -109,7 +151,7 @@ private void printHeader(List cols) { */ public void printRow(double overquery, List cols) { - initializeHeader(cols); + initializeHeader(cols); // lazy: prints header on the first row after resetTable() // Build argument array: First overquery, then each Metric.extract(...) Object[] vals = new Object[cols.size() + 1]; @@ -129,4 +171,31 @@ public void printRow(double overquery, public void printFooter() { System.out.println(); } + + // Helper for splitting the header into two rows + private static String[] splitHeader2(String hdr, int colWidth) { + if (hdr == null) return new String[] { "", "" }; + + // Manual break: "Line1\nLine2" + int nl = hdr.indexOf('\n'); + if (nl >= 0) { + String a = hdr.substring(0, nl).trim(); + String b = hdr.substring(nl + 1).trim(); + return new String[] { a, b }; + } + + // Leave a little slack for padding/alignment + int max = Math.max(1, colWidth - MIN_HEADER_PADDING); + + String s = hdr.trim(); + if (s.length() <= max) return new String[] { s, "" }; + + // Find last space before max; if none, hard-split + int cut = s.lastIndexOf(' ', max); + if (cut <= 0) cut = max; + + String a = s.substring(0, cut).trim(); + String b = s.substring(cut).trim(); + return new String[] { a, b }; + } } diff --git a/jvector-examples/src/main/java/io/github/jbellis/jvector/example/benchmarks/ThroughputBenchmark.java b/jvector-examples/src/main/java/io/github/jbellis/jvector/example/benchmarks/ThroughputBenchmark.java index a0b9036da..eaca57a17 100644 --- a/jvector-examples/src/main/java/io/github/jbellis/jvector/example/benchmarks/ThroughputBenchmark.java +++ b/jvector-examples/src/main/java/io/github/jbellis/jvector/example/benchmarks/ThroughputBenchmark.java @@ -257,7 +257,7 @@ public List runBenchmark( var list = new ArrayList(); if (computeAvgQps) { list.add(Metric.of("search.throughput.avg_qps", - "Avg QPS (of " + numTestRuns + ")", + "Avg QPS\n (of " + numTestRuns + ")", formatAvgQps, avgQps)); @@ -273,18 +273,18 @@ public List runBenchmark( } if (computeMedianQps) { list.add(Metric.of("search.throughput.median_qps", - "Median QPS (of " + numTestRuns + ")", + "Median QPS\n (of " + numTestRuns + ")", formatMedianQps, medianQps)); } if (computeMaxQps) { list.add(Metric.of("search.throughput.max_qps", - "Max QPS (of " + numTestRuns + ")", + "Max QPS\n (of " + numTestRuns + ")", formatMaxQps, maxQps)); list.add(Metric.of("search.throughput.min_qps", - "Min QPS (of " + numTestRuns + ")", + "Min QPS\n (of " + numTestRuns + ")", formatMaxQps, minQps)); } diff --git a/jvector-examples/src/main/java/io/github/jbellis/jvector/example/reporting/LoggingSchemaPlanner.java b/jvector-examples/src/main/java/io/github/jbellis/jvector/example/reporting/LoggingSchemaPlanner.java index c7cf7288e..70b2b97c2 100644 --- a/jvector-examples/src/main/java/io/github/jbellis/jvector/example/reporting/LoggingSchemaPlanner.java +++ b/jvector-examples/src/main/java/io/github/jbellis/jvector/example/reporting/LoggingSchemaPlanner.java @@ -56,6 +56,7 @@ public static List unionLoggingMetricKeys(RunConfig runCfg, List union, List ordered, String ordered.add(key); } } + + private static void expandPrefixes(Set union, + Set prefixes, + List allConfigs) { + if (prefixes == null || prefixes.isEmpty()) return; + + // Collect quant types from YAML configs (domain is defined by yaml `compression.type` and `reranking`) + Set indexQuantTypes = new HashSet<>(); + Set searchQuantTypes = new HashSet<>(); + boolean wantsNVQ = false; + + for (MultiConfig cfg : allConfigs) { + if (cfg == null) continue; + + // construction compression types (PQ/BQ/None) + if (cfg.construction != null && cfg.construction.compression != null) { + for (var c : cfg.construction.compression) { + if (c != null && c.type != null && !"None".equals(c.type)) { + indexQuantTypes.add(c.type); // "PQ" or "BQ" + } + } + } + + // construction reranking types (FP/NVQ) + if (cfg.construction != null && cfg.construction.reranking != null) { + if (cfg.construction.reranking.contains("NVQ")) { + wantsNVQ = true; + } + } + + // search compression types (PQ/BQ/None) + if (cfg.search != null && cfg.search.compression != null) { + for (var c : cfg.search.compression) { + if (c != null && c.type != null && !"None".equals(c.type)) { + searchQuantTypes.add(c.type); // "PQ" or "BQ" + } + } + } + } + + for (String p : prefixes) { + if ("construction.index_quant_time_s".equals(p)) { + for (String qt : indexQuantTypes) { + union.add(p + "." + qt + ".compute_time_s"); + union.add(p + "." + qt + ".encoding_time_s"); + } + if (wantsNVQ) { + union.add(p + ".NVQ.compute_time_s"); // we intentionally do not time NVQ encode + } + } else if ("search.search_quant_time_s".equals(p)) { + for (String qt : searchQuantTypes) { + union.add(p + "." + qt + ".compute_time_s"); + union.add(p + "." + qt + ".encoding_time_s"); + } + } + } + } } diff --git a/jvector-examples/src/main/java/io/github/jbellis/jvector/example/reporting/ReportingSelectionResolver.java b/jvector-examples/src/main/java/io/github/jbellis/jvector/example/reporting/ReportingSelectionResolver.java index c39f4c4a7..8baa7de4f 100644 --- a/jvector-examples/src/main/java/io/github/jbellis/jvector/example/reporting/ReportingSelectionResolver.java +++ b/jvector-examples/src/main/java/io/github/jbellis/jvector/example/reporting/ReportingSelectionResolver.java @@ -103,20 +103,45 @@ public Catalog(Map>> benchmarkKeyTemplates, /** Result: concrete keys + a YAML-ish label per key for warnings/errors. */ public static final class ResolvedSelection { - private final Set keys; + private final Set keys; // exact Metric.key strings + private final Set prefixes; // prefix selectors like "construction.quant." private final Map keyToSelector; + private final Map prefixToSelector; - public ResolvedSelection(Set keys, Map keyToSelector) { + public ResolvedSelection(Set keys, + Set prefixes, + Map keyToSelector, + Map prefixToSelector) { this.keys = Collections.unmodifiableSet(new HashSet<>(keys)); + this.prefixes = Collections.unmodifiableSet(new HashSet<>(prefixes)); this.keyToSelector = Collections.unmodifiableMap(new HashMap<>(keyToSelector)); + this.prefixToSelector = Collections.unmodifiableMap(new HashMap<>(prefixToSelector)); } + /** Exact selected Metric.key values. */ public Set keys() { return keys; } - /** For warnings/errors: returns something like "search.console.benchmarks.latency.P999". */ + /** Prefix selections. */ + public Set prefixes() { return prefixes; } + + /** True if key is explicitly selected OR matches any selected prefix. */ + public boolean matchesKey(String key) { + if (keys.contains(key)) return true; + for (String p : prefixes) { + if (key.startsWith(p)) return true; + } + return false; + } + + /** For warnings/errors: returns selector if known, else the key itself. */ public String selectorForKey(String key) { return keyToSelector.getOrDefault(key, key); } + + /** For warnings/errors: returns selector if known, else the prefix itself. */ + public String selectorForPrefix(String prefix) { + return prefixToSelector.getOrDefault(prefix, prefix); + } } // ----------------------------- @@ -193,11 +218,13 @@ public static void validateNamedMetricSelectionNames(MetricSelection metricsToSe public static ResolvedSelection resolve(BenchmarkSelection selection, Catalog catalog, Context ctx) { if (selection == null) { - return new ResolvedSelection(Set.of(), Map.of()); + return new ResolvedSelection(Set.of(), Set.of(), Map.of(), Map.of()); } Set keys = new HashSet<>(); + Set prefixes = new HashSet<>(); Map keyToSelector = new HashMap<>(); + Map prefixToSelector = new HashMap<>(); // Benchmarks (type/stat -> templates -> keys) if (selection.benchmarks != null && !selection.benchmarks.isEmpty()) { @@ -220,18 +247,27 @@ public static ResolvedSelection resolve(BenchmarkSelection selection, Catalog ca if (selection.metrics != null && !selection.metrics.isEmpty()) { // caller should validate names pre-build, but keep this defensive validateNamedMetricSelectionNames(selection.metrics, catalog); - + // Recognize @prefix for (var e : selection.metrics.entrySet()) { String category = e.getKey(); for (String name : e.getValue()) { - String k = substitute(catalog.namedMetricKeys.get(category).get(name), ctx); - keys.add(k); - keyToSelector.putIfAbsent(k, catalog.metricsYamlPrefix + "." + category + "." + name); + String raw = catalog.namedMetricKeys.get(category).get(name); + String k = substitute(raw, ctx); + String selector = catalog.metricsYamlPrefix + "." + category + "." + name; + + if (k != null && k.startsWith("@prefix:")) { + String prefix = k.substring("@prefix:".length()); + prefixes.add(prefix); + prefixToSelector.putIfAbsent(prefix, selector); + } else { + keys.add(k); + keyToSelector.putIfAbsent(k, selector); + } } } } - return new ResolvedSelection(keys, keyToSelector); + return new ResolvedSelection(keys, prefixes, keyToSelector, prefixToSelector); } private static List expandBenchmarkStat(String type, String stat, Catalog catalog, Context ctx) { diff --git a/jvector-examples/src/main/java/io/github/jbellis/jvector/example/reporting/SearchReportingCatalog.java b/jvector-examples/src/main/java/io/github/jbellis/jvector/example/reporting/SearchReportingCatalog.java index 3b7bf3229..9f9e4ae9a 100644 --- a/jvector-examples/src/main/java/io/github/jbellis/jvector/example/reporting/SearchReportingCatalog.java +++ b/jvector-examples/src/main/java/io/github/jbellis/jvector/example/reporting/SearchReportingCatalog.java @@ -91,8 +91,9 @@ private static Map> namedMetricKeys() { "file_count", "search.disk.file_count" ), "construction", Map.of( - "index_build_time_s", "construction.index_build_time_s", - "encoding_time_s", "construction.encoding_time_s" + "index_build_time_s","construction.index_build_time_s", + "index_quant_time_s","@prefix:construction.index_quant_time_s", + "search_quant_time_s","@prefix:search.search_quant_time_s" ) ); } diff --git a/jvector-examples/src/main/java/io/github/jbellis/jvector/example/reporting/SearchSelection.java b/jvector-examples/src/main/java/io/github/jbellis/jvector/example/reporting/SearchSelection.java index b95276f04..fbe3e96c4 100644 --- a/jvector-examples/src/main/java/io/github/jbellis/jvector/example/reporting/SearchSelection.java +++ b/jvector-examples/src/main/java/io/github/jbellis/jvector/example/reporting/SearchSelection.java @@ -73,9 +73,12 @@ public ReportingSelectionResolver.ResolvedSelection resolveForTopK(int topK) { /** * Warn once per missing selected key (best-effort availability at runtime). * Uses YAML-ish selectors for readability, and includes the underlying key. + * + * Prefix selections (@prefix:...) are treated as a family: we warn if expected sub-keys + * (compute_time_s / encoding_time_s) are missing for any quantType that appears under that prefix. */ public void warnMissing(List results, ReportingSelectionResolver.ResolvedSelection resolved) { - if (resolved.keys().isEmpty()) { + if ((resolved.keys().isEmpty()) && (resolved.prefixes().isEmpty())) { return; } @@ -84,24 +87,76 @@ public void warnMissing(List results, ReportingSelectionResolver.Resolve present.add(m.getKey()); } + // Warn for missing exact keys for (String k : resolved.keys()) { if (!present.contains(k) && warnedMissingKeys.add(k)) { System.err.println("WARNING: selected " + purpose + " output not available; skipping " + resolved.selectorForKey(k) + " (" + k + ")"); } } + + // Warn for missing sub-keys under each selected prefix, per quantType observed at runtime. + for (String p : resolved.prefixes()) { + String pDot = p + "."; + Map seen = new HashMap<>(); // qt -> [hasCompute, hasAnyEncoding] + + for (String k : present) { + if (!k.startsWith(pDot)) continue; + + int qtStart = pDot.length(); + int qtEnd = k.indexOf('.', qtStart); + if (qtEnd < 0) continue; + + String qt = k.substring(qtStart, qtEnd); + String rest = k.substring(qtEnd + 1); + + boolean[] flags = seen.computeIfAbsent(qt, __ -> new boolean[2]); + if (rest.equals("compute_time_s")) { + flags[0] = true; + } else if (rest.equals("encoding_time_s") || rest.startsWith("encoding_time_s.")) { + flags[1] = true; + } + } + + // For each quant type that showed up under the prefix, warn if compute/encoding missing + for (var e : seen.entrySet()) { + String qt = e.getKey(); + boolean hasCompute = e.getValue()[0]; + boolean hasEnc = e.getValue()[1]; + + String sel = resolved.selectorForPrefix(p); + + if (!hasCompute) { + String missingKey = p + "." + qt + ".compute_time_s"; + if (warnedMissingKeys.add(missingKey)) { + System.err.println("WARNING: selected " + purpose + " output not available; skipping " + + sel + " (" + missingKey + ")"); + } + } + // We don't measure encoding time for NVQ since it is encoded incrementally. + boolean expectEncoding = "PQ".equals(qt) || "BQ".equals(qt); + if (expectEncoding && !hasEnc) { + String missingKey = p + "." + qt + ".encoding_time_s"; + if (warnedMissingKeys.add(missingKey)) { + System.err.println("WARNING: selected " + purpose + " output not available; skipping " + + sel + " (" + missingKey + ")"); + } + } + } + } } - /** Apply the resolved selection as an intersection over Metric.key, preserving original order. */ + /** Apply the resolved selection as an intersection over Metric.key, preserving original order. + * Keep metrics whose key matches exact OR prefix. + * */ public List apply(List results, ReportingSelectionResolver.ResolvedSelection resolved) { - if (resolved.keys().isEmpty()) { + if ((resolved.keys().isEmpty()) && (resolved.prefixes().isEmpty())) { return results; } - Set allowed = resolved.keys(); List out = new ArrayList<>(results.size()); for (Metric m : results) { - if (allowed.contains(m.getKey())) { + if (resolved.matchesKey(m.getKey())) { out.add(m); } } diff --git a/jvector-examples/src/main/java/io/github/jbellis/jvector/example/yaml/MultiConfig.java b/jvector-examples/src/main/java/io/github/jbellis/jvector/example/yaml/MultiConfig.java index 1e55ea4b0..0f2b79805 100644 --- a/jvector-examples/src/main/java/io/github/jbellis/jvector/example/yaml/MultiConfig.java +++ b/jvector-examples/src/main/java/io/github/jbellis/jvector/example/yaml/MultiConfig.java @@ -203,9 +203,6 @@ public static void printDefaultConfigUsageSummary() { var datasets = new java.util.ArrayList<>(DEFAULT_USED_FOR_DATASETS); if (datasets.isEmpty()) return; - System.out.printf("Default YAML config file: %s (used for %d dataset(s))%n", - f.getAbsolutePath(), datasets.size()); - // Print a wrapped bracket-list similar to "Executing the following datasets" System.out.println("Default YAML used for datasets: " + wrapList(datasets, 6, " ")); } diff --git a/jvector-examples/yaml-configs/run.yml b/jvector-examples/yaml-configs/run.yml index 4859b6f8d..8f9eccf96 100644 --- a/jvector-examples/yaml-configs/run.yml +++ b/jvector-examples/yaml-configs/run.yml @@ -4,7 +4,7 @@ onDiskIndexVersion: 6 # Benchmark stats to compute benchmarks: - throughput: [ AVG ] # [AVG, MEDIAN, MAX] + throughput: [ AVG ] # additional options: [AVG, MEDIAN, MAX] latency: [ AVG ] # [AVG, STD, P999] count: [ visited ] # [visited, expanded, expanded base layer] accuracy: [ recall ] # [recall, MAP] @@ -12,27 +12,27 @@ benchmarks: # Console display (omit/leave blank to print all computed outputs) console: benchmarks: - throughput: [ AVG ] - latency: [ AVG ] - count: [ visited ] - accuracy: [ recall ] + throughput: [ AVG ] # [AVG, MEDIAN, MAX] + latency: [ AVG ] # [AVG, STD, P999] + count: [ visited ] # [visited, expanded, expanded base layer] + accuracy: [ recall ] # [recall, MAP] metrics: - system: [ max_heap_mb, max_offheap_mb ] - disk: [ total_file_size_mb, file_count ] - construction: [ index_build_time_s ] + system: [ max_heap_mb ] # [ max_heap_mb, max_offheap_mb ] +# disk: total_file_size_mb # [ total_file_size_mb, file_count ] +# construction: index_build_time_s # [ index_build_time_s, index_quant_time_s, search_quant_time_s ] # Experiments CSV logging (selection + run metadata) -logging: +#logging: # dir: logging # set as desired -# jvectorRef: "no ref" # tag or git sha (recommended) +# jvectorRef: "no-tag" # tag or git sha (recommended) # runId: "my-experiment-{ts}" # optional; template with {ts} (UTC timestamp). Default: "{ts}" # type: csv # only csv is presently supported # benchmarks: -# throughput: [ AVG ] # additional options: [AVG, MEDIAN, MAX] -# latency: [ AVG ] # additional options: [ AVG, STD, P999 ] -# count: [ visited ] # additional options: [ visited, expanded, expanded base layer ] -# accuracy: [ recall ] # additional options: [ recall, MAP ] +# throughput: [ AVG ] # [AVG, MEDIAN, MAX] +# latency: [ AVG ] # [ AVG, STD, P999 ] +# count: [ visited ] # [ visited, expanded, expanded base layer ] +# accuracy: [ recall ] # [ recall, MAP ] # metrics: -# system: [ max_heap_mb, max_offheap_mb ] # additional options: [ max_heap_mb, max_offheap_mb ] -# disk: [ total_file_size_mb, file_count ] # additional options: [ total_file_size_mb, file_count ] -# construction: [ encoding_time_s, index_build_time_s ] +# system: [ max_heap_mb ] # [ max_heap_mb, max_offheap_mb ] +# disk: [ total_file_size_mb, file_count ] # [ total_file_size_mb, file_count ] +# construction: [ index_build_time_s ] # [ index_build_time_s, index_quant_time_s, search_quant_time_s ] diff --git a/logging/2026-02-18_02-34-17/dataset_info.csv b/logging/2026-02-18_02-34-17/dataset_info.csv deleted file mode 100644 index 55ec38f32..000000000 --- a/logging/2026-02-18_02-34-17/dataset_info.csv +++ /dev/null @@ -1,4 +0,0 @@ -schema_version,run_id,dataset_name,dataset_id,base_path,query_path,ground_truth_path,base_count,query_count,ground_truth_count,dimension,similarity_function -1,2026-02-18_02-34-17,cohere-english-v3-100k,3ad6f6e1dd04,/mnt/raid0/jvector-logging/fvec/wikipedia_squad/100k/cohere_embed-english-v3.0_1024_base_vectors_100000.fvec,/mnt/raid0/jvector-logging/fvec/wikipedia_squad/100k/cohere_embed-english-v3.0_1024_query_vectors_10000.fvec,/mnt/raid0/jvector-logging/fvec/wikipedia_squad/100k/cohere_embed-english-v3.0_1024_indices_b100000_q10000_k100.ivec,99740,10000,10000,1024,COSINE -1,2026-02-18_02-34-17,ada002-100k,f5cec7f5c804,/mnt/raid0/jvector-logging/fvec/wikipedia_squad/100k/ada_002_100000_base_vectors.fvec,/mnt/raid0/jvector-logging/fvec/wikipedia_squad/100k/ada_002_100000_query_vectors_10000.fvec,/mnt/raid0/jvector-logging/fvec/wikipedia_squad/100k/ada_002_100000_indices_query_10000.ivec,99562,9753,9753,1536,COSINE -1,2026-02-18_02-34-17,openai-v3-small-100k,b913d76d616a,/mnt/raid0/jvector-logging/fvec/wikipedia_squad/100k/text-embedding-3-small_1536_100000_base_vectors.fvec,/mnt/raid0/jvector-logging/fvec/wikipedia_squad/100k/text-embedding-3-small_1536_100000_query_vectors_10000.fvec,/mnt/raid0/jvector-logging/fvec/wikipedia_squad/100k/text-embedding-3-small_1536_100000_indices_query_10000.ivec,99353,9230,9230,1536,COSINE diff --git a/logging/2026-02-18_02-34-17/experiments.csv b/logging/2026-02-18_02-34-17/experiments.csv deleted file mode 100644 index 05e98ec8f..000000000 --- a/logging/2026-02-18_02-34-17/experiments.csv +++ /dev/null @@ -1,13 +0,0 @@ -schema_version,run_id,run_uuid,system_id,dataset_name,M,efConstruction,neighborOverflow,addHierarchy,refineFinalGraph,feature_set,usePruning,topK,overquery,rerankK,search.throughput.avg_qps,search.throughput.stddev_qps,search.throughput.cv_pct,search.latency.mean_ms,search.count.avg_visited,search.accuracy.recall_at_10,search.accuracy.recall_at_100,search.system.max_heap_mb,search.system.max_offheap_mb,search.disk.total_file_size_mb,search.disk.file_count,construction.index_build_time_s,construction.encoding_time_s -1,2026-02-18_02-34-17,d2634675-0f00-4f54-bee1-19c769f94d7e,d628e0f3aa8e,cohere-english-v3-100k,32,100,1.2,true,true,[NVQ_VECTORS],true,10,1.0,10,137880.69043523297,6121.039498589511,4.439373982874536,0.5192119925500023,359.0251,0.75005,,4630.5832595825195,116.53974437713623,116.48505783081055,1.0,11.317464915,0.268308486 -1,2026-02-18_02-34-17,d2634675-0f00-4f54-bee1-19c769f94d7e,d628e0f3aa8e,cohere-english-v3-100k,32,100,1.2,true,true,[NVQ_VECTORS],true,10,2.0,20,110893.7026773079,16010.048737132667,14.437292966689617,0.5747843745000012,518.4289,0.88632,,1724.5005111694336,116.53974437713623,116.48505783081055,1.0,11.317464915,0.268308486 -1,2026-02-18_02-34-17,d2634675-0f00-4f54-bee1-19c769f94d7e,d628e0f3aa8e,cohere-english-v3-100k,32,100,1.2,true,true,[NVQ_VECTORS],true,100,1.0,100,35486.61925505186,877.8795385890394,2.4738325515865105,1.206743071749995,1536.7348,,0.84955,3727.08056640625,116.53974437713623,116.48505783081055,1.0,11.317464915,0.268308486 -1,2026-02-18_02-34-17,d2634675-0f00-4f54-bee1-19c769f94d7e,d628e0f3aa8e,ada002-100k,32,100,1.2,true,true,[NVQ_VECTORS],true,10,1.0,10,209418.16542447967,19545.29645568491,9.333142813121114,0.35784149277145494,298.0428586076079,0.68594278683482,,3074.8428649902344,1033.3042936325073,0.0,0.0,,0.171918062 -1,2026-02-18_02-34-17,d2634675-0f00-4f54-bee1-19c769f94d7e,d628e0f3aa8e,ada002-100k,32,100,1.2,true,true,[NVQ_VECTORS],true,10,2.0,20,117402.27804744054,36233.162911056694,30.862401917291077,0.44727318676304856,409.92525376807134,0.8619501691787143,,6226.269477844238,1033.3042936325073,0.0,0.0,,0.171918062 -1,2026-02-18_02-34-17,d2634675-0f00-4f54-bee1-19c769f94d7e,d628e0f3aa8e,ada002-100k,32,100,1.2,true,true,[NVQ_VECTORS],true,100,1.0,100,33161.86351213665,1784.4380838220109,5.380994596907795,1.1393342634061348,1092.7116784579105,,0.7818999282272121,3047.906478881836,1033.3042936325073,0.0,0.0,,0.171918062 -1,2026-02-18_02-34-17,d2634675-0f00-4f54-bee1-19c769f94d7e,d628e0f3aa8e,ada002-100k,32,100,1.2,true,true,"[NVQ_VECTORS, FUSED_PQ]",true,10,1.0,10,201357.95088594776,958.7784630781841,0.47615624754805486,0.3263765463447149,297.32010663385626,0.688434327899108,,1991.497703552246,1033.3042936325073,0.0,0.0,,0.171918062 -1,2026-02-18_02-34-17,d2634675-0f00-4f54-bee1-19c769f94d7e,d628e0f3aa8e,ada002-100k,32,100,1.2,true,true,"[NVQ_VECTORS, FUSED_PQ]",true,10,2.0,20,110222.43312696244,33917.45913993111,30.7718294522336,0.4401976440582375,409.4947195734646,0.8652824771865067,,4536.834716796875,1033.3121061325073,0.0,0.0,,0.171918062 -1,2026-02-18_02-34-17,d2634675-0f00-4f54-bee1-19c769f94d7e,d628e0f3aa8e,ada002-100k,32,100,1.2,true,true,"[NVQ_VECTORS, FUSED_PQ]",true,100,1.0,100,31112.193443535554,693.7756262823608,2.2299155073764574,1.098944888136981,1092.217266482108,,0.7821296011483646,5466.7703857421875,1033.3121061325073,0.0,0.0,,0.171918062 -1,2026-02-18_02-34-17,d2634675-0f00-4f54-bee1-19c769f94d7e,d628e0f3aa8e,openai-v3-small-100k,32,100,1.2,true,true,[NVQ_VECTORS],true,10,1.0,10,19764.324524480086,355.26253830730155,1.7974939536500398,1.3911533892199333,334.8011917659805,0.7782881906825568,,1823.7971954345703,1197.8609342575073,164.548828125,1.0,9.754568967,0.155282791 -1,2026-02-18_02-34-17,d2634675-0f00-4f54-bee1-19c769f94d7e,d628e0f3aa8e,openai-v3-small-100k,32,100,1.2,true,true,[NVQ_VECTORS],true,10,2.0,20,18607.034544821472,511.54493291888696,2.7492018230344777,1.4860113169014155,473.6457204767064,0.9313001083423619,,8801.474113464355,1197.8531217575073,164.548828125,1.0,9.754568967,0.155282791 -1,2026-02-18_02-34-17,d2634675-0f00-4f54-bee1-19c769f94d7e,d628e0f3aa8e,openai-v3-small-100k,32,100,1.2,true,true,[NVQ_VECTORS],true,100,1.0,100,12906.165371531191,124.56405961131571,0.9651515847308362,2.20463880509208,1314.7923076923078,,0.8437638136511376,1513.2001876831055,1197.8531217575073,164.548828125,1.0,9.754568967,0.155282791 diff --git a/logging/2026-02-18_02-34-17/sys_info.json b/logging/2026-02-18_02-34-17/sys_info.json deleted file mode 100644 index 1df1a7b7a..000000000 --- a/logging/2026-02-18_02-34-17/sys_info.json +++ /dev/null @@ -1 +0,0 @@ -{"schema_version":1,"run_id":"2026-02-18_02-34-17","run_uuid":"d2634675-0f00-4f54-bee1-19c769f94d7e","created_at":"2026-02-18T02:34:17.686903351Z","jvector_ref":"4.0.0-rc.7-SNAPSHOT","host":{"name":"jvector-benchmark-1b-max","user":"ted_willke"},"os":{"name":"Linux","version":"6.1.0-41-cloud-amd64","arch":"amd64"},"jvm":{"vendor":"Oracle Corporation","version":"23.0.2","vm_name":"OpenJDK 64-Bit Server VM","runtime":"OpenJDK Runtime Environment"},"threads":{"build_executor_parallelism":90,"parallel_stream_parallelism":179,"query_throughput_parallelism":179,"query_latency_threads":1,"fjp_common_parallelism":179},"memory":{"max_heap_bytes":31675383808,"total_physical_bytes":3043900555264},"simd":{"vectorization_provider":"PanamaVectorizationProvider","simd_config_present":true,"configured_vector_bit_size":"512","configured_float_lanes":16},"cpu":{"available_processors":180,"model":"AMD EPYC 9B14","flags":["3dnowprefetch","abm","adx","aes","apic","arat","avx","avx2","avx512_bf16","avx512_bitalg","avx512_vbmi2","avx512_vnni","avx512_vpopcntdq","avx512bw","avx512cd","avx512dq","avx512f","avx512ifma","avx512vbmi","avx512vl","bmi1","bmi2","clflush","clflushopt","clwb","clzero","cmov","cmp_legacy","constant_tsc","cpuid","cr8_legacy","cx16","cx8","de","erms","extd_apicid","f16c","fma","fpu","fsgsbase","fsrm","fxsr","fxsr_opt","gfni","ht","hypervisor","ibpb","ibrs","invpcid","invpcid_single","lahf_lm","lm","mca","mce","misalignsse","mmx","mmxext","movbe","msr","mtrr","nonstop_tsc","nopl","nx","osvw","pae","pat","pcid","pclmulqdq","pdpe1gb","pge","pni","popcnt","pse","pse36","rdpid","rdrand","rdseed","rdtscp","rep_good","sep","sha_ni","smap","smep","ssbd","sse","sse2","sse4_1","sse4_2","sse4a","ssse3","stibp","syscall","topoext","tsc","tsc_adjust","tsc_known_freq","umip","vaes","vme","vmmcall","vpclmulqdq","wbnoinvd","x2apic","xgetbv1","xsave","xsavec","xsaveerptr","xsaveopt","xsaves"]},"system_id":"d628e0f3aa8e"} From 6b194de56812f6d536ca3a4d776dba192e9b9efe Mon Sep 17 00:00:00 2001 From: Ted Willke Date: Thu, 19 Feb 2026 20:43:03 +0000 Subject: [PATCH 3/4] Added default logging directory to .gitignore. --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 95af5f14c..ea443370d 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,9 @@ local/ pq_cache/ index_cache/ +### Logging (or whatever you use) +logging/ + ### JVM crashes hs_err_pid* replay_pid* From 1b83b7c4072dec514fb2d23a9c71c4e912290825 Mon Sep 17 00:00:00 2001 From: Ted Willke Date: Mon, 23 Feb 2026 19:26:54 +0000 Subject: [PATCH 4/4] Refactor overlapping getCompressor overloads into a unified implementation with helpers. --- .../github/jbellis/jvector/example/Grid.java | 129 ++++++++---------- 1 file changed, 54 insertions(+), 75 deletions(-) diff --git a/jvector-examples/src/main/java/io/github/jbellis/jvector/example/Grid.java b/jvector-examples/src/main/java/io/github/jbellis/jvector/example/Grid.java index a690409e0..aadafe58c 100644 --- a/jvector-examples/src/main/java/io/github/jbellis/jvector/example/Grid.java +++ b/jvector-examples/src/main/java/io/github/jbellis/jvector/example/Grid.java @@ -69,6 +69,7 @@ import java.util.function.Function; import java.util.function.IntFunction; import java.util.stream.IntStream; +import java.util.function.Supplier; /** * Tests a grid of configurations against a dataset @@ -962,44 +963,9 @@ public static List runAllAndCollectResults( return results; } + /** Overload for non-reporting use */ private static VectorCompressor getCompressor(Function cpSupplier, DataSet ds) { - var cp = cpSupplier.apply(ds); - if (!cp.supportsCaching()) { - return cp.computeCompressor(ds); - } - - var fname = cp.idStringFor(ds); - return cachedCompressors.computeIfAbsent(fname, __ -> { - var path = Paths.get(pqCacheDir).resolve(fname); - if (path.toFile().exists()) { - try { - try (var readerSupplier = ReaderSupplierFactory.open(path)) { - try (var rar = readerSupplier.get()) { - var pq = ProductQuantization.load(rar); - System.out.format("%s: %s codebooks loaded from %s%n", ds.getName(), pq, fname); - return pq; - } - } - } catch (IOException e) { - throw new UncheckedIOException(e); - } - } - - var start = System.nanoTime(); - var compressor = cp.computeCompressor(ds); - System.out.format("%s: %s codebooks computed in %.2fs,%n", ds.getName(), compressor, (System.nanoTime() - start) / 1_000_000_000.0); - if (cp.supportsCaching()) { - try { - Files.createDirectories(path.getParent()); - try (var writer = new BufferedRandomAccessWriter(path)) { - compressor.write(writer, OnDiskGraphIndex.CURRENT_VERSION); - } - } catch (IOException e) { - throw new UncheckedIOException(e); - } - } - return compressor; - }); + return getCompressor(cpSupplier, ds, null, null, null); } /** @@ -1019,60 +985,73 @@ private static VectorCompressor getCompressor(Function getCompressor(Function cpSupplier, + private static VectorCompressor getCompressor(Function cpSupplier, DataSet ds, ConstructionMetrics metrics, Phase phase, String quantType) { var cp = cpSupplier.apply(ds); - if (!cp.supportsCaching()) { - // treat this as "compute" if we want to capture it + + // Core compute and save logic + Supplier> computeAndSaveAction = () -> { + var start = System.nanoTime(); + var compressor = cp.computeCompressor(ds); + System.out.format("%s: %s codebooks computed in %.2fs,%n", + ds.getName(), compressor, (System.nanoTime() - start) / 1_000_000_000.0); + + if (cp.supportsCaching()) { + saveToCache(cp.idStringFor(ds), compressor); + } + return compressor; + }; + + // Execute (with or without metrics) + Supplier> executionWrapper = () -> { if (metrics != null && quantType != null) { var h = (phase == Phase.INDEX) ? metrics.index(quantType) : metrics.search(quantType); - return h.timeCompute(() -> cp.computeCompressor(ds)); + return h.timeCompute(computeAndSaveAction::get); } - return cp.computeCompressor(ds); + return computeAndSaveAction.get(); + }; + + // No-cache path + if (!cp.supportsCaching()) { + return executionWrapper.get(); } + // Cache path var fname = cp.idStringFor(ds); return cachedCompressors.computeIfAbsent(fname, __ -> { - var path = Paths.get(pqCacheDir).resolve(fname); - if (path.toFile().exists()) { - try (var readerSupplier = ReaderSupplierFactory.open(path); - var rar = readerSupplier.get()) { - // Cache hit: intentionally do not record any timing (no load_time_s); compute_time_s stays null/blank. - var pq = ProductQuantization.load(rar); - System.out.format("%s: %s codebooks loaded from %s%n", ds.getName(), pq, fname); - return pq; - } catch (IOException e) { - throw new UncheckedIOException(e); - } + Path path = Paths.get(pqCacheDir).resolve(fname); + if (Files.exists(path)) { + return loadFromCache(ds, fname, path); } + return executionWrapper.get(); + }); + } - // cache miss -> compute (record) - java.util.function.Supplier> compute = () -> { - var start = System.nanoTime(); - var compressor = cp.computeCompressor(ds); - System.out.format("%s: %s computed codebooks in %.2fs,%n", ds.getName(), compressor, (System.nanoTime() - start) / 1_000_000_000.0); - if (cp.supportsCaching()) { - try { - Files.createDirectories(path.getParent()); - try (var writer = new BufferedRandomAccessWriter(path)) { - compressor.write(writer, OnDiskGraphIndex.CURRENT_VERSION); - } - } catch (IOException e) { - throw new UncheckedIOException(e); - } - } - return compressor; - }; + // Load/save helpers for getCompressor + private static VectorCompressor loadFromCache(DataSet ds, String fname, Path path) { + try (var readerSupplier = ReaderSupplierFactory.open(path); + var rar = readerSupplier.get()) { + var pq = ProductQuantization.load(rar); + System.out.format("%s: %s codebooks loaded from %s%n", ds.getName(), pq, fname); + return pq; + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } - if (metrics != null && quantType != null) { - var h = (phase == Phase.INDEX) ? metrics.index(quantType) : metrics.search(quantType); - return h.timeCompute(compute); + private static void saveToCache(String fname, VectorCompressor compressor) { + try { + Path path = Paths.get(pqCacheDir).resolve(fname); + Files.createDirectories(path.getParent()); + try (var writer = new BufferedRandomAccessWriter(path)) { + compressor.write(writer, OnDiskGraphIndex.CURRENT_VERSION); } - return compute.get(); - }); + } catch (IOException e) { + throw new UncheckedIOException(e); + } } public static class ConfiguredSystem implements AutoCloseable {