diff --git a/.gitignore b/.gitignore index 95af5f14c..ea443370d 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,9 @@ local/ pq_cache/ index_cache/ +### Logging (or whatever you use) +logging/ + ### JVM crashes hs_err_pid* replay_pid* diff --git a/jvector-examples/src/main/java/io/github/jbellis/jvector/example/Grid.java b/jvector-examples/src/main/java/io/github/jbellis/jvector/example/Grid.java index d5a1e9bb2..aadafe58c 100644 --- a/jvector-examples/src/main/java/io/github/jbellis/jvector/example/Grid.java +++ b/jvector-examples/src/main/java/io/github/jbellis/jvector/example/Grid.java @@ -69,6 +69,7 @@ import java.util.function.Function; import java.util.function.IntFunction; import java.util.stream.IntStream; +import java.util.function.Supplier; /** * Tests a grid of configurations against a dataset @@ -81,6 +82,8 @@ public class Grid { private static final String dirPrefix = "BenchGraphDir"; + private enum Phase { INDEX, SEARCH } + private static final Map indexBuildTimes = new HashMap<>(); private static int diagnostic_level; @@ -112,7 +115,7 @@ static void runAll(DataSet ds, // Always use a fresh temp directory for per-run artifacts final Path workDir = Files.createTempDirectory(dirPrefix); - // Initialize cache (creates stable directory when enabled, cleans up stale temp files; no-op otherwise) + // Initialize index cache (creates stable directory when enabled, cleans up stale temp files; no-op otherwise) final OnDiskGraphIndexCache cache = enableIndexCache ? OnDiskGraphIndexCache.initialize(Paths.get(indexCacheDir)) @@ -125,9 +128,8 @@ static void runAll(DataSet ds, for (float neighborOverflow : neighborOverflowGrid) { for (int efC : efConstructionGrid) { for (var bc : buildCompressors) { - var compressor = getCompressor(bc, ds); runOneGraph(cache, featureSets, M, efC, neighborOverflow, addHierarchy, refineFinalGraph, - compressor, compressionGrid, topKGrid, usePruningGrid, artifacts, ds, workDir); + bc, compressionGrid, topKGrid, usePruningGrid, artifacts, ds, workDir); } } } @@ -217,7 +219,7 @@ static void runOneGraph(OnDiskGraphIndexCache cache, float neighborOverflow, boolean addHierarchy, boolean refineFinalGraph, - VectorCompressor buildCompressor, + Function buildCompressor, List> compressionGrid, Map> topKGrid, List usePruningGrid, @@ -225,15 +227,27 @@ static void runOneGraph(OnDiskGraphIndexCache cache, DataSet ds, Path workDirectory) throws IOException { + // Prepare to collect index construction metrics for reporting.... + var constructionMetrics = new ConstructionMetrics(); + // TODO this does not capture disk usage for cached indexes. Need to update // Capture initial memory and disk state try (var diagnostics = new BenchmarkDiagnostics(getDiagnosticLevel())) { diagnostics.setMonitoredDirectory(workDirectory); diagnostics.capturePrePhaseSnapshot("Graph Build"); + // Resolve build compressor (and label quant type) so we can record compute time + VectorCompressor buildCompressorObj = null; + String buildQuantType = null; + + if (buildCompressor != null) { + var buildParams = buildCompressor.apply(ds); + buildQuantType = quantTypeOf(buildParams); // "PQ", "BQ", or null + buildCompressorObj = getCompressor(buildCompressor, ds, constructionMetrics, Phase.INDEX, buildQuantType); + } Map, ImmutableGraphIndex> indexes = new HashMap<>(); - if (buildCompressor == null) { + if (buildCompressorObj == null) { indexes = buildInMemory(featureSets, M, efConstruction, neighborOverflow, addHierarchy, refineFinalGraph, ds, workDirectory); } else { // If cache is disabled, we use the (tmp) workDirectory as the output @@ -244,7 +258,7 @@ static void runOneGraph(OnDiskGraphIndexCache cache, Map, OnDiskGraphIndexCache.WriteHandle> handles = new HashMap<>(); for (Set fs : featureSets) { - var key = cache.key(ds.getName(), fs, M, efConstruction, neighborOverflow, 1.2f, addHierarchy, refineFinalGraph, buildCompressor); + var key = cache.key(ds.getName(), fs, M, efConstruction, neighborOverflow, 1.2f, addHierarchy, refineFinalGraph, buildCompressorObj); var cached = cache.tryLoad(key); if (cached.isPresent()) { @@ -255,8 +269,7 @@ static void runOneGraph(OnDiskGraphIndexCache cache, if (cache.isEnabled()) { var handle = cache.beginWrite(key, OnDiskGraphIndexCache.Overwrite.ALLOW); // Log cache miss / build start - System.out.printf("%s: Building graph index (cached enabled) for %s%n", - key.datasetName, fs); + System.out.printf("%s: Building graph index (cached enabled) for %s%n", key.datasetName, fs); // Prepare the atomic write handle immediately handles.put(fs, handle); } else { @@ -269,7 +282,7 @@ static void runOneGraph(OnDiskGraphIndexCache cache, // At least one index needs to be built (b/c not in cache or cache is disabled) // We pass the handles map so buildOnDisk knows exactly where to write var newIndexes = buildOnDisk(missing, M, efConstruction, neighborOverflow, addHierarchy, refineFinalGraph, - ds, outputDir, buildCompressor, handles); + ds, outputDir, buildCompressorObj, handles, constructionMetrics); indexes.putAll(newIndexes); } } @@ -278,7 +291,8 @@ static void runOneGraph(OnDiskGraphIndexCache cache, diagnostics.capturePostPhaseSnapshot("Graph Build"); diagnostics.printDiskStatistics("Graph Index Build"); - System.out.printf("Index build time: %f seconds%n", Grid.getIndexBuildTimeSeconds(ds.getName())); + System.out.printf("Index build time: %f seconds%n%n", Grid.getIndexBuildTimeSeconds(ds.getName())); + constructionMetrics.indexBuildTimeS = Grid.getIndexBuildTimeSeconds(ds.getName()); try { for (var cpSupplier : compressionGrid) { @@ -288,23 +302,27 @@ static void runOneGraph(OnDiskGraphIndexCache cache, CompressedVectors cv; if (featureSetForIndex.contains(FeatureId.FUSED_PQ)) { cv = null; - System.out.format("Fused graph index%n"); } else { - var compressor = getCompressor(cpSupplier, ds); + constructionMetrics.resetSearch(); // per (index, cpSupplier) config + + var searchParams = cpSupplier.apply(ds); + String searchQuantType = quantTypeOf(searchParams); // "PQ", "BQ", or null + var compressor = getCompressor(cpSupplier, ds, constructionMetrics, Phase.SEARCH, searchQuantType); if (compressor == null) { cv = null; - System.out.format("Uncompressed vectors%n"); } else { long start = System.nanoTime(); - cv = compressor.encodeAll(ds.getBaseRavv()); - System.out.format("%s encoded %d vectors [%.2f MB] in %.2fs%n", compressor, ds.getBaseVectors().size(), (cv.ramBytesUsed() / 1024f / 1024f), (System.nanoTime() - start) / 1_000_000_000.0); + cv = constructionMetrics.search(searchQuantType) + .timeEncode(() -> compressor.encodeAll(ds.getBaseRavv())); + double encodingTimeS = (System.nanoTime() - start) / 1_000_000_000.0; + System.out.format("%s: %s encoded %d vectors [%.2f MB] in %.2fs%n", ds.getName(), compressor, ds.getBaseVectors().size(), (cv.ramBytesUsed() / 1024f / 1024f), encodingTimeS); } } try (var cs = new ConfiguredSystem(ds, index, cv, featureSetForIndex)) { testConfiguration(cs, topKGrid, usePruningGrid, M, efConstruction, neighborOverflow, addHierarchy, refineFinalGraph, featureSetForIndex, - artifacts, workDirectory); + artifacts, constructionMetrics, workDirectory); } catch (Exception e) { throw new RuntimeException(e); } @@ -342,14 +360,17 @@ private static Map, ImmutableGraphIndex> buildOnDisk(List buildCompressor, - Map, OnDiskGraphIndexCache.WriteHandle> handles) - throws IOException + Map, OnDiskGraphIndexCache.WriteHandle> handles, + ConstructionMetrics constructionMetrics) throws IOException { Files.createDirectories(outputDir); var floatVectors = ds.getBaseRavv(); - var pq = (PQVectors) buildCompressor.encodeAll(floatVectors); + // Record the encoding time if asked for.... + PQVectors pq = (PQVectors) ((constructionMetrics != null) + ? constructionMetrics.index("PQ").timeEncode(() -> buildCompressor.encodeAll(floatVectors)) + : buildCompressor.encodeAll(floatVectors)); var bsp = BuildScoreProvider.pqBuildScoreProvider(ds.getSimilarityFunction(), pq); GraphIndexBuilder builder = new GraphIndexBuilder(bsp, floatVectors.dimension(), M, efConstruction, neighborOverflow, 1.2f, addHierarchy, refineFinalGraph); @@ -366,7 +387,7 @@ private static Map, ImmutableGraphIndex> buildOnDisk(List features ImmutableGraphIndex onHeapGraph, Path outPath, RandomAccessVectorValues floatVectors, - ProductQuantization pq) + ProductQuantization pq, + ConstructionMetrics constructionMetrics) throws FileNotFoundException { var identityMapper = new OrdinalMapper.IdentityMapper(floatVectors.size() - 1); @@ -472,7 +494,9 @@ private static BuilderWithSuppliers builderWithSuppliers(Set features break; case NVQ_VECTORS: int nSubVectors = floatVectors.dimension() == 2 ? 1 : 2; - var nvq = NVQuantization.compute(floatVectors, nSubVectors); + var nvq = (constructionMetrics != null) + ? constructionMetrics.index("NVQ").timeCompute(() -> NVQuantization.compute(floatVectors, nSubVectors)) + : NVQuantization.compute(floatVectors, nSubVectors); builder.with(new NVQ(nvq)); suppliers.put(FeatureId.NVQ_VECTORS, ordinal -> new NVQ.State(nvq.encode(floatVectors.getVector(ordinal)))); break; @@ -556,7 +580,7 @@ private static Map, ImmutableGraphIndex> buildInMemory(List featureSetForIndex, RunArtifacts artifacts, + ConstructionMetrics constructionMetrics, Path testDirectory) { int queryRuns = 2; - System.out.format("Using %s:%n", cs.index); + System.out.format("%s: Using %s:%n", cs.ds.getName(), cs.index); Map> benchmarksToCompute = artifacts.benchmarksToCompute(); Map> benchmarksToDisplay = artifacts.benchmarksToDisplay(); @@ -606,25 +631,38 @@ private static void testConfiguration(ConfiguredSystem cs, QueryTester tester = new QueryTester(benchmarks, testDirectory, cs.ds.getName()); // 3) Setup benchmark table for printing - for (var topK : topKGrid.keySet()) { - // Resolving selections depends on topK - var consoleResolved = consoleSel.resolveForTopK(topK); - var logResolved = logSel.resolveForTopK(topK); - - for (var usePruning : usePruningGrid) { - BenchmarkTablePrinter printer = new BenchmarkTablePrinter(); - printer.printConfig(Map.of( + for (var usePruning : usePruningGrid) { + BenchmarkTablePrinter printer = new BenchmarkTablePrinter(); + + // Print the config once per index and query configuration (not per topK) + printer.printConfig( + ordered( // Index configuration + "featureSetForIndex", featureSetForIndex, "M", M, "efConstruction", efConstruction, "neighborOverflow", neighborOverflow, "addHierarchy", addHierarchy, - "usePruning", usePruning - )); + "refineFinalGraph", refineFinalGraph + ), + ordered( // Query configuration + "usePruning", usePruning + ) + ); + + for (var topK : topKGrid.keySet()) { + // Resolving selections depends on topK + var consoleResolved = consoleSel.resolveForTopK(topK); + var logResolved = logSel.resolveForTopK(topK); + + // Force header to re-print for this topK (columns change) + printer.resetTable(); for (var overquery : topKGrid.get(topK)) { int rerankK = (int) (topK * overquery); var results = tester.run(cs, topK, rerankK, usePruning, queryRuns); + // Merge construction-phase metrics into the runtime results so selections can see them. + constructionMetrics.appendTo(results); // Best-effort runtime availability warnings (warn once per key) consoleSel.warnMissing(results, consoleResolved); @@ -740,6 +778,18 @@ private static List setupBenchmarks(Map> be return benchmarks; } + // Helper for printConfig (preserves insertion order) + private static Map ordered(Object... kv) { + if ((kv.length & 1) != 0) + throw new IllegalArgumentException("Need even number of args (k,v)*"); + + var m = new LinkedHashMap(kv.length / 2); + for (int i = 0; i < kv.length; i += 2) { + m.put((String) kv[i], kv[i + 1]); + } + return m; // insertion-ordered + } + public static List runAllAndCollectResults( DataSet ds, boolean enableIndexCache, @@ -813,7 +863,7 @@ public static List runAllAndCollectResults( // At least one index needs to be built (b/c not in cache or cache is disabled) // We pass the handles map so buildOnDisk knows exactly where to write var newIndexes = buildOnDisk(missing, m, ef, neighborOverflow, addHierarchy, refineFinalGraph, - ds, outputDir, compressor, handles); + ds, outputDir, compressor, handles, null); indexes.putAll(newIndexes); } @@ -913,44 +963,95 @@ public static List runAllAndCollectResults( return results; } + /** Overload for non-reporting use */ private static VectorCompressor getCompressor(Function cpSupplier, DataSet ds) { + return getCompressor(cpSupplier, ds, null, null, null); + } + + /** + * Resolve a {@link VectorCompressor} from YAML {@link CompressorParameters}, with optional instrumentation. + * + *

This overload exists for reporting: when the compressor is constructed (i.e., cache miss), it records + * the compressor build time as a quantization compute metric under either + * {@code construction.index_quant_time_s.*} or {@code search.search_quant_time_s.*}, keyed by {@code quantType} + * (e.g., {@code PQ}, {@code BQ}).

+ * + *

We do not record load times. If a cached compressor is loaded, no compute metric is emitted + * (value remains {@code null} and will be blank in CSV/console).

+ * + * @param cpSupplier supplies dataset-specific compressor parameters (YAML-derived) + * @param ds dataset being benchmarked + * @param metrics per-run construction metrics sink (may be null) + * @param phase which phase to attribute compute time to (index construction vs search-time setup) + * @param quantType YAML quantization type label (e.g., PQ/BQ); if null, no quant metric is recorded + */ + private static VectorCompressor getCompressor(Function cpSupplier, + DataSet ds, + ConstructionMetrics metrics, + Phase phase, + String quantType) { var cp = cpSupplier.apply(ds); + + // Core compute and save logic + Supplier> computeAndSaveAction = () -> { + var start = System.nanoTime(); + var compressor = cp.computeCompressor(ds); + System.out.format("%s: %s codebooks computed in %.2fs,%n", + ds.getName(), compressor, (System.nanoTime() - start) / 1_000_000_000.0); + + if (cp.supportsCaching()) { + saveToCache(cp.idStringFor(ds), compressor); + } + return compressor; + }; + + // Execute (with or without metrics) + Supplier> executionWrapper = () -> { + if (metrics != null && quantType != null) { + var h = (phase == Phase.INDEX) ? metrics.index(quantType) : metrics.search(quantType); + return h.timeCompute(computeAndSaveAction::get); + } + return computeAndSaveAction.get(); + }; + + // No-cache path if (!cp.supportsCaching()) { - return cp.computeCompressor(ds); + return executionWrapper.get(); } + // Cache path var fname = cp.idStringFor(ds); return cachedCompressors.computeIfAbsent(fname, __ -> { - var path = Paths.get(pqCacheDir).resolve(fname); - if (path.toFile().exists()) { - try { - try (var readerSupplier = ReaderSupplierFactory.open(path)) { - try (var rar = readerSupplier.get()) { - var pq = ProductQuantization.load(rar); - System.out.format("%s loaded from %s%n", pq, fname); - return pq; - } - } - } catch (IOException e) { - throw new UncheckedIOException(e); - } + Path path = Paths.get(pqCacheDir).resolve(fname); + if (Files.exists(path)) { + return loadFromCache(ds, fname, path); } + return executionWrapper.get(); + }); + } - var start = System.nanoTime(); - var compressor = cp.computeCompressor(ds); - System.out.format("%s build in %.2fs,%n", compressor, (System.nanoTime() - start) / 1_000_000_000.0); - if (cp.supportsCaching()) { - try { - Files.createDirectories(path.getParent()); - try (var writer = new BufferedRandomAccessWriter(path)) { - compressor.write(writer, OnDiskGraphIndex.CURRENT_VERSION); - } - } catch (IOException e) { - throw new UncheckedIOException(e); - } + // Load/save helpers for getCompressor + private static VectorCompressor loadFromCache(DataSet ds, String fname, Path path) { + try (var readerSupplier = ReaderSupplierFactory.open(path); + var rar = readerSupplier.get()) { + var pq = ProductQuantization.load(rar); + System.out.format("%s: %s codebooks loaded from %s%n", ds.getName(), pq, fname); + return pq; + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + private static void saveToCache(String fname, VectorCompressor compressor) { + try { + Path path = Paths.get(pqCacheDir).resolve(fname); + Files.createDirectories(path.getParent()); + try (var writer = new BufferedRandomAccessWriter(path)) { + compressor.write(writer, OnDiskGraphIndex.CURRENT_VERSION); } - return compressor; - }); + } catch (IOException e) { + throw new UncheckedIOException(e); + } } public static class ConfiguredSystem implements AutoCloseable { @@ -1000,4 +1101,122 @@ public void close() throws Exception { searchers.close(); } } + + // Helper that maps YAML-derived CompressorParameters to a stable label + private static String quantTypeOf(CompressorParameters cp) { + if (cp == null || cp == CompressorParameters.NONE) return null; + if (cp instanceof CompressorParameters.PQParameters) return "PQ"; + if (cp instanceof CompressorParameters.BQParameters) return "BQ"; + // Unknown/unsupported type for quant timing purposes + return null; + } + + /** + * Construction/search quantization timing metrics captured while building an index and while encoding + * vectors for search-time evaluation. + * + *

Values may be {@code null} when not measured or not applicable (e.g., cache hit).

+ * + *

Call {@link #appendTo(List)} to emit these as {@link Metric} entries so they can be selected for + * console/CSV reporting.

+ */ + static final class ConstructionMetrics { + // null means “not applicable / not measured” + public Double indexBuildTimeS; + + // Index-construction phase (single pass per run) + private final Map indexByType = new HashMap<>(); + + // Search phase (reset per evaluated config) + private final Map searchByType = new HashMap<>(); + + ConstructionMetrics() {} + + /** Clear search-phase quant timings for the next evaluated (index, compression) configuration. */ + void resetSearch() { + searchByType.clear(); + } + + QuantHandle index(String quantType) { return handle(indexByType, quantType); } + QuantHandle search(String quantType) { return handle(searchByType, quantType); } + + private static QuantHandle handle(Map map, String quantType) { + if (quantType == null) return QuantHandle.NOOP; + QuantStats qs = map.computeIfAbsent(quantType, __ -> new QuantStats()); + return new QuantHandle(qs); + } + + /** Adds construction + quant metrics as Metric entries (null => blank in CSV). */ + void appendTo(List out) { + if (indexBuildTimeS != null) { + out.add(Metric.of("construction.index_build_time_s", + "Index build time (s)", ".2f", indexBuildTimeS)); + } + + // Index-construction quant timings + appendQuant(out, "construction.index_quant_time_s", "Idx", indexByType); + + // Search-phase quant timings (note: search.* namespace matches other query-time metrics) + appendQuant(out, "search.search_quant_time_s", "Search", searchByType); + } + + private static void appendQuant(List out, + String prefix, + String phaseLabel, + Map byType) { + final String hPrefix = (phaseLabel == null || phaseLabel.isBlank()) ? "" : (phaseLabel + " "); + + for (var e : byType.entrySet()) { + String qt = e.getKey(); + QuantStats qs = e.getValue(); + + // Compute + if (qs.computeTimeS != null) { + out.add(Metric.of(prefix + "." + qt + ".compute_time_s", + hPrefix + qt + " compute (s)", ".2f", qs.computeTimeS)); + } + + // Encoding (usually single; suffix only if multiple) + int n = qs.encodeTimesS.size(); + if (n == 1) { + out.add(Metric.of(prefix + "." + qt + ".encoding_time_s", + hPrefix + qt + " encoding (s)", ".2f", qs.encodeTimesS.get(0))); + } else if (n > 1) { + for (int i = 0; i < n; i++) { + out.add(Metric.of(prefix + "." + qt + ".encoding_time_s." + i, + hPrefix + qt + " encoding (s) #" + i, ".2f", qs.encodeTimesS.get(i))); + } + } + } + } + + private static final class QuantStats { + Double computeTimeS; + final List encodeTimesS = new ArrayList<>(); + } + + static final class QuantHandle { + static final QuantHandle NOOP = new QuantHandle(null); + + private final QuantStats qs; + + private QuantHandle(QuantStats qs) { + this.qs = qs; + } + + T timeCompute(java.util.function.Supplier f) { + if (qs == null) return f.get(); + long t0 = System.nanoTime(); + try { return f.get(); } + finally { qs.computeTimeS = (System.nanoTime() - t0) / 1_000_000_000.0; } + } + + T timeEncode(java.util.function.Supplier f) { + if (qs == null) return f.get(); + long t0 = System.nanoTime(); + try { return f.get(); } + finally { qs.encodeTimesS.add((System.nanoTime() - t0) / 1_000_000_000.0); } + } + } + } } diff --git a/jvector-examples/src/main/java/io/github/jbellis/jvector/example/benchmarks/BenchmarkTablePrinter.java b/jvector-examples/src/main/java/io/github/jbellis/jvector/example/benchmarks/BenchmarkTablePrinter.java index 7b66fa5b8..494644758 100644 --- a/jvector-examples/src/main/java/io/github/jbellis/jvector/example/benchmarks/BenchmarkTablePrinter.java +++ b/jvector-examples/src/main/java/io/github/jbellis/jvector/example/benchmarks/BenchmarkTablePrinter.java @@ -29,39 +29,59 @@ public class BenchmarkTablePrinter { private static final int MIN_COLUMN_WIDTH = 11; private static final int MIN_HEADER_PADDING = 3; + private static final int MAX_COLUMN_WIDTH = 15; // tune as desired private String headerFmt; private String rowFmt; + private int[] colWidths; public BenchmarkTablePrinter() { headerFmt = null; rowFmt = null; } + /** + * Clears header/row formats so the next printed row will emit a fresh header. + * Call this when the set of columns may change (e.g., when topK changes). + */ + public void resetTable() { + headerFmt = null; + rowFmt = null; + colWidths = null; + } private void initializeHeader(List cols) { if (headerFmt != null) { return; } + this.colWidths = new int[cols.size() + 1]; // Build the format strings for header & rows StringBuilder hsb = new StringBuilder(); StringBuilder rsb = new StringBuilder(); // 1) Overquery column width + // Overquery column width hsb.append("%-12s"); rsb.append("%-12.2f"); + colWidths[0] = 12; // 2) One column per Metric + int i = 0; for (Metric m : cols) { String hdr = m.getHeader(); String spec = m.getFmtSpec(); int width = Math.max(MIN_COLUMN_WIDTH, hdr.length() + MIN_HEADER_PADDING); + width = Math.min(width, MAX_COLUMN_WIDTH); + + colWidths[i + 1] = width; // Header: Always a string hsb.append(" %-").append(width).append("s"); // Row: Use the Metric’s fmtSpec (e.g. ".2f", ".3f") rsb.append(" %-").append(width).append(spec); + + i++; } this.headerFmt = hsb.toString(); @@ -72,33 +92,55 @@ private void initializeHeader(List cols) { } /** - * Call this once to print all the global parameters before the table. + * Prints the run-wide configuration header (index settings followed by query settings) + * once per run, before any results table output. * - * @param params a map from parameter name (e.g. "mGrid") to its List value + * Iteration order is preserved (use an insertion-ordered map such as {@link java.util.LinkedHashMap}). + * + * @param indexParams ordered map of index-construction parameters to print + * @param queryParams ordered map of query/search parameters to print */ - public void printConfig(Map params) { - System.out.println(); - System.out.println("Configuration:"); - params.forEach((name, values) -> - System.out.printf(Locale.US, " %-22s: %s%n", name, values) - ); + public void printConfig(Map indexParams, Map queryParams) { + printSection("\nIndex configuration", indexParams); + printSection("\nQuery configuration", queryParams); + } + + private void printSection(String title, Map params) { + System.out.println(title + ":"); + for (var e : params.entrySet()) { + System.out.printf(" %-20s %s%n", e.getKey(), String.valueOf(e.getValue())); + } } private void printHeader(List cols) { - // Prepare array: First "Overquery", then each Metric header - Object[] hdrs = new Object[cols.size() + 1]; - hdrs[0] = "Overquery"; + // Two header lines: split long headers onto a second line when possible + Object[] hdrs1 = new Object[cols.size() + 1]; + Object[] hdrs2 = new Object[cols.size() + 1]; + + hdrs1[0] = "Overquery"; + hdrs2[0] = ""; + + boolean anySecondLine = false; + for (int i = 0; i < cols.size(); i++) { - hdrs[i + 1] = cols.get(i).getHeader(); + String hdr = cols.get(i).getHeader(); + int width = colWidths[i + 1]; + + String[] parts = splitHeader2(hdr, width); + hdrs1[i + 1] = parts[0]; + hdrs2[i + 1] = parts[1]; + if (!parts[1].isEmpty()) anySecondLine = true; } - // Print header line - String line = String.format(Locale.US, headerFmt, hdrs); - System.out.println(line); - // Underline of same length - System.out.println(String.join("", - Collections.nCopies(line.length(), "-") - )); + String line1 = String.format(Locale.US, headerFmt, hdrs1); + System.out.println(line1); + + if (anySecondLine) { + String line2 = String.format(Locale.US, headerFmt, hdrs2); + System.out.println(line2); + } + + System.out.println(String.join("", Collections.nCopies(line1.length(), "-"))); } /** @@ -109,7 +151,7 @@ private void printHeader(List cols) { */ public void printRow(double overquery, List cols) { - initializeHeader(cols); + initializeHeader(cols); // lazy: prints header on the first row after resetTable() // Build argument array: First overquery, then each Metric.extract(...) Object[] vals = new Object[cols.size() + 1]; @@ -129,4 +171,31 @@ public void printRow(double overquery, public void printFooter() { System.out.println(); } + + // Helper for splitting the header into two rows + private static String[] splitHeader2(String hdr, int colWidth) { + if (hdr == null) return new String[] { "", "" }; + + // Manual break: "Line1\nLine2" + int nl = hdr.indexOf('\n'); + if (nl >= 0) { + String a = hdr.substring(0, nl).trim(); + String b = hdr.substring(nl + 1).trim(); + return new String[] { a, b }; + } + + // Leave a little slack for padding/alignment + int max = Math.max(1, colWidth - MIN_HEADER_PADDING); + + String s = hdr.trim(); + if (s.length() <= max) return new String[] { s, "" }; + + // Find last space before max; if none, hard-split + int cut = s.lastIndexOf(' ', max); + if (cut <= 0) cut = max; + + String a = s.substring(0, cut).trim(); + String b = s.substring(cut).trim(); + return new String[] { a, b }; + } } diff --git a/jvector-examples/src/main/java/io/github/jbellis/jvector/example/benchmarks/Metric.java b/jvector-examples/src/main/java/io/github/jbellis/jvector/example/benchmarks/Metric.java index e31c2086c..f62f6d9eb 100644 --- a/jvector-examples/src/main/java/io/github/jbellis/jvector/example/benchmarks/Metric.java +++ b/jvector-examples/src/main/java/io/github/jbellis/jvector/example/benchmarks/Metric.java @@ -50,6 +50,10 @@ public static Metric of(String key, String header, String fmtSpec, double value) @Override public String toString() { - return String.format(header + " = " + fmtSpec, value); + // Create a safe template: "%s = %" + fmtSpec + String template = "%s = %" + fmtSpec; + + // Pass 'header' as the first argument (%s) and 'value' as the second argument (%.1f) + return String.format(template, header, value); } } diff --git a/jvector-examples/src/main/java/io/github/jbellis/jvector/example/benchmarks/QueryTester.java b/jvector-examples/src/main/java/io/github/jbellis/jvector/example/benchmarks/QueryTester.java index 75b451243..34fb21968 100644 --- a/jvector-examples/src/main/java/io/github/jbellis/jvector/example/benchmarks/QueryTester.java +++ b/jvector-examples/src/main/java/io/github/jbellis/jvector/example/benchmarks/QueryTester.java @@ -102,13 +102,25 @@ public List run( var diskSnapshot = diagnostics.getLatestDiskSnapshot(); if (systemSnapshot != null) { - results.add(Metric.of("search.system.max_heap_mb", "Max heap usage", ".1f", + // Max heap usage in MB + results.add(Metric.of("search.system.max_heap_mb", "Max heap usage (MB)", ".1f", systemSnapshot.memoryStats.heapUsed / (1024.0 * 1024.0))); - results.add(Metric.of("search.system.max_offheap_mb", "Max offheap usage", ".1f", + // Max off-heap usage (direct + mapped) in MB + results.add(Metric.of("search.system.max_offheap_mb", "Max offheap usage (MB)", ".1f", systemSnapshot.memoryStats.getTotalOffHeapMemory() / (1024.0 * 1024.0))); } + if (diskSnapshot != null) { + // Number of index files created + results.add(Metric.of("search.disk.file_count", "File count", ".0f", + diskSnapshot.fileCount)); + + // Total size of index files created + results.add(Metric.of("search.disk.total_file_size_mb", "Total file size (MB)", ".1f", + diskSnapshot.totalBytes / (1024.0 * 1024.0))); + } + } catch (IOException e) { throw new RuntimeException(e); } diff --git a/jvector-examples/src/main/java/io/github/jbellis/jvector/example/benchmarks/ThroughputBenchmark.java b/jvector-examples/src/main/java/io/github/jbellis/jvector/example/benchmarks/ThroughputBenchmark.java index a0b9036da..eaca57a17 100644 --- a/jvector-examples/src/main/java/io/github/jbellis/jvector/example/benchmarks/ThroughputBenchmark.java +++ b/jvector-examples/src/main/java/io/github/jbellis/jvector/example/benchmarks/ThroughputBenchmark.java @@ -257,7 +257,7 @@ public List runBenchmark( var list = new ArrayList(); if (computeAvgQps) { list.add(Metric.of("search.throughput.avg_qps", - "Avg QPS (of " + numTestRuns + ")", + "Avg QPS\n (of " + numTestRuns + ")", formatAvgQps, avgQps)); @@ -273,18 +273,18 @@ public List runBenchmark( } if (computeMedianQps) { list.add(Metric.of("search.throughput.median_qps", - "Median QPS (of " + numTestRuns + ")", + "Median QPS\n (of " + numTestRuns + ")", formatMedianQps, medianQps)); } if (computeMaxQps) { list.add(Metric.of("search.throughput.max_qps", - "Max QPS (of " + numTestRuns + ")", + "Max QPS\n (of " + numTestRuns + ")", formatMaxQps, maxQps)); list.add(Metric.of("search.throughput.min_qps", - "Min QPS (of " + numTestRuns + ")", + "Min QPS\n (of " + numTestRuns + ")", formatMaxQps, minQps)); } diff --git a/jvector-examples/src/main/java/io/github/jbellis/jvector/example/reporting/LoggingSchemaPlanner.java b/jvector-examples/src/main/java/io/github/jbellis/jvector/example/reporting/LoggingSchemaPlanner.java index c7cf7288e..70b2b97c2 100644 --- a/jvector-examples/src/main/java/io/github/jbellis/jvector/example/reporting/LoggingSchemaPlanner.java +++ b/jvector-examples/src/main/java/io/github/jbellis/jvector/example/reporting/LoggingSchemaPlanner.java @@ -56,6 +56,7 @@ public static List unionLoggingMetricKeys(RunConfig runCfg, List union, List ordered, String ordered.add(key); } } + + private static void expandPrefixes(Set union, + Set prefixes, + List allConfigs) { + if (prefixes == null || prefixes.isEmpty()) return; + + // Collect quant types from YAML configs (domain is defined by yaml `compression.type` and `reranking`) + Set indexQuantTypes = new HashSet<>(); + Set searchQuantTypes = new HashSet<>(); + boolean wantsNVQ = false; + + for (MultiConfig cfg : allConfigs) { + if (cfg == null) continue; + + // construction compression types (PQ/BQ/None) + if (cfg.construction != null && cfg.construction.compression != null) { + for (var c : cfg.construction.compression) { + if (c != null && c.type != null && !"None".equals(c.type)) { + indexQuantTypes.add(c.type); // "PQ" or "BQ" + } + } + } + + // construction reranking types (FP/NVQ) + if (cfg.construction != null && cfg.construction.reranking != null) { + if (cfg.construction.reranking.contains("NVQ")) { + wantsNVQ = true; + } + } + + // search compression types (PQ/BQ/None) + if (cfg.search != null && cfg.search.compression != null) { + for (var c : cfg.search.compression) { + if (c != null && c.type != null && !"None".equals(c.type)) { + searchQuantTypes.add(c.type); // "PQ" or "BQ" + } + } + } + } + + for (String p : prefixes) { + if ("construction.index_quant_time_s".equals(p)) { + for (String qt : indexQuantTypes) { + union.add(p + "." + qt + ".compute_time_s"); + union.add(p + "." + qt + ".encoding_time_s"); + } + if (wantsNVQ) { + union.add(p + ".NVQ.compute_time_s"); // we intentionally do not time NVQ encode + } + } else if ("search.search_quant_time_s".equals(p)) { + for (String qt : searchQuantTypes) { + union.add(p + "." + qt + ".compute_time_s"); + union.add(p + "." + qt + ".encoding_time_s"); + } + } + } + } } diff --git a/jvector-examples/src/main/java/io/github/jbellis/jvector/example/reporting/ReportingSelectionResolver.java b/jvector-examples/src/main/java/io/github/jbellis/jvector/example/reporting/ReportingSelectionResolver.java index c39f4c4a7..8baa7de4f 100644 --- a/jvector-examples/src/main/java/io/github/jbellis/jvector/example/reporting/ReportingSelectionResolver.java +++ b/jvector-examples/src/main/java/io/github/jbellis/jvector/example/reporting/ReportingSelectionResolver.java @@ -103,20 +103,45 @@ public Catalog(Map>> benchmarkKeyTemplates, /** Result: concrete keys + a YAML-ish label per key for warnings/errors. */ public static final class ResolvedSelection { - private final Set keys; + private final Set keys; // exact Metric.key strings + private final Set prefixes; // prefix selectors like "construction.quant." private final Map keyToSelector; + private final Map prefixToSelector; - public ResolvedSelection(Set keys, Map keyToSelector) { + public ResolvedSelection(Set keys, + Set prefixes, + Map keyToSelector, + Map prefixToSelector) { this.keys = Collections.unmodifiableSet(new HashSet<>(keys)); + this.prefixes = Collections.unmodifiableSet(new HashSet<>(prefixes)); this.keyToSelector = Collections.unmodifiableMap(new HashMap<>(keyToSelector)); + this.prefixToSelector = Collections.unmodifiableMap(new HashMap<>(prefixToSelector)); } + /** Exact selected Metric.key values. */ public Set keys() { return keys; } - /** For warnings/errors: returns something like "search.console.benchmarks.latency.P999". */ + /** Prefix selections. */ + public Set prefixes() { return prefixes; } + + /** True if key is explicitly selected OR matches any selected prefix. */ + public boolean matchesKey(String key) { + if (keys.contains(key)) return true; + for (String p : prefixes) { + if (key.startsWith(p)) return true; + } + return false; + } + + /** For warnings/errors: returns selector if known, else the key itself. */ public String selectorForKey(String key) { return keyToSelector.getOrDefault(key, key); } + + /** For warnings/errors: returns selector if known, else the prefix itself. */ + public String selectorForPrefix(String prefix) { + return prefixToSelector.getOrDefault(prefix, prefix); + } } // ----------------------------- @@ -193,11 +218,13 @@ public static void validateNamedMetricSelectionNames(MetricSelection metricsToSe public static ResolvedSelection resolve(BenchmarkSelection selection, Catalog catalog, Context ctx) { if (selection == null) { - return new ResolvedSelection(Set.of(), Map.of()); + return new ResolvedSelection(Set.of(), Set.of(), Map.of(), Map.of()); } Set keys = new HashSet<>(); + Set prefixes = new HashSet<>(); Map keyToSelector = new HashMap<>(); + Map prefixToSelector = new HashMap<>(); // Benchmarks (type/stat -> templates -> keys) if (selection.benchmarks != null && !selection.benchmarks.isEmpty()) { @@ -220,18 +247,27 @@ public static ResolvedSelection resolve(BenchmarkSelection selection, Catalog ca if (selection.metrics != null && !selection.metrics.isEmpty()) { // caller should validate names pre-build, but keep this defensive validateNamedMetricSelectionNames(selection.metrics, catalog); - + // Recognize @prefix for (var e : selection.metrics.entrySet()) { String category = e.getKey(); for (String name : e.getValue()) { - String k = substitute(catalog.namedMetricKeys.get(category).get(name), ctx); - keys.add(k); - keyToSelector.putIfAbsent(k, catalog.metricsYamlPrefix + "." + category + "." + name); + String raw = catalog.namedMetricKeys.get(category).get(name); + String k = substitute(raw, ctx); + String selector = catalog.metricsYamlPrefix + "." + category + "." + name; + + if (k != null && k.startsWith("@prefix:")) { + String prefix = k.substring("@prefix:".length()); + prefixes.add(prefix); + prefixToSelector.putIfAbsent(prefix, selector); + } else { + keys.add(k); + keyToSelector.putIfAbsent(k, selector); + } } } } - return new ResolvedSelection(keys, keyToSelector); + return new ResolvedSelection(keys, prefixes, keyToSelector, prefixToSelector); } private static List expandBenchmarkStat(String type, String stat, Catalog catalog, Context ctx) { diff --git a/jvector-examples/src/main/java/io/github/jbellis/jvector/example/reporting/SearchReportingCatalog.java b/jvector-examples/src/main/java/io/github/jbellis/jvector/example/reporting/SearchReportingCatalog.java index eb1c74902..9f9e4ae9a 100644 --- a/jvector-examples/src/main/java/io/github/jbellis/jvector/example/reporting/SearchReportingCatalog.java +++ b/jvector-examples/src/main/java/io/github/jbellis/jvector/example/reporting/SearchReportingCatalog.java @@ -91,7 +91,9 @@ private static Map> namedMetricKeys() { "file_count", "search.disk.file_count" ), "construction", Map.of( - "index_build_time_s", "construction.index_build_time_s" + "index_build_time_s","construction.index_build_time_s", + "index_quant_time_s","@prefix:construction.index_quant_time_s", + "search_quant_time_s","@prefix:search.search_quant_time_s" ) ); } diff --git a/jvector-examples/src/main/java/io/github/jbellis/jvector/example/reporting/SearchSelection.java b/jvector-examples/src/main/java/io/github/jbellis/jvector/example/reporting/SearchSelection.java index b95276f04..fbe3e96c4 100644 --- a/jvector-examples/src/main/java/io/github/jbellis/jvector/example/reporting/SearchSelection.java +++ b/jvector-examples/src/main/java/io/github/jbellis/jvector/example/reporting/SearchSelection.java @@ -73,9 +73,12 @@ public ReportingSelectionResolver.ResolvedSelection resolveForTopK(int topK) { /** * Warn once per missing selected key (best-effort availability at runtime). * Uses YAML-ish selectors for readability, and includes the underlying key. + * + * Prefix selections (@prefix:...) are treated as a family: we warn if expected sub-keys + * (compute_time_s / encoding_time_s) are missing for any quantType that appears under that prefix. */ public void warnMissing(List results, ReportingSelectionResolver.ResolvedSelection resolved) { - if (resolved.keys().isEmpty()) { + if ((resolved.keys().isEmpty()) && (resolved.prefixes().isEmpty())) { return; } @@ -84,24 +87,76 @@ public void warnMissing(List results, ReportingSelectionResolver.Resolve present.add(m.getKey()); } + // Warn for missing exact keys for (String k : resolved.keys()) { if (!present.contains(k) && warnedMissingKeys.add(k)) { System.err.println("WARNING: selected " + purpose + " output not available; skipping " + resolved.selectorForKey(k) + " (" + k + ")"); } } + + // Warn for missing sub-keys under each selected prefix, per quantType observed at runtime. + for (String p : resolved.prefixes()) { + String pDot = p + "."; + Map seen = new HashMap<>(); // qt -> [hasCompute, hasAnyEncoding] + + for (String k : present) { + if (!k.startsWith(pDot)) continue; + + int qtStart = pDot.length(); + int qtEnd = k.indexOf('.', qtStart); + if (qtEnd < 0) continue; + + String qt = k.substring(qtStart, qtEnd); + String rest = k.substring(qtEnd + 1); + + boolean[] flags = seen.computeIfAbsent(qt, __ -> new boolean[2]); + if (rest.equals("compute_time_s")) { + flags[0] = true; + } else if (rest.equals("encoding_time_s") || rest.startsWith("encoding_time_s.")) { + flags[1] = true; + } + } + + // For each quant type that showed up under the prefix, warn if compute/encoding missing + for (var e : seen.entrySet()) { + String qt = e.getKey(); + boolean hasCompute = e.getValue()[0]; + boolean hasEnc = e.getValue()[1]; + + String sel = resolved.selectorForPrefix(p); + + if (!hasCompute) { + String missingKey = p + "." + qt + ".compute_time_s"; + if (warnedMissingKeys.add(missingKey)) { + System.err.println("WARNING: selected " + purpose + " output not available; skipping " + + sel + " (" + missingKey + ")"); + } + } + // We don't measure encoding time for NVQ since it is encoded incrementally. + boolean expectEncoding = "PQ".equals(qt) || "BQ".equals(qt); + if (expectEncoding && !hasEnc) { + String missingKey = p + "." + qt + ".encoding_time_s"; + if (warnedMissingKeys.add(missingKey)) { + System.err.println("WARNING: selected " + purpose + " output not available; skipping " + + sel + " (" + missingKey + ")"); + } + } + } + } } - /** Apply the resolved selection as an intersection over Metric.key, preserving original order. */ + /** Apply the resolved selection as an intersection over Metric.key, preserving original order. + * Keep metrics whose key matches exact OR prefix. + * */ public List apply(List results, ReportingSelectionResolver.ResolvedSelection resolved) { - if (resolved.keys().isEmpty()) { + if ((resolved.keys().isEmpty()) && (resolved.prefixes().isEmpty())) { return results; } - Set allowed = resolved.keys(); List out = new ArrayList<>(results.size()); for (Metric m : results) { - if (allowed.contains(m.getKey())) { + if (resolved.matchesKey(m.getKey())) { out.add(m); } } diff --git a/jvector-examples/src/main/java/io/github/jbellis/jvector/example/yaml/MultiConfig.java b/jvector-examples/src/main/java/io/github/jbellis/jvector/example/yaml/MultiConfig.java index 1e55ea4b0..0f2b79805 100644 --- a/jvector-examples/src/main/java/io/github/jbellis/jvector/example/yaml/MultiConfig.java +++ b/jvector-examples/src/main/java/io/github/jbellis/jvector/example/yaml/MultiConfig.java @@ -203,9 +203,6 @@ public static void printDefaultConfigUsageSummary() { var datasets = new java.util.ArrayList<>(DEFAULT_USED_FOR_DATASETS); if (datasets.isEmpty()) return; - System.out.printf("Default YAML config file: %s (used for %d dataset(s))%n", - f.getAbsolutePath(), datasets.size()); - // Print a wrapped bracket-list similar to "Executing the following datasets" System.out.println("Default YAML used for datasets: " + wrapList(datasets, 6, " ")); } diff --git a/jvector-examples/yaml-configs/run.yml b/jvector-examples/yaml-configs/run.yml index 4853701bd..8f9eccf96 100644 --- a/jvector-examples/yaml-configs/run.yml +++ b/jvector-examples/yaml-configs/run.yml @@ -4,7 +4,7 @@ onDiskIndexVersion: 6 # Benchmark stats to compute benchmarks: - throughput: [ AVG ] # [AVG, MEDIAN, MAX] + throughput: [ AVG ] # additional options: [AVG, MEDIAN, MAX] latency: [ AVG ] # [AVG, STD, P999] count: [ visited ] # [visited, expanded, expanded base layer] accuracy: [ recall ] # [recall, MAP] @@ -12,27 +12,27 @@ benchmarks: # Console display (omit/leave blank to print all computed outputs) console: benchmarks: - throughput: [ AVG ] - latency: [ AVG ] - count: [ visited ] - accuracy: [ recall ] + throughput: [ AVG ] # [AVG, MEDIAN, MAX] + latency: [ AVG ] # [AVG, STD, P999] + count: [ visited ] # [visited, expanded, expanded base layer] + accuracy: [ recall ] # [recall, MAP] metrics: - system: [ max_heap_mb, max_offheap_mb ] - disk: [ total_file_size_mb, file_count ] - construction: [ index_build_time_s ] + system: [ max_heap_mb ] # [ max_heap_mb, max_offheap_mb ] +# disk: total_file_size_mb # [ total_file_size_mb, file_count ] +# construction: index_build_time_s # [ index_build_time_s, index_quant_time_s, search_quant_time_s ] # Experiments CSV logging (selection + run metadata) -logging: +#logging: # dir: logging # set as desired -# jvectorRef: "no ref" # tag or git sha (recommended) +# jvectorRef: "no-tag" # tag or git sha (recommended) # runId: "my-experiment-{ts}" # optional; template with {ts} (UTC timestamp). Default: "{ts}" # type: csv # only csv is presently supported # benchmarks: -# throughput: [ AVG ] # additional options: [AVG, MEDIAN, MAX] -# latency: [ AVG ] # additional options: [ AVG, STD, P999 ] -# count: [ visited ] # additional options: [ visited, expanded, expanded base layer ] -# accuracy: [ recall ] # additional options: [ recall, MAP ] +# throughput: [ AVG ] # [AVG, MEDIAN, MAX] +# latency: [ AVG ] # [ AVG, STD, P999 ] +# count: [ visited ] # [ visited, expanded, expanded base layer ] +# accuracy: [ recall ] # [ recall, MAP ] # metrics: -# system: [ max_heap_mb, max_offheap_mb ] # additional options: [ max_heap_mb, max_offheap_mb ] -# disk: [ total_file_size_mb, file_count ] # additional options: [ total_file_size_mb, file_count ] -# construction: [ index_build_time_s ] +# system: [ max_heap_mb ] # [ max_heap_mb, max_offheap_mb ] +# disk: [ total_file_size_mb, file_count ] # [ total_file_size_mb, file_count ] +# construction: [ index_build_time_s ] # [ index_build_time_s, index_quant_time_s, search_quant_time_s ]