Skip to content

Commit f269ff1

Browse files
tlwillketlwillke
committed
Cherry pick the commits and fix merge for adding quantization timing metrics, prefix selection, and logging schema support.
1 parent 5fc4dd3 commit f269ff1

12 files changed

Lines changed: 529 additions & 126 deletions

File tree

‎jvector-examples/src/main/java/io/github/jbellis/jvector/example/Grid.java‎

Lines changed: 253 additions & 48 deletions
Large diffs are not rendered by default.

‎jvector-examples/src/main/java/io/github/jbellis/jvector/example/benchmarks/BenchmarkTablePrinter.java‎

Lines changed: 89 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -29,39 +29,59 @@
2929
public class BenchmarkTablePrinter {
3030
private static final int MIN_COLUMN_WIDTH = 11;
3131
private static final int MIN_HEADER_PADDING = 3;
32+
private static final int MAX_COLUMN_WIDTH = 15; // tune as desired
3233

3334
private String headerFmt;
3435
private String rowFmt;
36+
private int[] colWidths;
3537

3638
public BenchmarkTablePrinter() {
3739
headerFmt = null;
3840
rowFmt = null;
3941
}
4042

43+
/**
44+
* Clears header/row formats so the next printed row will emit a fresh header.
45+
* Call this when the set of columns may change (e.g., when topK changes).
46+
*/
47+
public void resetTable() {
48+
headerFmt = null;
49+
rowFmt = null;
50+
colWidths = null;
51+
}
4152

4253
private void initializeHeader(List<Metric> cols) {
4354
if (headerFmt != null) {
4455
return;
4556
}
57+
this.colWidths = new int[cols.size() + 1];
4658

4759
// Build the format strings for header & rows
4860
StringBuilder hsb = new StringBuilder();
4961
StringBuilder rsb = new StringBuilder();
5062

5163
// 1) Overquery column width
64+
// Overquery column width
5265
hsb.append("%-12s");
5366
rsb.append("%-12.2f");
67+
colWidths[0] = 12;
5468

5569
// 2) One column per Metric
70+
int i = 0;
5671
for (Metric m : cols) {
5772
String hdr = m.getHeader();
5873
String spec = m.getFmtSpec();
5974
int width = Math.max(MIN_COLUMN_WIDTH, hdr.length() + MIN_HEADER_PADDING);
75+
width = Math.min(width, MAX_COLUMN_WIDTH);
76+
77+
colWidths[i + 1] = width;
6078

6179
// Header: Always a string
6280
hsb.append(" %-").append(width).append("s");
6381
// Row: Use the Metric’s fmtSpec (e.g. ".2f", ".3f")
6482
rsb.append(" %-").append(width).append(spec);
83+
84+
i++;
6585
}
6686

6787
this.headerFmt = hsb.toString();
@@ -72,33 +92,55 @@ private void initializeHeader(List<Metric> cols) {
7292
}
7393

7494
/**
75-
* Call this once to print all the global parameters before the table.
95+
* Prints the run-wide configuration header (index settings followed by query settings)
96+
* once per run, before any results table output.
7697
*
77-
* @param params a map from parameter name (e.g. "mGrid") to its List value
98+
* Iteration order is preserved (use an insertion-ordered map such as {@link java.util.LinkedHashMap}).
99+
*
100+
* @param indexParams ordered map of index-construction parameters to print
101+
* @param queryParams ordered map of query/search parameters to print
78102
*/
79-
public void printConfig(Map<String, ?> params) {
80-
System.out.println();
81-
System.out.println("Configuration:");
82-
params.forEach((name, values) ->
83-
System.out.printf(Locale.US, " %-22s: %s%n", name, values)
84-
);
103+
public void printConfig(Map<String, ?> indexParams, Map<String, ?> queryParams) {
104+
printSection("\nIndex configuration", indexParams);
105+
printSection("\nQuery configuration", queryParams);
106+
}
107+
108+
private void printSection(String title, Map<String, ?> params) {
109+
System.out.println(title + ":");
110+
for (var e : params.entrySet()) {
111+
System.out.printf(" %-20s %s%n", e.getKey(), String.valueOf(e.getValue()));
112+
}
85113
}
86114

87115
private void printHeader(List<Metric> cols) {
88-
// Prepare array: First "Overquery", then each Metric header
89-
Object[] hdrs = new Object[cols.size() + 1];
90-
hdrs[0] = "Overquery";
116+
// Two header lines: split long headers onto a second line when possible
117+
Object[] hdrs1 = new Object[cols.size() + 1];
118+
Object[] hdrs2 = new Object[cols.size() + 1];
119+
120+
hdrs1[0] = "Overquery";
121+
hdrs2[0] = "";
122+
123+
boolean anySecondLine = false;
124+
91125
for (int i = 0; i < cols.size(); i++) {
92-
hdrs[i + 1] = cols.get(i).getHeader();
126+
String hdr = cols.get(i).getHeader();
127+
int width = colWidths[i + 1];
128+
129+
String[] parts = splitHeader2(hdr, width);
130+
hdrs1[i + 1] = parts[0];
131+
hdrs2[i + 1] = parts[1];
132+
if (!parts[1].isEmpty()) anySecondLine = true;
93133
}
94134

95-
// Print header line
96-
String line = String.format(Locale.US, headerFmt, hdrs);
97-
System.out.println(line);
98-
// Underline of same length
99-
System.out.println(String.join("",
100-
Collections.nCopies(line.length(), "-")
101-
));
135+
String line1 = String.format(Locale.US, headerFmt, hdrs1);
136+
System.out.println(line1);
137+
138+
if (anySecondLine) {
139+
String line2 = String.format(Locale.US, headerFmt, hdrs2);
140+
System.out.println(line2);
141+
}
142+
143+
System.out.println(String.join("", Collections.nCopies(line1.length(), "-")));
102144
}
103145

104146
/**
@@ -109,7 +151,7 @@ private void printHeader(List<Metric> cols) {
109151
*/
110152
public void printRow(double overquery,
111153
List<Metric> cols) {
112-
initializeHeader(cols);
154+
initializeHeader(cols); // lazy: prints header on the first row after resetTable()
113155

114156
// Build argument array: First overquery, then each Metric.extract(...)
115157
Object[] vals = new Object[cols.size() + 1];
@@ -129,4 +171,31 @@ public void printRow(double overquery,
129171
public void printFooter() {
130172
System.out.println();
131173
}
174+
175+
// Helper for splitting the header into two rows
176+
private static String[] splitHeader2(String hdr, int colWidth) {
177+
if (hdr == null) return new String[] { "", "" };
178+
179+
// Manual break: "Line1\nLine2"
180+
int nl = hdr.indexOf('\n');
181+
if (nl >= 0) {
182+
String a = hdr.substring(0, nl).trim();
183+
String b = hdr.substring(nl + 1).trim();
184+
return new String[] { a, b };
185+
}
186+
187+
// Leave a little slack for padding/alignment
188+
int max = Math.max(1, colWidth - MIN_HEADER_PADDING);
189+
190+
String s = hdr.trim();
191+
if (s.length() <= max) return new String[] { s, "" };
192+
193+
// Find last space before max; if none, hard-split
194+
int cut = s.lastIndexOf(' ', max);
195+
if (cut <= 0) cut = max;
196+
197+
String a = s.substring(0, cut).trim();
198+
String b = s.substring(cut).trim();
199+
return new String[] { a, b };
200+
}
132201
}

‎jvector-examples/src/main/java/io/github/jbellis/jvector/example/benchmarks/ThroughputBenchmark.java‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -257,7 +257,7 @@ public List<Metric> runBenchmark(
257257
var list = new ArrayList<Metric>();
258258
if (computeAvgQps) {
259259
list.add(Metric.of("search.throughput.avg_qps",
260-
"Avg QPS (of " + numTestRuns + ")",
260+
"Avg QPS\n (of " + numTestRuns + ")",
261261
formatAvgQps,
262262
avgQps));
263263

@@ -273,18 +273,18 @@ public List<Metric> runBenchmark(
273273
}
274274
if (computeMedianQps) {
275275
list.add(Metric.of("search.throughput.median_qps",
276-
"Median QPS (of " + numTestRuns + ")",
276+
"Median QPS\n (of " + numTestRuns + ")",
277277
formatMedianQps,
278278
medianQps));
279279
}
280280
if (computeMaxQps) {
281281
list.add(Metric.of("search.throughput.max_qps",
282-
"Max QPS (of " + numTestRuns + ")",
282+
"Max QPS\n (of " + numTestRuns + ")",
283283
formatMaxQps,
284284
maxQps));
285285

286286
list.add(Metric.of("search.throughput.min_qps",
287-
"Min QPS (of " + numTestRuns + ")",
287+
"Min QPS\n (of " + numTestRuns + ")",
288288
formatMaxQps,
289289
minQps));
290290
}

‎jvector-examples/src/main/java/io/github/jbellis/jvector/example/reporting/LoggingSchemaPlanner.java‎

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,7 @@ public static List<String> unionLoggingMetricKeys(RunConfig runCfg, List<MultiCo
5656
var ctx = ReportingSelectionResolver.Context.of("topK", Integer.toString(topK));
5757
var resolved = ReportingSelectionResolver.resolve(runCfg.logging, SearchReportingCatalog.catalog(), ctx);
5858
union.addAll(resolved.keys());
59+
expandPrefixes(union, resolved.prefixes(), allConfigs);
5960
}
6061

6162
// 2) Emit keys in canonical order, only if present in union
@@ -110,4 +111,61 @@ private static void addIfPresent(Set<String> union, List<String> ordered, String
110111
ordered.add(key);
111112
}
112113
}
114+
115+
private static void expandPrefixes(Set<String> union,
116+
Set<String> prefixes,
117+
List<MultiConfig> allConfigs) {
118+
if (prefixes == null || prefixes.isEmpty()) return;
119+
120+
// Collect quant types from YAML configs (domain is defined by yaml `compression.type` and `reranking`)
121+
Set<String> indexQuantTypes = new HashSet<>();
122+
Set<String> searchQuantTypes = new HashSet<>();
123+
boolean wantsNVQ = false;
124+
125+
for (MultiConfig cfg : allConfigs) {
126+
if (cfg == null) continue;
127+
128+
// construction compression types (PQ/BQ/None)
129+
if (cfg.construction != null && cfg.construction.compression != null) {
130+
for (var c : cfg.construction.compression) {
131+
if (c != null && c.type != null && !"None".equals(c.type)) {
132+
indexQuantTypes.add(c.type); // "PQ" or "BQ"
133+
}
134+
}
135+
}
136+
137+
// construction reranking types (FP/NVQ)
138+
if (cfg.construction != null && cfg.construction.reranking != null) {
139+
if (cfg.construction.reranking.contains("NVQ")) {
140+
wantsNVQ = true;
141+
}
142+
}
143+
144+
// search compression types (PQ/BQ/None)
145+
if (cfg.search != null && cfg.search.compression != null) {
146+
for (var c : cfg.search.compression) {
147+
if (c != null && c.type != null && !"None".equals(c.type)) {
148+
searchQuantTypes.add(c.type); // "PQ" or "BQ"
149+
}
150+
}
151+
}
152+
}
153+
154+
for (String p : prefixes) {
155+
if ("construction.index_quant_time_s".equals(p)) {
156+
for (String qt : indexQuantTypes) {
157+
union.add(p + "." + qt + ".compute_time_s");
158+
union.add(p + "." + qt + ".encoding_time_s");
159+
}
160+
if (wantsNVQ) {
161+
union.add(p + ".NVQ.compute_time_s"); // we intentionally do not time NVQ encode
162+
}
163+
} else if ("search.search_quant_time_s".equals(p)) {
164+
for (String qt : searchQuantTypes) {
165+
union.add(p + "." + qt + ".compute_time_s");
166+
union.add(p + "." + qt + ".encoding_time_s");
167+
}
168+
}
169+
}
170+
}
113171
}

‎jvector-examples/src/main/java/io/github/jbellis/jvector/example/reporting/ReportingSelectionResolver.java‎

Lines changed: 45 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -103,20 +103,45 @@ public Catalog(Map<String, Map<String, List<String>>> benchmarkKeyTemplates,
103103

104104
/** Result: concrete keys + a YAML-ish label per key for warnings/errors. */
105105
public static final class ResolvedSelection {
106-
private final Set<String> keys;
106+
private final Set<String> keys; // exact Metric.key strings
107+
private final Set<String> prefixes; // prefix selectors like "construction.quant."
107108
private final Map<String, String> keyToSelector;
109+
private final Map<String, String> prefixToSelector;
108110

109-
public ResolvedSelection(Set<String> keys, Map<String, String> keyToSelector) {
111+
public ResolvedSelection(Set<String> keys,
112+
Set<String> prefixes,
113+
Map<String, String> keyToSelector,
114+
Map<String, String> prefixToSelector) {
110115
this.keys = Collections.unmodifiableSet(new HashSet<>(keys));
116+
this.prefixes = Collections.unmodifiableSet(new HashSet<>(prefixes));
111117
this.keyToSelector = Collections.unmodifiableMap(new HashMap<>(keyToSelector));
118+
this.prefixToSelector = Collections.unmodifiableMap(new HashMap<>(prefixToSelector));
112119
}
113120

121+
/** Exact selected Metric.key values. */
114122
public Set<String> keys() { return keys; }
115123

116-
/** For warnings/errors: returns something like "search.console.benchmarks.latency.P999". */
124+
/** Prefix selections. */
125+
public Set<String> prefixes() { return prefixes; }
126+
127+
/** True if key is explicitly selected OR matches any selected prefix. */
128+
public boolean matchesKey(String key) {
129+
if (keys.contains(key)) return true;
130+
for (String p : prefixes) {
131+
if (key.startsWith(p)) return true;
132+
}
133+
return false;
134+
}
135+
136+
/** For warnings/errors: returns selector if known, else the key itself. */
117137
public String selectorForKey(String key) {
118138
return keyToSelector.getOrDefault(key, key);
119139
}
140+
141+
/** For warnings/errors: returns selector if known, else the prefix itself. */
142+
public String selectorForPrefix(String prefix) {
143+
return prefixToSelector.getOrDefault(prefix, prefix);
144+
}
120145
}
121146

122147
// -----------------------------
@@ -193,11 +218,13 @@ public static void validateNamedMetricSelectionNames(MetricSelection metricsToSe
193218

194219
public static ResolvedSelection resolve(BenchmarkSelection selection, Catalog catalog, Context ctx) {
195220
if (selection == null) {
196-
return new ResolvedSelection(Set.of(), Map.of());
221+
return new ResolvedSelection(Set.of(), Set.of(), Map.of(), Map.of());
197222
}
198223

199224
Set<String> keys = new HashSet<>();
225+
Set<String> prefixes = new HashSet<>();
200226
Map<String, String> keyToSelector = new HashMap<>();
227+
Map<String, String> prefixToSelector = new HashMap<>();
201228

202229
// Benchmarks (type/stat -> templates -> keys)
203230
if (selection.benchmarks != null && !selection.benchmarks.isEmpty()) {
@@ -220,18 +247,27 @@ public static ResolvedSelection resolve(BenchmarkSelection selection, Catalog ca
220247
if (selection.metrics != null && !selection.metrics.isEmpty()) {
221248
// caller should validate names pre-build, but keep this defensive
222249
validateNamedMetricSelectionNames(selection.metrics, catalog);
223-
250+
// Recognize @prefix
224251
for (var e : selection.metrics.entrySet()) {
225252
String category = e.getKey();
226253
for (String name : e.getValue()) {
227-
String k = substitute(catalog.namedMetricKeys.get(category).get(name), ctx);
228-
keys.add(k);
229-
keyToSelector.putIfAbsent(k, catalog.metricsYamlPrefix + "." + category + "." + name);
254+
String raw = catalog.namedMetricKeys.get(category).get(name);
255+
String k = substitute(raw, ctx);
256+
String selector = catalog.metricsYamlPrefix + "." + category + "." + name;
257+
258+
if (k != null && k.startsWith("@prefix:")) {
259+
String prefix = k.substring("@prefix:".length());
260+
prefixes.add(prefix);
261+
prefixToSelector.putIfAbsent(prefix, selector);
262+
} else {
263+
keys.add(k);
264+
keyToSelector.putIfAbsent(k, selector);
265+
}
230266
}
231267
}
232268
}
233269

234-
return new ResolvedSelection(keys, keyToSelector);
270+
return new ResolvedSelection(keys, prefixes, keyToSelector, prefixToSelector);
235271
}
236272

237273
private static List<String> expandBenchmarkStat(String type, String stat, Catalog catalog, Context ctx) {

‎jvector-examples/src/main/java/io/github/jbellis/jvector/example/reporting/SearchReportingCatalog.java‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -91,8 +91,9 @@ private static Map<String, Map<String, String>> namedMetricKeys() {
9191
"file_count", "search.disk.file_count"
9292
),
9393
"construction", Map.of(
94-
"index_build_time_s", "construction.index_build_time_s",
95-
"encoding_time_s", "construction.encoding_time_s"
94+
"index_build_time_s","construction.index_build_time_s",
95+
"index_quant_time_s","@prefix:construction.index_quant_time_s",
96+
"search_quant_time_s","@prefix:search.search_quant_time_s"
9697
)
9798
);
9899
}

0 commit comments

Comments
 (0)