Skip to content

Multidirectory #634

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
MarkWolters merged 5 commits into from
Feb 24, 2026
Merged

Multidirectory #634

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
7135b97
monitor both test and cache directories
MarkWolters Feb 17, 2026
0b7e42e
resource management
MarkWolters Feb 17, 2026
e8b850f
rebase on main
MarkWolters Feb 17, 2026
a8fc43d
Merge branch 'main' into multidirectory
MarkWolters Feb 18, 2026
d6c5c27
rebase on main
MarkWolters Feb 24, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 8 additions & 6 deletions jvector-examples/src/main/java/io/github/jbellis/jvector/example/Grid.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,12 +29,12 @@
import io.github.jbellis.jvector.example.benchmarks.diagnostics.BenchmarkDiagnostics;
import io.github.jbellis.jvector.example.benchmarks.diagnostics.DiagnosticLevel;
import io.github.jbellis.jvector.example.benchmarks.datasets.DataSet;
import io.github.jbellis.jvector.example.benchmarks.diagnostics.DiskUsageMonitor;
import io.github.jbellis.jvector.example.reporting.*;
import io.github.jbellis.jvector.example.reporting.RunArtifacts;
import io.github.jbellis.jvector.example.util.CompressorParameters;
import io.github.jbellis.jvector.example.util.FilteredForkJoinPool;
import io.github.jbellis.jvector.example.util.OnDiskGraphIndexCache;
import io.github.jbellis.jvector.example.yaml.BenchmarkSelection;
import io.github.jbellis.jvector.example.yaml.MetricSelection;
import io.github.jbellis.jvector.graph.ImmutableGraphIndex;
import io.github.jbellis.jvector.graph.GraphIndexBuilder;
Expand Down Expand Up @@ -233,7 +233,8 @@ static void runOneGraph(OnDiskGraphIndexCache cache,
// TODO this does not capture disk usage for cached indexes. Need to update
// Capture initial memory and disk state
try (var diagnostics = new BenchmarkDiagnostics(getDiagnosticLevel())) {
diagnostics.setMonitoredDirectory(workDirectory);
diagnostics.startMonitoring("testDirectory", workDirectory);
diagnostics.startMonitoring("indexCache", Paths.get(indexCacheDir));
diagnostics.capturePrePhaseSnapshot("Graph Build");

// Resolve build compressor (and label quant type) so we can record compute time
Expand Down Expand Up @@ -822,7 +823,8 @@ public static List<BenchResult> runAllAndCollectResults(
Path testDirectory = Files.createTempDirectory("bench");
try (var diagnostics = new BenchmarkDiagnostics(getDiagnosticLevel())) {
// Capture initial state
diagnostics.setMonitoredDirectory(testDirectory);
diagnostics.startMonitoring("testDirectory", testDirectory);
diagnostics.startMonitoring("indexCache", Paths.get(indexCacheDir));
diagnostics.capturePrePhaseSnapshot("Build");
Map<Set<FeatureId>, ImmutableGraphIndex> indexes = new HashMap<>();

Expand Down Expand Up @@ -873,7 +875,7 @@ public static List<BenchResult> runAllAndCollectResults(
diagnostics.capturePostPhaseSnapshot("Build");
diagnostics.printDiskStatistics("Graph Index Build");
var buildSnapshot = diagnostics.getLatestSystemSnapshot();
var buildDiskSnapshot = diagnostics.getLatestDiskSnapshot();
DiskUsageMonitor.MultiDirectorySnapshot buildDiskSnapshot = diagnostics.getLatestDiskSnapshot();

try (ConfiguredSystem cs = new ConfiguredSystem(ds, index, cvArg, features)) {
int queryRuns = 2;
Expand Down Expand Up @@ -927,8 +929,8 @@ public static List<BenchResult> runAllAndCollectResults(

// Add disk metrics if available
if (buildDiskSnapshot != null) {
allMetrics.put("Disk Usage (MB)", buildDiskSnapshot.totalBytes / 1024.0 / 1024.0);
allMetrics.put("File Count", buildDiskSnapshot.fileCount);
allMetrics.put("Disk Usage (MB)", buildDiskSnapshot.getTotalBytes() / 1024.0 / 1024.0);
allMetrics.put("File Count", buildDiskSnapshot.getTotalFileCount());
}

results.add(new BenchResult(ds.getName(), params, allMetrics));
Expand Down
4 changes: 2 additions & 2 deletions jvector-examples/src/main/java/io/github/jbellis/jvector/example/benchmarks/QueryTester.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -114,11 +114,11 @@ public List<Metric> run(
if (diskSnapshot != null) {
// Number of index files created
results.add(Metric.of("search.disk.file_count", "File count", ".0f",
diskSnapshot.fileCount));
diskSnapshot.getTotalFileCount()));

// Total size of index files created
results.add(Metric.of("search.disk.total_file_size_mb", "Total file size (MB)", ".1f",
diskSnapshot.totalBytes / (1024.0 * 1024.0)));
diskSnapshot.getTotalBytes() / (1024.0 * 1024.0)));
}

} catch (IOException e) {
Expand Down
75 changes: 52 additions & 23 deletions ...n/java/io/github/jbellis/jvector/example/benchmarks/diagnostics/BenchmarkDiagnostics.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,9 +36,8 @@ public class BenchmarkDiagnostics implements AutoCloseable {
private final DiskUsageMonitor diskUsageMonitor;
private final PerformanceAnalyzer performanceAnalyzer;
private final List<SystemMonitor.SystemSnapshot> snapshots;
private final List<DiskUsageMonitor.DiskUsageSnapshot> diskSnapshots;
private final List<DiskUsageMonitor.MultiDirectorySnapshot> diskSnapshots;
private final List<PerformanceAnalyzer.TimingAnalysis> timingAnalyses;
private Path monitoredDirectory;
private boolean diskMonitorStarted = false;

public BenchmarkDiagnostics(DiagnosticLevel level) {
Expand All @@ -49,7 +48,6 @@ public BenchmarkDiagnostics(DiagnosticLevel level) {
this.snapshots = new ArrayList<>();
this.diskSnapshots = new ArrayList<>();
this.timingAnalyses = new ArrayList<>();
this.monitoredDirectory = null;
}

/**
Expand Down Expand Up @@ -79,12 +77,27 @@ public static BenchmarkDiagnostics createVerbose() {
*
* @param directory the directory to monitor
* @throws IOException if unable to start monitoring
* @deprecated Use {@link #startMonitoring(String, Path)} instead
*/
@Deprecated
public void setMonitoredDirectory(Path directory) throws IOException {
this.monitoredDirectory = directory;
if (directory != null && !diskMonitorStarted) {
diskUsageMonitor.start(directory);
startMonitoring("default", directory);
}

/**
* Starts monitoring a labeled directory for disk usage.
* This should be called before capturing any snapshots for optimal performance.
*
* @param label a label to identify this directory in reports
* @param directory the directory to monitor
* @throws IOException if unable to start monitoring
*/
public void startMonitoring(String label, Path directory) throws IOException {
if (!diskMonitorStarted) {
diskUsageMonitor.startMonitoring(label, directory);
diskMonitorStarted = true;
} else {
diskUsageMonitor.addDirectory(label, directory);
}
}

Expand All @@ -95,12 +108,12 @@ public void capturePrePhaseSnapshot(String phase) {
SystemMonitor.SystemSnapshot snapshot = systemMonitor.captureSnapshot();
snapshots.add(snapshot);

// Capture disk usage if directory is set
if (monitoredDirectory != null) {
// Capture disk usage if monitoring is started
if (diskMonitorStarted) {
try {
DiskUsageMonitor.DiskUsageSnapshot diskSnapshot = diskUsageMonitor.captureSnapshot(monitoredDirectory);
DiskUsageMonitor.MultiDirectorySnapshot diskSnapshot = diskUsageMonitor.captureSnapshot();
diskSnapshots.add(diskSnapshot);
} catch (IOException e) {
} catch (Exception e) {
if (level != DiagnosticLevel.NONE) {
System.err.printf("[%s] Failed to capture disk usage: %s%n", phase, e.getMessage());
}
Expand All @@ -127,15 +140,15 @@ public void capturePostPhaseSnapshot(String phase) {
snapshots.add(postSnapshot);

// Capture and log disk usage changes
if (monitoredDirectory != null) {
if (diskMonitorStarted) {
try {
DiskUsageMonitor.DiskUsageSnapshot postDiskSnapshot = diskUsageMonitor.captureSnapshot(monitoredDirectory);
DiskUsageMonitor.MultiDirectorySnapshot postDiskSnapshot = diskUsageMonitor.captureSnapshot();
if (!diskSnapshots.isEmpty() && level != DiagnosticLevel.NONE) {
DiskUsageMonitor.DiskUsageSnapshot preDiskSnapshot = diskSnapshots.get(diskSnapshots.size() - 1);
DiskUsageMonitor.MultiDirectorySnapshot preDiskSnapshot = diskSnapshots.get(diskSnapshots.size() - 1);
diskUsageMonitor.logDifference(phase, preDiskSnapshot, postDiskSnapshot);
}
diskSnapshots.add(postDiskSnapshot);
} catch (IOException e) {
} catch (Exception e) {
if (level != DiagnosticLevel.NONE) {
System.err.printf("[%s] Failed to capture disk usage: %s%n", phase, e.getMessage());
}
Expand Down Expand Up @@ -223,7 +236,7 @@ public SystemMonitor.SystemSnapshot getLatestSystemSnapshot() {
/**
* Gets the latest disk usage snapshot, or null if none captured
*/
public DiskUsageMonitor.DiskUsageSnapshot getLatestDiskSnapshot() {
public DiskUsageMonitor.MultiDirectorySnapshot getLatestDiskSnapshot() {
return diskSnapshots.isEmpty() ? null : diskSnapshots.get(diskSnapshots.size() - 1);
}

Expand Down Expand Up @@ -298,17 +311,33 @@ public void logSummary() {
public void printDiskStatistics(String label) {
// Disk usage summary
if (!diskSnapshots.isEmpty()) {
DiskUsageMonitor.DiskUsageSnapshot firstDisk = diskSnapshots.get(0);
DiskUsageMonitor.DiskUsageSnapshot lastDisk = diskSnapshots.get(diskSnapshots.size() - 1);
DiskUsageMonitor.DiskUsageSnapshot totalDisk = lastDisk.subtract(firstDisk);
DiskUsageMonitor.MultiDirectorySnapshot firstDisk = diskSnapshots.get(0);
DiskUsageMonitor.MultiDirectorySnapshot lastDisk = diskSnapshots.get(diskSnapshots.size() - 1);
DiskUsageMonitor.MultiDirectorySnapshot totalDisk = lastDisk.subtract(firstDisk);

System.out.printf("\nDisk Usage Summary %s:%n", label);
System.out.printf(" Total Disk Used: %s%n", DiskUsageMonitor.formatBytes(lastDisk.totalBytes));
System.out.printf(" Total Files: %d%n", lastDisk.fileCount);
System.out.printf(" Net Change: %s, %+d files%n",
DiskUsageMonitor.formatBytes(totalDisk.totalBytes), totalDisk.fileCount);

// Print statistics for each monitored directory
for (String dirLabel : lastDisk.snapshots.keySet()) {
DiskUsageMonitor.DiskUsageSnapshot lastSnap = lastDisk.get(dirLabel);
DiskUsageMonitor.DiskUsageSnapshot totalSnap = totalDisk.get(dirLabel);

System.out.printf(" [%s]:%n", dirLabel);
System.out.printf(" Total Disk Used: %s%n", DiskUsageMonitor.formatBytes(lastSnap.totalBytes));
System.out.printf(" Total Files: %d%n", lastSnap.fileCount);
if (totalSnap != null) {
System.out.printf(" Net Change: %s, %+d files%n",
DiskUsageMonitor.formatBytes(totalSnap.totalBytes), totalSnap.fileCount);
}
}

// Print overall totals
System.out.printf(" [Overall Total]:%n");
System.out.printf(" Total Disk Used: %s%n", DiskUsageMonitor.formatBytes(lastDisk.getTotalBytes()));
System.out.printf(" Total Files: %d%n", lastDisk.getTotalFileCount());
System.out.printf(" Net Change: %s, %+d files%n",
DiskUsageMonitor.formatBytes(totalDisk.getTotalBytes()), totalDisk.getTotalFileCount());
}

}

/**
Expand Down
Loading