Skip to content

Add buffer-native serialization hooks and mmap-backed partition path; benchmark MessagePack + mmap #314

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
Copilot wants to merge 7 commits into from
Closed

Add buffer-native serialization hooks and mmap-backed partition path; benchmark MessagePack + mmap #314

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
b1a5375
feat: add buffer serializer hooks and mmap-backed write buffer path
Copilot May 14, 2026
597e9f4
fix: use compatible mmap-backed temp file buffer and benchmark msgpac…
Copilot May 14, 2026
be49e41
test: expand serializer hook coverage and harden mmap temp naming
Copilot May 14, 2026
b5ad377
fix: map actual partition file for mmap writes and skip serializedSiz…
Copilot May 14, 2026
ff12b45
refactor: simplify mmap writes and remove pending mmap tests
Copilot May 14, 2026
8596930
chore: improve mmap page-size assertion message
Copilot May 14, 2026
be7db61
refactor: streamline mmap remap growth and close flushing
Copilot May 14, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 9 additions & 1 deletion bench/bench-eventstore.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ import benchmarks from 'beautify-benchmark';
import fs from 'fs-extra';
import Stable from 'event-storage';
import { EventStore as Latest } from '../index.js';
import { createMsgpackSerializer } from './msgpackSerializer.js';

const Suite = new Benchmark.Suite('eventstore');
Suite.on('start', () => fs.emptyDirSync('data'));
Expand Down Expand Up @@ -37,4 +38,11 @@ Suite.add('eventstore [latest]', function() {
bench(new Latest('eventstore', { storageDirectory: 'data/latest' }), this.cycles);
});

Suite.run();
Suite.add('eventstore [latest+msgpack+mmap]', function() {
bench(new Latest('eventstore', {
storageDirectory: 'data/latest-msgpack-mmap',
storageConfig: { mmapWriteBuffer: true, serializer: createMsgpackSerializer() }
}), this.cycles);
});

Suite.run();
21 changes: 16 additions & 5 deletions bench/bench-read-scenarios.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ import benchmarks from 'beautify-benchmark';
import fs from 'fs-extra';
import Stable from 'event-storage';
import { EventStore as Latest } from '../index.js';
import { createMsgpackSerializer } from './msgpackSerializer.js';

const Suite = new Benchmark.Suite('read-scenarios');
Suite.on('cycle', (event) => benchmarks.add(event.target));
Expand All @@ -21,10 +22,10 @@ function countAll(iter) {
return n;
}

function populateStore(EventStore, directory) {
function populateStore(EventStore, directory, config = {}) {
return new Promise((resolve, reject) => {
fs.emptyDirSync(directory);
const store = new EventStore('bench', { storageDirectory: directory });
const store = new EventStore('bench', Object.assign({ storageDirectory: directory }, config));
store.once('ready', () => {
for (let i = 0; i < EVENTS; i++) {
store.commit(i % 2 === 0 ? STREAM_1 : STREAM_2, Object.assign({ seq: i }, EVENT_DOC));
Expand All @@ -36,33 +37,43 @@ function populateStore(EventStore, directory) {
});
}

function openReadOnly(EventStore, directory) {
function openReadOnly(EventStore, directory, config = {}) {
return new Promise((resolve, reject) => {
const store = new EventStore('bench', { storageDirectory: directory, readOnly: true });
const store = new EventStore('bench', Object.assign({ storageDirectory: directory, readOnly: true }, config));
store.once('ready', () => resolve(store));
store.once('error', reject);
});
}

populateStore(Stable, 'data/stable')
.then(() => populateStore(Latest, 'data/latest'))
.then(() => populateStore(Latest, 'data/latest-msgpack-mmap', {
storageConfig: { mmapWriteBuffer: true, serializer: createMsgpackSerializer() }
}))
.then(() => Promise.all([
openReadOnly(Stable, 'data/stable'),
openReadOnly(Latest, 'data/latest'),
openReadOnly(Latest, 'data/latest-msgpack-mmap', {
storageConfig: { mmapWriteBuffer: true, serializer: createMsgpackSerializer() }
}),
]))
.then(([stableStore, latestStore]) => {
.then(([stableStore, latestStore, latestMsgpackMmap]) => {
const third = Math.ceil(EVENTS / 3);
const twoThirds = Math.floor(2 * EVENTS / 3);

Suite
.add('1 - forward full scan [stable]', () => countAll(stableStore.getAllEvents()))
.add('1 - forward full scan [latest]', () => countAll(latestStore.getAllEvents()))
.add('1 - forward full scan [latest+msgpack+mmap]', () => countAll(latestMsgpackMmap.getAllEvents()))
.add('2 - backwards full scan [stable]', () => countAll(stableStore.getAllEvents(-1, 1)))
.add('2 - backwards full scan [latest]', () => countAll(latestStore.getAllEvents(-1, 1)))
.add('2 - backwards full scan [latest+msgpack+mmap]', () => countAll(latestMsgpackMmap.getAllEvents(-1, 1)))
.add('3 - join stream [stable]', () => countAll(stableStore.fromStreams('join', [STREAM_1, STREAM_2])))
.add('3 - join stream [latest]', () => countAll(latestStore.fromStreams('join', [STREAM_1, STREAM_2])))
.add('3 - join stream [latest+msgpack+mmap]', () => countAll(latestMsgpackMmap.fromStreams('join', [STREAM_1, STREAM_2])))
.add('4 - range scan [stable]', () => countAll(stableStore.getAllEvents(third, twoThirds)))
.add('4 - range scan [latest]', () => countAll(latestStore.getAllEvents(third, twoThirds)))
.add('4 - range scan [latest+msgpack+mmap]', () => countAll(latestMsgpackMmap.getAllEvents(third, twoThirds)))
.run();
})
.catch((e) => { console.error(e); process.exit(1); });
11 changes: 10 additions & 1 deletion bench/bench-storage.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ import benchmarks from 'beautify-benchmark';
import fs from 'fs-extra';
import Stable from 'event-storage';
import { Storage as LatestStorage } from '../index.js';
import { createMsgpackSerializer } from './msgpackSerializer.js';

const Suite = new Benchmark.Suite('storage');
Suite.on('start', () => fs.emptyDirSync('data'));
Expand Down Expand Up @@ -38,4 +39,12 @@ Suite.add('storage [latest]', function() {
bench(new LatestStorage('storage-' + this.cycles, { dataDirectory: 'data/latest' }));
});

Suite.run();
Suite.add('storage [latest+msgpack+mmap]', function() {
bench(new LatestStorage('storage-' + this.cycles, {
dataDirectory: 'data/latest-msgpack-mmap',
mmapWriteBuffer: true,
serializer: createMsgpackSerializer()
}));
});

Suite.run();
211 changes: 210 additions & 1 deletion bench/package-lock.json
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

3 changes: 2 additions & 1 deletion bench/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
"beautify-benchmark": "^0.2.4",
"benchmark": "^2.1.4",
"event-storage": "^0.7.2",
"fs-extra": "^11.1.1"
"fs-extra": "^11.1.1",
"msgpackr": "^2.0.1"
}
}
Loading