Skip to content

Commit c35f6bc

Browse files
brendan-kellambrendan-kellam
authored
Add reindexInterval and resyncInterval to config settings (#134)
1 parent d4e7256 commit c35f6bc

File tree

9 files changed

+140
-19
lines changed

9 files changed

+140
-19
lines changed

‎CHANGELOG.md‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
77

88
## [Unreleased]
99

10+
### Added
11+
12+
- Added config option `settings.reindexInterval` and `settings.resyncInterval` to control how often the index should be re-indexed and re-synced. ([#134](https://github.com/sourcebot-dev/sourcebot/pull/134))
13+
1014
## [2.6.2] - 2024-12-13
1115

1216
### Added

‎demo-site-config.json‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,9 @@
11
{
22
"$schema": "./schemas/v2/index.json",
33
"settings": {
4-
"autoDeleteStaleRepos": true
4+
"autoDeleteStaleRepos": true,
5+
"reindexInterval": 86400000, // 24 hours
6+
"resyncInterval": 86400000 // 24 hours
57
},
68
"repos": [
79
{

‎packages/backend/src/constants.ts‎

Lines changed: 2 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,19 +1,11 @@
11
import { Settings } from "./types.js";
22

3-
/**
4-
* The interval to reindex a given repository.
5-
*/
6-
export const REINDEX_INTERVAL_MS = 1000 * 60 * 60;
7-
8-
/**
9-
* The interval to re-sync the config.
10-
*/
11-
export const RESYNC_CONFIG_INTERVAL_MS = 1000 * 60 * 60 * 24;
12-
133
/**
144
* Default settings.
155
*/
166
export const DEFAULT_SETTINGS: Settings = {
177
maxFileSize: 2 * 1024 * 1024, // 2MB in bytes
188
autoDeleteStaleRepos: true,
9+
reindexInterval: 1000 * 60 * 60, // 1 hour in milliseconds
10+
resyncInterval: 1000 * 60 * 60 * 24, // 1 day in milliseconds
1911
}

‎packages/backend/src/db.test.ts‎

Lines changed: 64 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
import { expect, test } from 'vitest';
2-
import { DEFAULT_DB_DATA, migration_addDeleteStaleRepos, migration_addMaxFileSize, migration_addSettings, Schema } from './db';
2+
import { DEFAULT_DB_DATA, migration_addDeleteStaleRepos, migration_addMaxFileSize, migration_addReindexInterval, migration_addResyncInterval, migration_addSettings, Schema } from './db';
33
import { DEFAULT_SETTINGS } from './constants';
44
import { DeepPartial } from './types';
55
import { Low } from 'lowdb';
@@ -60,4 +60,66 @@ test('migration_addDeleteStaleRepos adds the `autoDeleteStaleRepos` field with t
6060
autoDeleteStaleRepos: DEFAULT_SETTINGS.autoDeleteStaleRepos,
6161
}
6262
});
63-
});
63+
});
64+
65+
test('migration_addReindexInterval adds the `reindexInterval` field with the default value if it does not exist', () => {
66+
const schema: DeepPartial<Schema> = {
67+
settings: {
68+
maxFileSize: DEFAULT_SETTINGS.maxFileSize,
69+
autoDeleteStaleRepos: DEFAULT_SETTINGS.autoDeleteStaleRepos,
70+
},
71+
}
72+
73+
const migratedSchema = migration_addReindexInterval(schema as Schema);
74+
expect(migratedSchema).toStrictEqual({
75+
settings: {
76+
maxFileSize: DEFAULT_SETTINGS.maxFileSize,
77+
autoDeleteStaleRepos: DEFAULT_SETTINGS.autoDeleteStaleRepos,
78+
reindexInterval: DEFAULT_SETTINGS.reindexInterval,
79+
}
80+
});
81+
});
82+
83+
test('migration_addReindexInterval preserves existing reindexInterval value if already set', () => {
84+
const customInterval = 60;
85+
const schema: DeepPartial<Schema> = {
86+
settings: {
87+
maxFileSize: DEFAULT_SETTINGS.maxFileSize,
88+
reindexInterval: customInterval,
89+
},
90+
}
91+
92+
const migratedSchema = migration_addReindexInterval(schema as Schema);
93+
expect(migratedSchema.settings.reindexInterval).toBe(customInterval);
94+
});
95+
96+
test('migration_addResyncInterval adds the `resyncInterval` field with the default value if it does not exist', () => {
97+
const schema: DeepPartial<Schema> = {
98+
settings: {
99+
maxFileSize: DEFAULT_SETTINGS.maxFileSize,
100+
autoDeleteStaleRepos: DEFAULT_SETTINGS.autoDeleteStaleRepos,
101+
},
102+
}
103+
104+
const migratedSchema = migration_addResyncInterval(schema as Schema);
105+
expect(migratedSchema).toStrictEqual({
106+
settings: {
107+
maxFileSize: DEFAULT_SETTINGS.maxFileSize,
108+
autoDeleteStaleRepos: DEFAULT_SETTINGS.autoDeleteStaleRepos,
109+
resyncInterval: DEFAULT_SETTINGS.resyncInterval,
110+
}
111+
});
112+
});
113+
114+
test('migration_addResyncInterval preserves existing resyncInterval value if already set', () => {
115+
const customInterval = 120;
116+
const schema: DeepPartial<Schema> = {
117+
settings: {
118+
maxFileSize: DEFAULT_SETTINGS.maxFileSize,
119+
resyncInterval: customInterval,
120+
},
121+
}
122+
123+
const migratedSchema = migration_addResyncInterval(schema as Schema);
124+
expect(migratedSchema.settings.resyncInterval).toBe(customInterval);
125+
});

‎packages/backend/src/db.ts‎

Lines changed: 27 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,8 @@ export const applyMigrations = async (db: Database) => {
5656
schema = migration_addSettings(schema, log);
5757
schema = migration_addMaxFileSize(schema, log);
5858
schema = migration_addDeleteStaleRepos(schema, log);
59+
schema = migration_addReindexInterval(schema, log);
60+
schema = migration_addResyncInterval(schema, log);
5961
return schema;
6062
});
6163
}
@@ -89,9 +91,33 @@ export const migration_addMaxFileSize = (schema: Schema, log?: (name: string) =>
8991
*/
9092
export const migration_addDeleteStaleRepos = (schema: Schema, log?: (name: string) => void) => {
9193
if (schema.settings.autoDeleteStaleRepos === undefined) {
92-
log?.("deleteStaleRepos");
94+
log?.("addDeleteStaleRepos");
9395
schema.settings.autoDeleteStaleRepos = DEFAULT_SETTINGS.autoDeleteStaleRepos;
9496
}
9597

98+
return schema;
99+
}
100+
101+
/**
102+
* @see: https://github.com/sourcebot-dev/sourcebot/pull/134
103+
*/
104+
export const migration_addReindexInterval = (schema: Schema, log?: (name: string) => void) => {
105+
if (schema.settings.reindexInterval === undefined) {
106+
log?.("addReindexInterval");
107+
schema.settings.reindexInterval = DEFAULT_SETTINGS.reindexInterval;
108+
}
109+
110+
return schema;
111+
}
112+
113+
/**
114+
* @see: https://github.com/sourcebot-dev/sourcebot/pull/134
115+
*/
116+
export const migration_addResyncInterval = (schema: Schema, log?: (name: string) => void) => {
117+
if (schema.settings.resyncInterval === undefined) {
118+
log?.("addResyncInterval");
119+
schema.settings.resyncInterval = DEFAULT_SETTINGS.resyncInterval;
120+
}
121+
96122
return schema;
97123
}

‎packages/backend/src/main.ts‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ import { cloneRepository, fetchRepository } from "./git.js";
1010
import { createLogger } from "./logger.js";
1111
import { createRepository, Database, loadDB, updateRepository, updateSettings } from './db.js';
1212
import { arraysEqualShallow, isRemotePath, measure } from "./utils.js";
13-
import { DEFAULT_SETTINGS, REINDEX_INTERVAL_MS, RESYNC_CONFIG_INTERVAL_MS } from "./constants.js";
13+
import { DEFAULT_SETTINGS } from "./constants.js";
1414
import stripJsonComments from 'strip-json-comments';
1515
import { indexGitRepository, indexLocalRepository } from "./zoekt.js";
1616
import { getLocalRepoFromConfig, initLocalRepoFileWatchers } from "./local.js";
@@ -205,6 +205,8 @@ const syncConfig = async (configPath: string, db: Database, signal: AbortSignal,
205205
const updatedSettings: Settings = {
206206
maxFileSize: config.settings?.maxFileSize ?? DEFAULT_SETTINGS.maxFileSize,
207207
autoDeleteStaleRepos: config.settings?.autoDeleteStaleRepos ?? DEFAULT_SETTINGS.autoDeleteStaleRepos,
208+
reindexInterval: config.settings?.reindexInterval ?? DEFAULT_SETTINGS.reindexInterval,
209+
resyncInterval: config.settings?.resyncInterval ?? DEFAULT_SETTINGS.resyncInterval,
208210
}
209211
const _isAllRepoReindexingRequired = isAllRepoReindexingRequired(db.data.settings, updatedSettings);
210212
await updateSettings(updatedSettings, db);
@@ -345,11 +347,10 @@ export const main = async (context: AppContext) => {
345347
});
346348
}
347349

348-
// Re-sync every 24 hours
350+
// Re-sync at a fixed interval
349351
setInterval(() => {
350-
logger.info(`Re-syncing configuration file ${context.configPath}`);
351352
_syncConfig();
352-
}, RESYNC_CONFIG_INTERVAL_MS);
353+
}, db.data.settings.resyncInterval);
353354

354355
// Sync immediately on startup
355356
await _syncConfig();
@@ -369,7 +370,7 @@ export const main = async (context: AppContext) => {
369370
continue;
370371
}
371372

372-
if (lastIndexed.getTime() > Date.now() - REINDEX_INTERVAL_MS) {
373+
if (lastIndexed.getTime() > (Date.now() - db.data.settings.reindexInterval)) {
373374
continue;
374375
}
375376

‎packages/backend/src/schemas/v2.ts‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,14 @@ export interface Settings {
2525
* Automatically delete stale repositories from the index. Defaults to true.
2626
*/
2727
autoDeleteStaleRepos?: boolean;
28+
/**
29+
* The interval (in milliseconds) at which the indexer should re-index all repositories. Repositories are always indexed when first added. Defaults to 1 hour (3600000 milliseconds).
30+
*/
31+
reindexInterval?: number;
32+
/**
33+
* The interval (in milliseconds) at which the configuration file should be re-synced. The configuration file is always synced on startup. Defaults to 24 hours (86400000 milliseconds).
34+
*/
35+
resyncInterval?: number;
2836
}
2937
export interface GitHubConfig {
3038
/**

‎packages/backend/src/types.ts‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,8 +44,22 @@ export type AppContext = {
4444
}
4545

4646
export type Settings = {
47+
/**
48+
* The maximum size of a file (in bytes) to be indexed. Files that exceed this maximum will not be inexed.
49+
*/
4750
maxFileSize: number;
51+
/**
52+
* Automatically delete stale repositories from the index. Defaults to true.
53+
*/
4854
autoDeleteStaleRepos: boolean;
55+
/**
56+
* The interval (in milliseconds) at which the indexer should re-index all repositories.
57+
*/
58+
reindexInterval: number;
59+
/**
60+
* The interval (in milliseconds) at which the configuration file should be re-synced.
61+
*/
62+
resyncInterval: number;
4963
}
5064

5165
// @see : https://stackoverflow.com/a/61132308

‎schemas/v2/index.json‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -534,6 +534,18 @@
534534
"type": "boolean",
535535
"description": "Automatically delete stale repositories from the index. Defaults to true.",
536536
"default": true
537+
},
538+
"reindexInterval": {
539+
"type": "integer",
540+
"description": "The interval (in milliseconds) at which the indexer should re-index all repositories. Repositories are always indexed when first added. Defaults to 1 hour (3600000 milliseconds).",
541+
"default": 3600000,
542+
"minimum": 1
543+
},
544+
"resyncInterval": {
545+
"type": "integer",
546+
"description": "The interval (in milliseconds) at which the configuration file should be re-synced. The configuration file is always synced on startup. Defaults to 24 hours (86400000 milliseconds).",
547+
"default": 86400000,
548+
"minimum": 1
537549
}
538550
},
539551
"additionalProperties": false

0 commit comments

Comments
 (0)