Skip to content

Commit

Permalink
Add reindexInterval and resyncInterval to config settings (#134)
Browse files Browse the repository at this point in the history
  • Loading branch information
brendan-kellam authored Dec 17, 2024
1 parent d4e7256 commit c35f6bc
Show file tree
Hide file tree
Showing 9 changed files with 140 additions and 19 deletions.
4 changes: 4 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0

## [Unreleased]

### Added

- Added config option `settings.reindexInterval` and `settings.resyncInterval` to control how often the index should be re-indexed and re-synced. ([#134](https://github.com/sourcebot-dev/sourcebot/pull/134))

## [2.6.2] - 2024-12-13

### Added
Expand Down
4 changes: 3 additions & 1 deletion demo-site-config.json
Original file line number Diff line number Diff line change
@@ -1,7 +1,9 @@
{
"$schema": "./schemas/v2/index.json",
"settings": {
"autoDeleteStaleRepos": true
"autoDeleteStaleRepos": true,
"reindexInterval": 86400000, // 24 hours
"resyncInterval": 86400000 // 24 hours
},
"repos": [
{
Expand Down
12 changes: 2 additions & 10 deletions packages/backend/src/constants.ts
Original file line number Diff line number Diff line change
@@ -1,19 +1,11 @@
import { Settings } from "./types.js";

/**
* The interval to reindex a given repository.
*/
export const REINDEX_INTERVAL_MS = 1000 * 60 * 60;

/**
* The interval to re-sync the config.
*/
export const RESYNC_CONFIG_INTERVAL_MS = 1000 * 60 * 60 * 24;

/**
* Default settings.
*/
export const DEFAULT_SETTINGS: Settings = {
maxFileSize: 2 * 1024 * 1024, // 2MB in bytes
autoDeleteStaleRepos: true,
reindexInterval: 1000 * 60 * 60, // 1 hour in milliseconds
resyncInterval: 1000 * 60 * 60 * 24, // 1 day in milliseconds
}
66 changes: 64 additions & 2 deletions packages/backend/src/db.test.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import { expect, test } from 'vitest';
import { DEFAULT_DB_DATA, migration_addDeleteStaleRepos, migration_addMaxFileSize, migration_addSettings, Schema } from './db';
import { DEFAULT_DB_DATA, migration_addDeleteStaleRepos, migration_addMaxFileSize, migration_addReindexInterval, migration_addResyncInterval, migration_addSettings, Schema } from './db';
import { DEFAULT_SETTINGS } from './constants';
import { DeepPartial } from './types';
import { Low } from 'lowdb';
Expand Down Expand Up @@ -60,4 +60,66 @@ test('migration_addDeleteStaleRepos adds the `autoDeleteStaleRepos` field with t
autoDeleteStaleRepos: DEFAULT_SETTINGS.autoDeleteStaleRepos,
}
});
});
});

test('migration_addReindexInterval adds the `reindexInterval` field with the default value if it does not exist', () => {
const schema: DeepPartial<Schema> = {
settings: {
maxFileSize: DEFAULT_SETTINGS.maxFileSize,
autoDeleteStaleRepos: DEFAULT_SETTINGS.autoDeleteStaleRepos,
},
}

const migratedSchema = migration_addReindexInterval(schema as Schema);
expect(migratedSchema).toStrictEqual({
settings: {
maxFileSize: DEFAULT_SETTINGS.maxFileSize,
autoDeleteStaleRepos: DEFAULT_SETTINGS.autoDeleteStaleRepos,
reindexInterval: DEFAULT_SETTINGS.reindexInterval,
}
});
});

test('migration_addReindexInterval preserves existing reindexInterval value if already set', () => {
const customInterval = 60;
const schema: DeepPartial<Schema> = {
settings: {
maxFileSize: DEFAULT_SETTINGS.maxFileSize,
reindexInterval: customInterval,
},
}

const migratedSchema = migration_addReindexInterval(schema as Schema);
expect(migratedSchema.settings.reindexInterval).toBe(customInterval);
});

test('migration_addResyncInterval adds the `resyncInterval` field with the default value if it does not exist', () => {
const schema: DeepPartial<Schema> = {
settings: {
maxFileSize: DEFAULT_SETTINGS.maxFileSize,
autoDeleteStaleRepos: DEFAULT_SETTINGS.autoDeleteStaleRepos,
},
}

const migratedSchema = migration_addResyncInterval(schema as Schema);
expect(migratedSchema).toStrictEqual({
settings: {
maxFileSize: DEFAULT_SETTINGS.maxFileSize,
autoDeleteStaleRepos: DEFAULT_SETTINGS.autoDeleteStaleRepos,
resyncInterval: DEFAULT_SETTINGS.resyncInterval,
}
});
});

test('migration_addResyncInterval preserves existing resyncInterval value if already set', () => {
const customInterval = 120;
const schema: DeepPartial<Schema> = {
settings: {
maxFileSize: DEFAULT_SETTINGS.maxFileSize,
resyncInterval: customInterval,
},
}

const migratedSchema = migration_addResyncInterval(schema as Schema);
expect(migratedSchema.settings.resyncInterval).toBe(customInterval);
});
28 changes: 27 additions & 1 deletion packages/backend/src/db.ts
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,8 @@ export const applyMigrations = async (db: Database) => {
schema = migration_addSettings(schema, log);
schema = migration_addMaxFileSize(schema, log);
schema = migration_addDeleteStaleRepos(schema, log);
schema = migration_addReindexInterval(schema, log);
schema = migration_addResyncInterval(schema, log);
return schema;
});
}
Expand Down Expand Up @@ -89,9 +91,33 @@ export const migration_addMaxFileSize = (schema: Schema, log?: (name: string) =>
*/
export const migration_addDeleteStaleRepos = (schema: Schema, log?: (name: string) => void) => {
if (schema.settings.autoDeleteStaleRepos === undefined) {
log?.("deleteStaleRepos");
log?.("addDeleteStaleRepos");
schema.settings.autoDeleteStaleRepos = DEFAULT_SETTINGS.autoDeleteStaleRepos;
}

return schema;
}

/**
* @see: https://github.com/sourcebot-dev/sourcebot/pull/134
*/
export const migration_addReindexInterval = (schema: Schema, log?: (name: string) => void) => {
if (schema.settings.reindexInterval === undefined) {
log?.("addReindexInterval");
schema.settings.reindexInterval = DEFAULT_SETTINGS.reindexInterval;
}

return schema;
}

/**
* @see: https://github.com/sourcebot-dev/sourcebot/pull/134
*/
export const migration_addResyncInterval = (schema: Schema, log?: (name: string) => void) => {
if (schema.settings.resyncInterval === undefined) {
log?.("addResyncInterval");
schema.settings.resyncInterval = DEFAULT_SETTINGS.resyncInterval;
}

return schema;
}
11 changes: 6 additions & 5 deletions packages/backend/src/main.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ import { cloneRepository, fetchRepository } from "./git.js";
import { createLogger } from "./logger.js";
import { createRepository, Database, loadDB, updateRepository, updateSettings } from './db.js';
import { arraysEqualShallow, isRemotePath, measure } from "./utils.js";
import { DEFAULT_SETTINGS, REINDEX_INTERVAL_MS, RESYNC_CONFIG_INTERVAL_MS } from "./constants.js";
import { DEFAULT_SETTINGS } from "./constants.js";
import stripJsonComments from 'strip-json-comments';
import { indexGitRepository, indexLocalRepository } from "./zoekt.js";
import { getLocalRepoFromConfig, initLocalRepoFileWatchers } from "./local.js";
Expand Down Expand Up @@ -205,6 +205,8 @@ const syncConfig = async (configPath: string, db: Database, signal: AbortSignal,
const updatedSettings: Settings = {
maxFileSize: config.settings?.maxFileSize ?? DEFAULT_SETTINGS.maxFileSize,
autoDeleteStaleRepos: config.settings?.autoDeleteStaleRepos ?? DEFAULT_SETTINGS.autoDeleteStaleRepos,
reindexInterval: config.settings?.reindexInterval ?? DEFAULT_SETTINGS.reindexInterval,
resyncInterval: config.settings?.resyncInterval ?? DEFAULT_SETTINGS.resyncInterval,
}
const _isAllRepoReindexingRequired = isAllRepoReindexingRequired(db.data.settings, updatedSettings);
await updateSettings(updatedSettings, db);
Expand Down Expand Up @@ -345,11 +347,10 @@ export const main = async (context: AppContext) => {
});
}

// Re-sync every 24 hours
// Re-sync at a fixed interval
setInterval(() => {
logger.info(`Re-syncing configuration file ${context.configPath}`);
_syncConfig();
}, RESYNC_CONFIG_INTERVAL_MS);
}, db.data.settings.resyncInterval);

// Sync immediately on startup
await _syncConfig();
Expand All @@ -369,7 +370,7 @@ export const main = async (context: AppContext) => {
continue;
}

if (lastIndexed.getTime() > Date.now() - REINDEX_INTERVAL_MS) {
if (lastIndexed.getTime() > (Date.now() - db.data.settings.reindexInterval)) {
continue;
}

Expand Down
8 changes: 8 additions & 0 deletions packages/backend/src/schemas/v2.ts
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,14 @@ export interface Settings {
* Automatically delete stale repositories from the index. Defaults to true.
*/
autoDeleteStaleRepos?: boolean;
/**
* The interval (in milliseconds) at which the indexer should re-index all repositories. Repositories are always indexed when first added. Defaults to 1 hour (3600000 milliseconds).
*/
reindexInterval?: number;
/**
* The interval (in milliseconds) at which the configuration file should be re-synced. The configuration file is always synced on startup. Defaults to 24 hours (86400000 milliseconds).
*/
resyncInterval?: number;
}
export interface GitHubConfig {
/**
Expand Down
14 changes: 14 additions & 0 deletions packages/backend/src/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -44,8 +44,22 @@ export type AppContext = {
}

export type Settings = {
/**
* The maximum size of a file (in bytes) to be indexed. Files that exceed this maximum will not be inexed.
*/
maxFileSize: number;
/**
* Automatically delete stale repositories from the index. Defaults to true.
*/
autoDeleteStaleRepos: boolean;
/**
* The interval (in milliseconds) at which the indexer should re-index all repositories.
*/
reindexInterval: number;
/**
* The interval (in milliseconds) at which the configuration file should be re-synced.
*/
resyncInterval: number;
}

// @see : https://stackoverflow.com/a/61132308
Expand Down
12 changes: 12 additions & 0 deletions schemas/v2/index.json
Original file line number Diff line number Diff line change
Expand Up @@ -534,6 +534,18 @@
"type": "boolean",
"description": "Automatically delete stale repositories from the index. Defaults to true.",
"default": true
},
"reindexInterval": {
"type": "integer",
"description": "The interval (in milliseconds) at which the indexer should re-index all repositories. Repositories are always indexed when first added. Defaults to 1 hour (3600000 milliseconds).",
"default": 3600000,
"minimum": 1
},
"resyncInterval": {
"type": "integer",
"description": "The interval (in milliseconds) at which the configuration file should be re-synced. The configuration file is always synced on startup. Defaults to 24 hours (86400000 milliseconds).",
"default": 86400000,
"minimum": 1
}
},
"additionalProperties": false
Expand Down

0 comments on commit c35f6bc

Please sign in to comment.