[wip] v3 effort

.gitmodules

@@ -1,3 +1,5 @@
 [submodule "vendor/zoekt"]
 	path = vendor/zoekt
 	url = https://github.com/sourcebot-dev/zoekt
+	# @nocheckin : update this to main when we have a release
+	branch=v3

Dockerfile

@@ -31,13 +31,24 @@ ENV NEXT_PUBLIC_POSTHOG_PAPIK=BAKED_NEXT_PUBLIC_POSTHOG_PAPIK
 ARG NEXT_PUBLIC_DOMAIN_SUB_PATH=/BAKED_NEXT_PUBLIC_DOMAIN_SUB_PATH
 RUN yarn workspace @sourcebot/web build
 
+# ------ Build Database ------
+FROM node-alpine AS database-builder
+WORKDIR /app
+
+COPY package.json yarn.lock* ./
+COPY ./packages/db ./packages/db
+RUN yarn workspace @sourcebot/db install --frozen-lockfile
+
+
 # ------ Build Backend ------
 FROM node-alpine AS backend-builder
 WORKDIR /app
 
 COPY package.json yarn.lock* ./
 COPY ./schemas ./schemas
 COPY ./packages/backend ./packages/backend
+COPY --from=database-builder /app/node_modules ./node_modules
+COPY --from=database-builder /app/packages/db ./packages/db
 RUN yarn workspace @sourcebot/backend install --frozen-lockfile
 RUN yarn workspace @sourcebot/backend build
 
@@ -100,6 +111,10 @@ COPY --from=web-builder /app/packages/web/.next/static ./packages/web/.next/stat
 COPY --from=backend-builder /app/node_modules ./node_modules
 COPY --from=backend-builder /app/packages/backend ./packages/backend
 
+# Configure the database
+COPY --from=database-builder /app/node_modules ./node_modules
+COPY --from=database-builder /app/packages/db ./packages/db
+
 COPY supervisord.conf /etc/supervisor/conf.d/supervisord.conf
 COPY prefix-output.sh ./prefix-output.sh
 RUN chmod +x ./prefix-output.sh

Makefile

@@ -1,10 +1,11 @@
 
-CMDS := zoekt ui
+CMDS := zoekt yarn
 
 ALL: $(CMDS)
 
-ui:
+yarn:
 	yarn install
+	yarn workspace @sourcebot/db prisma:migrate:dev
 
 zoekt:
 	mkdir -p bin
@@ -20,6 +21,8 @@ clean:
 		packages/web/.next \
 		packages/backend/dist \
 		packages/backend/node_modules \
+		packages/db/node_modules \
+		packages/db/dist \
 		.sourcebot
 
 .PHONY: bin

entrypoint.sh

@@ -19,6 +19,11 @@ if [ ! -d "$DATA_CACHE_DIR" ]; then
     mkdir -p "$DATA_CACHE_DIR"
 fi
 
+# Run a Database migration
+echo -e "\e[34m[Info] Running database migration...\e[0m"
+export DATABASE_URL="file:$DATA_CACHE_DIR/db.sqlite"
+yarn workspace @sourcebot/db prisma:migrate:prod
+
 # In order to detect if this is the first run, we create a `.installed` file in
 # the cache directory.
 FIRST_RUN_FILE="$DATA_CACHE_DIR/.installedv2"

packages/backend/src/config.ts

@@ -0,0 +1,134 @@
+import { PrismaClient } from '@sourcebot/db';
+import { readFile } from 'fs/promises';
+import stripJsonComments from 'strip-json-comments';
+import { getGitHubReposFromConfig } from "./github.js";
+import { getGitLabReposFromConfig, GITLAB_CLOUD_HOSTNAME } from "./gitlab.js";
+import { SourcebotConfigurationSchema } from "./schemas/v2.js";
+import { AppContext } from "./types.js";
+import { getTokenFromConfig, isRemotePath, marshalBool } from "./utils.js";
+
+export const syncConfig = async (configPath: string, db: PrismaClient, signal: AbortSignal, ctx: AppContext) => {
+    const configContent = await (async () => {
+        if (isRemotePath(configPath)) {
+            const response = await fetch(configPath, {
+                signal,
+            });
+            if (!response.ok) {
+                throw new Error(`Failed to fetch config file ${configPath}: ${response.statusText}`);
+            }
+            return response.text();
+        } else {
+            return readFile(configPath, {
+                encoding: 'utf-8',
+                signal,
+            });
+        }
+    })();
+
+    // @todo: we should validate the configuration file's structure here.
+    const config = JSON.parse(stripJsonComments(configContent)) as SourcebotConfigurationSchema;
+
+    for (const repoConfig of config.repos ?? []) {
+        switch (repoConfig.type) {
+            case 'github': {
+                const token = repoConfig.token ? getTokenFromConfig(repoConfig.token, ctx) : undefined;
+                const gitHubRepos = await getGitHubReposFromConfig(repoConfig, signal, ctx);
+                const hostUrl = repoConfig.url ?? 'https://github.com';
+                const hostname = repoConfig.url ? new URL(repoConfig.url).hostname : 'github.com';
+
+                await Promise.all(gitHubRepos.map((repo) => {
+                    const repoName = `${hostname}/${repo.full_name}`;
+                    const cloneUrl = new URL(repo.clone_url!);
+                    if (token) {
+                        cloneUrl.username = token;
+                    }
+
+                    const data = {
+                        external_id: repo.id.toString(),
+                        external_codeHostType: 'github',
+                        external_codeHostUrl: hostUrl,
+                        cloneUrl: cloneUrl.toString(),
+                        name: repoName,
+                        isFork: repo.fork,
+                        isArchived: !!repo.archived,
+                        metadata: {
+                            'zoekt.web-url-type': 'github',
+                            'zoekt.web-url': repo.html_url,
+                            'zoekt.name': repoName,
+                            'zoekt.github-stars': (repo.stargazers_count ?? 0).toString(),
+                            'zoekt.github-watchers': (repo.watchers_count ?? 0).toString(),
+                            'zoekt.github-subscribers': (repo.subscribers_count ?? 0).toString(),
+                            'zoekt.github-forks': (repo.forks_count ?? 0).toString(),
+                            'zoekt.archived': marshalBool(repo.archived),
+                            'zoekt.fork': marshalBool(repo.fork),
+                            'zoekt.public': marshalBool(repo.private === false) 
+                        },
+                    };
+
+                    return db.repo.upsert({
+                        where: {
+                            external_id_external_codeHostUrl: {
+                                external_id: repo.id.toString(),
+                                external_codeHostUrl: hostUrl,
+                            },
+                        },
+                        create: data,
+                        update: data,
+                    })
+                }));
+
+                break;
+            }
+            case 'gitlab': {
+                const hostUrl = repoConfig.url ?? 'https://gitlab.com';
+                const hostname = repoConfig.url ? new URL(repoConfig.url).hostname : GITLAB_CLOUD_HOSTNAME;
+                const token = repoConfig.token ? getTokenFromConfig(repoConfig.token, ctx) : undefined;
+                const gitLabRepos = await getGitLabReposFromConfig(repoConfig, ctx);
+
+                await Promise.all(gitLabRepos.map((project) => {
+                        const repoName = `${hostname}/${project.path_with_namespace}`;
+                        const isFork = project.forked_from_project !== undefined;
+
+                        const cloneUrl = new URL(project.http_url_to_repo);
+                        if (token) {
+                            cloneUrl.username = 'oauth2';
+                            cloneUrl.password = token;
+                        }
+
+                        const data = {
+                            external_id: project.id.toString(),
+                            external_codeHostType: 'gitlab',
+                            external_codeHostUrl: hostUrl,
+                            cloneUrl: cloneUrl.toString(),
+                            name: repoName,
+                            isFork,
+                            isArchived: project.archived,
+                            metadata: {
+                                'zoekt.web-url-type': 'gitlab',
+                                'zoekt.web-url': project.web_url,
+                                'zoekt.name': repoName,
+                                'zoekt.gitlab-stars': project.star_count?.toString() ?? '0',
+                                'zoekt.gitlab-forks': project.forks_count?.toString() ?? '0',
+                                'zoekt.archived': marshalBool(project.archived),
+                                'zoekt.fork': marshalBool(isFork),
+                                'zoekt.public': marshalBool(project.visibility === 'public'),
+                            }
+                        }
+
+                        return db.repo.upsert({
+                            where: {
+                                external_id_external_codeHostUrl: {
+                                    external_id: project.id.toString(),
+                                    external_codeHostUrl: hostUrl,
+                                },
+                            },
+                            create: data,
+                            update: data,
+                        })
+                }));
+
+                break;
+            }
+        }
+    }
+}

packages/backend/src/constants.ts

@@ -6,6 +6,6 @@ import { Settings } from "./types.js";
 export const DEFAULT_SETTINGS: Settings = {
     maxFileSize: 2 * 1024 * 1024, // 2MB in bytes
     autoDeleteStaleRepos: true,
-    reindexInterval: 1000 * 60 * 60, // 1 hour in milliseconds
-    resyncInterval: 1000 * 60 * 60 * 24, // 1 day in milliseconds
+    reindexIntervalMs: 1000 * 60 * 60, // 1 hour in milliseconds
+    resyncIntervalMs: 1000 * 60 * 60 * 24, // 1 day in milliseconds
 }