Skip to content

Commit 15b9e77

Browse files
brendan-kellambrendan-kellam
authored
SQL Database (#157)
1 parent 7599980 commit 15b9e77

30 files changed

Lines changed: 831 additions & 1066 deletions

‎Dockerfile‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,13 +31,24 @@ ENV NEXT_PUBLIC_POSTHOG_PAPIK=BAKED_NEXT_PUBLIC_POSTHOG_PAPIK
3131
ARG NEXT_PUBLIC_DOMAIN_SUB_PATH=/BAKED_NEXT_PUBLIC_DOMAIN_SUB_PATH
3232
RUN yarn workspace @sourcebot/web build
3333

34+
# ------ Build Database ------
35+
FROM node-alpine AS database-builder
36+
WORKDIR /app
37+
38+
COPY package.json yarn.lock* ./
39+
COPY ./packages/db ./packages/db
40+
RUN yarn workspace @sourcebot/db install --frozen-lockfile
41+
42+
3443
# ------ Build Backend ------
3544
FROM node-alpine AS backend-builder
3645
WORKDIR /app
3746

3847
COPY package.json yarn.lock* ./
3948
COPY ./schemas ./schemas
4049
COPY ./packages/backend ./packages/backend
50+
COPY --from=database-builder /app/node_modules ./node_modules
51+
COPY --from=database-builder /app/packages/db ./packages/db
4152
RUN yarn workspace @sourcebot/backend install --frozen-lockfile
4253
RUN yarn workspace @sourcebot/backend build
4354

@@ -100,6 +111,10 @@ COPY --from=web-builder /app/packages/web/.next/static ./packages/web/.next/stat
100111
COPY --from=backend-builder /app/node_modules ./node_modules
101112
COPY --from=backend-builder /app/packages/backend ./packages/backend
102113

114+
# Configure the database
115+
COPY --from=database-builder /app/node_modules ./node_modules
116+
COPY --from=database-builder /app/packages/db ./packages/db
117+
103118
COPY supervisord.conf /etc/supervisor/conf.d/supervisord.conf
104119
COPY prefix-output.sh ./prefix-output.sh
105120
RUN chmod +x ./prefix-output.sh

‎Makefile‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,11 @@
11

2-
CMDS := zoekt ui
2+
CMDS := zoekt yarn
33

44
ALL: $(CMDS)
55

6-
ui:
6+
yarn:
77
yarn install
8+
yarn workspace @sourcebot/db prisma:migrate:dev
89

910
zoekt:
1011
mkdir -p bin
@@ -20,6 +21,8 @@ clean:
2021
packages/web/.next \
2122
packages/backend/dist \
2223
packages/backend/node_modules \
24+
packages/db/node_modules \
25+
packages/db/dist \
2326
.sourcebot
2427

2528
.PHONY: bin

‎entrypoint.sh‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,11 @@ if [ ! -d "$DATA_CACHE_DIR" ]; then
1919
mkdir -p "$DATA_CACHE_DIR"
2020
fi
2121

22+
# Run a Database migration
23+
echo -e "\e[34m[Info] Running database migration...\e[0m"
24+
export DATABASE_URL="file:$DATA_CACHE_DIR/db.sqlite"
25+
yarn workspace @sourcebot/db prisma:migrate:prod
26+
2227
# In order to detect if this is the first run, we create a `.installed` file in
2328
# the cache directory.
2429
FIRST_RUN_FILE="$DATA_CACHE_DIR/.installedv2"

‎packages/backend/src/config.ts‎

Lines changed: 134 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,134 @@
1+
import { PrismaClient } from '@sourcebot/db';
2+
import { readFile } from 'fs/promises';
3+
import stripJsonComments from 'strip-json-comments';
4+
import { getGitHubReposFromConfig } from "./github.js";
5+
import { getGitLabReposFromConfig, GITLAB_CLOUD_HOSTNAME } from "./gitlab.js";
6+
import { SourcebotConfigurationSchema } from "./schemas/v2.js";
7+
import { AppContext } from "./types.js";
8+
import { getTokenFromConfig, isRemotePath, marshalBool } from "./utils.js";
9+
10+
export const syncConfig = async (configPath: string, db: PrismaClient, signal: AbortSignal, ctx: AppContext) => {
11+
const configContent = await (async () => {
12+
if (isRemotePath(configPath)) {
13+
const response = await fetch(configPath, {
14+
signal,
15+
});
16+
if (!response.ok) {
17+
throw new Error(`Failed to fetch config file ${configPath}: ${response.statusText}`);
18+
}
19+
return response.text();
20+
} else {
21+
return readFile(configPath, {
22+
encoding: 'utf-8',
23+
signal,
24+
});
25+
}
26+
})();
27+
28+
// @todo: we should validate the configuration file's structure here.
29+
const config = JSON.parse(stripJsonComments(configContent)) as SourcebotConfigurationSchema;
30+
31+
for (const repoConfig of config.repos ?? []) {
32+
switch (repoConfig.type) {
33+
case 'github': {
34+
const token = repoConfig.token ? getTokenFromConfig(repoConfig.token, ctx) : undefined;
35+
const gitHubRepos = await getGitHubReposFromConfig(repoConfig, signal, ctx);
36+
const hostUrl = repoConfig.url ?? 'https://github.com';
37+
const hostname = repoConfig.url ? new URL(repoConfig.url).hostname : 'github.com';
38+
39+
await Promise.all(gitHubRepos.map((repo) => {
40+
const repoName = `${hostname}/${repo.full_name}`;
41+
const cloneUrl = new URL(repo.clone_url!);
42+
if (token) {
43+
cloneUrl.username = token;
44+
}
45+
46+
const data = {
47+
external_id: repo.id.toString(),
48+
external_codeHostType: 'github',
49+
external_codeHostUrl: hostUrl,
50+
cloneUrl: cloneUrl.toString(),
51+
name: repoName,
52+
isFork: repo.fork,
53+
isArchived: !!repo.archived,
54+
metadata: {
55+
'zoekt.web-url-type': 'github',
56+
'zoekt.web-url': repo.html_url,
57+
'zoekt.name': repoName,
58+
'zoekt.github-stars': (repo.stargazers_count ?? 0).toString(),
59+
'zoekt.github-watchers': (repo.watchers_count ?? 0).toString(),
60+
'zoekt.github-subscribers': (repo.subscribers_count ?? 0).toString(),
61+
'zoekt.github-forks': (repo.forks_count ?? 0).toString(),
62+
'zoekt.archived': marshalBool(repo.archived),
63+
'zoekt.fork': marshalBool(repo.fork),
64+
'zoekt.public': marshalBool(repo.private === false)
65+
},
66+
};
67+
68+
return db.repo.upsert({
69+
where: {
70+
external_id_external_codeHostUrl: {
71+
external_id: repo.id.toString(),
72+
external_codeHostUrl: hostUrl,
73+
},
74+
},
75+
create: data,
76+
update: data,
77+
})
78+
}));
79+
80+
break;
81+
}
82+
case 'gitlab': {
83+
const hostUrl = repoConfig.url ?? 'https://gitlab.com';
84+
const hostname = repoConfig.url ? new URL(repoConfig.url).hostname : GITLAB_CLOUD_HOSTNAME;
85+
const token = repoConfig.token ? getTokenFromConfig(repoConfig.token, ctx) : undefined;
86+
const gitLabRepos = await getGitLabReposFromConfig(repoConfig, ctx);
87+
88+
await Promise.all(gitLabRepos.map((project) => {
89+
const repoName = `${hostname}/${project.path_with_namespace}`;
90+
const isFork = project.forked_from_project !== undefined;
91+
92+
const cloneUrl = new URL(project.http_url_to_repo);
93+
if (token) {
94+
cloneUrl.username = 'oauth2';
95+
cloneUrl.password = token;
96+
}
97+
98+
const data = {
99+
external_id: project.id.toString(),
100+
external_codeHostType: 'gitlab',
101+
external_codeHostUrl: hostUrl,
102+
cloneUrl: cloneUrl.toString(),
103+
name: repoName,
104+
isFork,
105+
isArchived: project.archived,
106+
metadata: {
107+
'zoekt.web-url-type': 'gitlab',
108+
'zoekt.web-url': project.web_url,
109+
'zoekt.name': repoName,
110+
'zoekt.gitlab-stars': project.star_count?.toString() ?? '0',
111+
'zoekt.gitlab-forks': project.forks_count?.toString() ?? '0',
112+
'zoekt.archived': marshalBool(project.archived),
113+
'zoekt.fork': marshalBool(isFork),
114+
'zoekt.public': marshalBool(project.visibility === 'public'),
115+
}
116+
}
117+
118+
return db.repo.upsert({
119+
where: {
120+
external_id_external_codeHostUrl: {
121+
external_id: project.id.toString(),
122+
external_codeHostUrl: hostUrl,
123+
},
124+
},
125+
create: data,
126+
update: data,
127+
})
128+
}));
129+
130+
break;
131+
}
132+
}
133+
}
134+
}

‎packages/backend/src/constants.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,6 @@ import { Settings } from "./types.js";
66
export const DEFAULT_SETTINGS: Settings = {
77
maxFileSize: 2 * 1024 * 1024, // 2MB in bytes
88
autoDeleteStaleRepos: true,
9-
reindexInterval: 1000 * 60 * 60, // 1 hour in milliseconds
10-
resyncInterval: 1000 * 60 * 60 * 24, // 1 day in milliseconds
9+
reindexIntervalMs: 1000 * 60 * 60, // 1 hour in milliseconds
10+
resyncIntervalMs: 1000 * 60 * 60 * 24, // 1 day in milliseconds
1111
}

‎packages/backend/src/db.test.ts‎

Lines changed: 0 additions & 125 deletions
This file was deleted.

0 commit comments

Comments
 (0)