Skip to content

Commit b452fd2

Browse files
KonradStanskiKonradStanski
authored
Gerrit sync (#104)
* Basic gerrit sync with working gitiles web-links functionality This adds basic support for gerrit repo code host syncing. Gerrit uses gitiles plugin for code browsing (in most cases). It may be usefull to allow users to provide their own web code-browsing url templates in the future. * Add gerrit readme update * Remove config arg from gerrit fetchAllProjects * Remove example urls * Resolve comments
1 parent d9710c7 commit b452fd2

File tree

8 files changed

+236
-10
lines changed

8 files changed

+236
-10
lines changed

‎README.md‎

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ https://github.com/user-attachments/assets/98d46192-5469-430f-ad9e-5c042adbb10d
3030

3131
## Features
3232
- 💻 **One-command deployment**: Get started instantly using Docker on your own machine.
33-
- 🔍 **Multi-repo search**: Effortlessly index and search through multiple public and private repositories in GitHub, GitLab, or Gitea.
33+
- 🔍 **Multi-repo search**: Effortlessly index and search through multiple public and private repositories in GitHub, GitLab, Gitea, or Gerrit.
3434
- ⚡**Lightning fast performance**: Built on top of the powerful [Zoekt](https://github.com/sourcegraph/zoekt) search engine.
3535
- 📂 **Full file visualization**: Instantly view the entire file when selecting any search result.
3636
- 🎨 **Modern web app**: Enjoy a sleek interface with features like syntax highlighting, light/dark mode, and vim-style navigation
@@ -62,7 +62,7 @@ Sourcebot supports indexing and searching through public and private repositorie
6262
<picture>
6363
<source media="(prefers-color-scheme: dark)" srcset=".github/images/github-favicon-inverted.png">
6464
<img src="https://github.com/favicon.ico" width="16" height="16" alt="GitHub icon">
65-
</picture> GitHub, <img src="https://gitlab.com/favicon.ico" width="16" height="16" /> GitLab and <img src="https://gitea.com/favicon.ico" width="16" height="16"> Gitea. This section will guide you through configuring the repositories that Sourcebot indexes.
65+
</picture> GitHub, <img src="https://gitlab.com/favicon.ico" width="16" height="16" /> GitLab, <img src="https://gitea.com/favicon.ico" width="16" height="16"> Gitea, and <img src="https://gerrit-review.googlesource.com/favicon.ico" width="16" height="16"> Gerrit. This section will guide you through configuring the repositories that Sourcebot indexes.
6666

6767
1. Create a new folder on your machine that stores your configs and `.sourcebot` cache, and navigate into it:
6868
```sh
@@ -261,6 +261,12 @@ docker run -e <b>GITEA_TOKEN=my-secret-token</b> /* additional args */ ghcr.io/s
261261

262262
</details>
263263

264+
<details>
265+
<summary><img src="https://gerrit-review.googlesource.com/favicon.ico" width="16" height="16"> Gerrit</summary>
266+
Gerrit authentication is not yet currently supported.
267+
</details>
268+
269+
264270
</div>
265271

266272
## Using a self-hosted GitLab / GitHub instance
@@ -397,4 +403,4 @@ NEXT_PUBLIC_SOURCEBOT_TELEMETRY_DISABLED=1
397403

398404
Sourcebot makes use of the following libraries:
399405

400-
- [@vscode/codicons](https://github.com/microsoft/vscode-codicons) under the [CC BY 4.0 License](https://github.com/microsoft/vscode-codicons/blob/main/LICENSE).
406+
- [@vscode/codicons](https://github.com/microsoft/vscode-codicons) under the [CC BY 4.0 License](https://github.com/microsoft/vscode-codicons/blob/main/LICENSE).

‎packages/backend/src/gerrit.ts‎

Lines changed: 109 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,109 @@
1+
import fetch from 'cross-fetch';
2+
import { GerritConfig } from './schemas/v2.js';
3+
import { AppContext, GitRepository } from './types.js';
4+
import { createLogger } from './logger.js';
5+
import path from 'path';
6+
import { measure, marshalBool, excludeReposByName, includeReposByName } from './utils.js';
7+
8+
// https://gerrit-review.googlesource.com/Documentation/rest-api.html
9+
interface GerritProjects {
10+
[projectName: string]: GerritProjectInfo;
11+
}
12+
13+
interface GerritProjectInfo {
14+
id: string;
15+
state?: string;
16+
web_links?: GerritWebLink[];
17+
}
18+
19+
interface GerritWebLink {
20+
name: string;
21+
url: string;
22+
}
23+
24+
const logger = createLogger('Gerrit');
25+
26+
export const getGerritReposFromConfig = async (config: GerritConfig, ctx: AppContext) => {
27+
28+
const url = config.url.endsWith('/') ? config.url : `${config.url}/`;
29+
const hostname = new URL(config.url).hostname;
30+
31+
const { durationMs, data: projects } = await measure(() =>
32+
fetchAllProjects(url)
33+
);
34+
35+
// exclude "All-Projects" and "All-Users" projects
36+
delete projects['All-Projects'];
37+
delete projects['All-Users'];
38+
39+
logger.debug(`Fetched ${Object.keys(projects).length} projects in ${durationMs}ms.`);
40+
41+
let repos: GitRepository[] = Object.keys(projects).map((projectName) => {
42+
const project = projects[projectName];
43+
let webUrl = "https://www.gerritcodereview.com/";
44+
// Gerrit projects can have multiple web links; use the first one
45+
if (project.web_links) {
46+
const webLink = project.web_links[0];
47+
if (webLink) {
48+
webUrl = webLink.url;
49+
}
50+
}
51+
const repoId = `${hostname}/${projectName}`;
52+
const repoPath = path.resolve(path.join(ctx.reposPath, `${repoId}.git`));
53+
54+
const cloneUrl = `${url}${encodeURIComponent(projectName)}`;
55+
56+
return {
57+
vcs: 'git',
58+
codeHost: 'gerrit',
59+
name: projectName,
60+
id: repoId,
61+
cloneUrl: cloneUrl,
62+
path: repoPath,
63+
isStale: false, // Gerrit projects are typically not stale
64+
isFork: false, // Gerrit doesn't have forks in the same way as GitHub
65+
isArchived: false,
66+
gitConfigMetadata: {
67+
// Gerrit uses Gitiles for web UI. This can sometimes be "browse" type in zoekt
68+
'zoekt.web-url-type': 'gitiles',
69+
'zoekt.web-url': webUrl,
70+
'zoekt.name': repoId,
71+
'zoekt.archived': marshalBool(false),
72+
'zoekt.fork': marshalBool(false),
73+
'zoekt.public': marshalBool(true), // Assuming projects are public; adjust as needed
74+
},
75+
branches: [],
76+
tags: []
77+
} satisfies GitRepository;
78+
});
79+
80+
// include repos by glob if specified in config
81+
if (config.projects) {
82+
repos = includeReposByName(repos, config.projects);
83+
}
84+
85+
if (config.exclude && config.exclude.projects) {
86+
repos = excludeReposByName(repos, config.exclude.projects);
87+
}
88+
89+
return repos;
90+
};
91+
92+
const fetchAllProjects = async (url: string): Promise<GerritProjects> => {
93+
94+
const projectsEndpoint = `${url}projects/`;
95+
logger.debug(`Fetching projects from Gerrit at ${projectsEndpoint}...`);
96+
const response = await fetch(projectsEndpoint);
97+
98+
if (!response.ok) {
99+
throw new Error(`Failed to fetch projects from Gerrit: ${response.statusText}`);
100+
}
101+
102+
const text = await response.text();
103+
104+
// Gerrit prepends ")]}'\n" to prevent XSSI attacks; remove it
105+
// https://gerrit-review.googlesource.com/Documentation/rest-api.html
106+
const jsonText = text.replace(")]}'\n", '');
107+
const data = JSON.parse(jsonText);
108+
return data;
109+
};

‎packages/backend/src/main.ts‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import { SourcebotConfigurationSchema } from "./schemas/v2.js";
44
import { getGitHubReposFromConfig } from "./github.js";
55
import { getGitLabReposFromConfig } from "./gitlab.js";
66
import { getGiteaReposFromConfig } from "./gitea.js";
7+
import { getGerritReposFromConfig } from "./gerrit.js";
78
import { AppContext, LocalRepository, GitRepository, Repository } from "./types.js";
89
import { cloneRepository, fetchRepository } from "./git.js";
910
import { createLogger } from "./logger.js";
@@ -139,6 +140,11 @@ const syncConfig = async (configPath: string, db: Database, signal: AbortSignal,
139140
configRepos.push(...giteaRepos);
140141
break;
141142
}
143+
case 'gerrit': {
144+
const gerritRepos = await getGerritReposFromConfig(repoConfig, ctx);
145+
configRepos.push(...gerritRepos);
146+
break;
147+
}
142148
case 'local': {
143149
const repo = getLocalRepoFromConfig(repoConfig, ctx);
144150
configRepos.push(repo);

‎packages/backend/src/schemas/v2.ts‎

Lines changed: 22 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
// THIS IS A AUTO-GENERATED FILE. DO NOT MODIFY MANUALLY!
22

3-
export type Repos = GitHubConfig | GitLabConfig | GiteaConfig | LocalConfig;
3+
export type Repos = GitHubConfig | GitLabConfig | GiteaConfig | GerritConfig | LocalConfig;
44

55
/**
66
* A Sourcebot configuration file outlines which repositories Sourcebot should sync and index.
@@ -173,6 +173,27 @@ export interface GiteaConfig {
173173
};
174174
revisions?: GitRevisions;
175175
}
176+
export interface GerritConfig {
177+
/**
178+
* Gerrit Configuration
179+
*/
180+
type: "gerrit";
181+
/**
182+
* The URL of the Gerrit host.
183+
*/
184+
url: string;
185+
/**
186+
* List of specific projects to sync. If not specified, all projects will be synced. Glob patterns are supported
187+
*/
188+
projects?: string[];
189+
exclude?: {
190+
/**
191+
* List of specific projects to exclude from syncing.
192+
*/
193+
projects?: string[];
194+
};
195+
revisions?: GitRevisions;
196+
}
176197
export interface LocalConfig {
177198
/**
178199
* Local Configuration

‎packages/backend/src/utils.ts‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,16 @@ export const excludeReposByName = <T extends Repository>(repos: T[], excludedRep
4848
});
4949
}
5050

51+
export const includeReposByName = <T extends Repository>(repos: T[], includedRepoNames: string[], logger?: Logger) => {
52+
return repos.filter((repo) => {
53+
if (micromatch.isMatch(repo.name, includedRepoNames)) {
54+
logger?.debug(`Including repo ${repo.id}. Reason: repos contain ${repo.name}`);
55+
return true;
56+
}
57+
return false;
58+
});
59+
}
60+
5161
export const getTokenFromConfig = (token: string | { env: string }, ctx: AppContext) => {
5262
if (typeof token === 'string') {
5363
return token;

‎packages/web/public/gerrit.svg‎

Lines changed: 8 additions & 0 deletions
Loading

‎packages/web/src/lib/utils.ts‎

Lines changed: 14 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ import { twMerge } from "tailwind-merge"
33
import githubLogo from "../../public/github.svg";
44
import gitlabLogo from "../../public/gitlab.svg";
55
import giteaLogo from "../../public/gitea.svg";
6+
import gerritLogo from "../../public/gerrit.svg";
67
import { ServiceError } from "./serviceError";
78
import { Repository } from "./types";
89

@@ -31,7 +32,7 @@ export const createPathWithQueryParams = (path: string, ...queryParams: [string,
3132
}
3233

3334
type CodeHostInfo = {
34-
type: "github" | "gitlab" | "gitea";
35+
type: "github" | "gitlab" | "gitea" | "gerrit";
3536
displayName: string;
3637
costHostName: string;
3738
repoLink: string;
@@ -44,15 +45,14 @@ export const getRepoCodeHostInfo = (repo?: Repository): CodeHostInfo | undefined
4445
return undefined;
4546
}
4647

47-
const hostType = repo.RawConfig ? repo.RawConfig['web-url-type'] : undefined;
48-
if (!hostType) {
48+
const webUrlType = repo.RawConfig ? repo.RawConfig['web-url-type'] : undefined;
49+
if (!webUrlType) {
4950
return undefined;
5051
}
5152

5253
const url = new URL(repo.URL);
5354
const displayName = url.pathname.slice(1);
54-
55-
switch (hostType) {
55+
switch (webUrlType) {
5656
case 'github':
5757
return {
5858
type: "github",
@@ -78,6 +78,14 @@ export const getRepoCodeHostInfo = (repo?: Repository): CodeHostInfo | undefined
7878
repoLink: repo.URL,
7979
icon: giteaLogo,
8080
}
81+
case 'gitiles':
82+
return {
83+
type: "gerrit",
84+
displayName: displayName,
85+
costHostName: "Gerrit",
86+
repoLink: repo.URL,
87+
icon: gerritLogo,
88+
}
8189
}
8290
}
8391

@@ -113,4 +121,4 @@ export const base64Decode = (base64: string): string => {
113121
// @see: https://stackoverflow.com/a/65959350/23221295
114122
export const isDefined = <T>(arg: T | null | undefined): arg is T extends null | undefined ? never : T => {
115123
return arg !== null && arg !== undefined;
116-
}
124+
}

‎schemas/v2/index.json‎

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -356,6 +356,61 @@
356356
],
357357
"additionalProperties": false
358358
},
359+
"GerritConfig": {
360+
"type": "object",
361+
"properties": {
362+
"type": {
363+
"const": "gerrit",
364+
"description": "Gerrit Configuration"
365+
},
366+
"url": {
367+
"type": "string",
368+
"format": "url",
369+
"description": "The URL of the Gerrit host.",
370+
"examples": [
371+
"https://gerrit.example.com"
372+
],
373+
"pattern": "^https?:\\/\\/[^\\s/$.?#].[^\\s]*$"
374+
},
375+
"projects": {
376+
"type": "array",
377+
"items": {
378+
"type": "string"
379+
},
380+
"description": "List of specific projects to sync. If not specified, all projects will be synced. Glob patterns are supported",
381+
"examples": [
382+
[
383+
"project1/repo1",
384+
"project2/**"
385+
]
386+
]
387+
},
388+
"exclude": {
389+
"type": "object",
390+
"properties": {
391+
"projects": {
392+
"type": "array",
393+
"items": {
394+
"type": "string"
395+
},
396+
"examples": [
397+
[
398+
"project1/repo1",
399+
"project2/**"
400+
]
401+
],
402+
"description": "List of specific projects to exclude from syncing."
403+
}
404+
},
405+
"additionalProperties": false
406+
}
407+
},
408+
"required": [
409+
"type",
410+
"url"
411+
],
412+
"additionalProperties": false
413+
},
359414
"LocalConfig": {
360415
"type": "object",
361416
"properties": {
@@ -415,6 +470,9 @@
415470
{
416471
"$ref": "#/definitions/GiteaConfig"
417472
},
473+
{
474+
"$ref": "#/definitions/GerritConfig"
475+
},
418476
{
419477
"$ref": "#/definitions/LocalConfig"
420478
}

0 commit comments

Comments
 (0)