Gerrit sync (#104)

* Basic gerrit sync with working gitiles web-links functionality

This adds basic support for gerrit repo code host syncing. Gerrit uses
gitiles plugin for code browsing (in most cases).
It may be usefull to allow users to provide their own web code-browsing
url templates in the future.

* Add gerrit readme update

* Remove config arg from gerrit fetchAllProjects

* Remove example urls

* Resolve comments

Author: KonradStanski

README.md

@@ -30,7 +30,7 @@ https://github.com/user-attachments/assets/98d46192-5469-430f-ad9e-5c042adbb10d
 
 ## Features
 - 💻 **One-command deployment**: Get started instantly using Docker on your own machine.
-- 🔍 **Multi-repo search**: Effortlessly index and search through multiple public and private repositories in GitHub, GitLab, or Gitea.
+- 🔍 **Multi-repo search**: Effortlessly index and search through multiple public and private repositories in GitHub, GitLab, Gitea, or Gerrit.
 - ⚡**Lightning fast performance**: Built on top of the powerful [Zoekt](https://github.com/sourcegraph/zoekt) search engine.
 - 📂 **Full file visualization**: Instantly view the entire file when selecting any search result.
 - 🎨 **Modern web app**: Enjoy a sleek interface with features like syntax highlighting, light/dark mode, and vim-style navigation 
@@ -62,7 +62,7 @@ Sourcebot supports indexing and searching through public and private repositorie
 <picture>
     <source media="(prefers-color-scheme: dark)" srcset=".github/images/github-favicon-inverted.png">
     <img src="https://github.com/favicon.ico" width="16" height="16" alt="GitHub icon">
-</picture> GitHub, <img src="https://gitlab.com/favicon.ico" width="16" height="16" /> GitLab and <img src="https://gitea.com/favicon.ico" width="16" height="16"> Gitea. This section will guide you through configuring the repositories that Sourcebot indexes. 
+</picture> GitHub, <img src="https://gitlab.com/favicon.ico" width="16" height="16" /> GitLab, <img src="https://gitea.com/favicon.ico" width="16" height="16"> Gitea, and <img src="https://gerrit-review.googlesource.com/favicon.ico" width="16" height="16"> Gerrit. This section will guide you through configuring the repositories that Sourcebot indexes. 
 
 1. Create a new folder on your machine that stores your configs and `.sourcebot` cache, and navigate into it:
     ```sh
@@ -261,6 +261,12 @@ docker run -e <b>GITEA_TOKEN=my-secret-token</b> /* additional args */ ghcr.io/s
 
 </details>
 
+<details>
+<summary><img src="https://gerrit-review.googlesource.com/favicon.ico" width="16" height="16"> Gerrit</summary>
+Gerrit authentication is not yet currently supported.
+</details>
+
+
 </div>
 
 ## Using a self-hosted GitLab / GitHub instance
@@ -397,4 +403,4 @@ NEXT_PUBLIC_SOURCEBOT_TELEMETRY_DISABLED=1
 
 Sourcebot makes use of the following libraries:
 
-- [@vscode/codicons](https://github.com/microsoft/vscode-codicons) under the [CC BY 4.0 License](https://github.com/microsoft/vscode-codicons/blob/main/LICENSE).
+- [@vscode/codicons](https://github.com/microsoft/vscode-codicons) under the [CC BY 4.0 License](https://github.com/microsoft/vscode-codicons/blob/main/LICENSE).

packages/backend/src/gerrit.ts

@@ -0,0 +1,109 @@
+import fetch from 'cross-fetch';
+import { GerritConfig } from './schemas/v2.js';
+import { AppContext, GitRepository } from './types.js';
+import { createLogger } from './logger.js';
+import path from 'path';
+import { measure, marshalBool, excludeReposByName, includeReposByName } from './utils.js';
+
+// https://gerrit-review.googlesource.com/Documentation/rest-api.html
+interface GerritProjects {
+   [projectName: string]: GerritProjectInfo;
+}
+
+interface GerritProjectInfo {
+   id: string;
+   state?: string;
+   web_links?: GerritWebLink[];
+}
+
+interface GerritWebLink {
+   name: string;
+   url: string;
+}
+
+const logger = createLogger('Gerrit');
+
+export const getGerritReposFromConfig = async (config: GerritConfig, ctx: AppContext) => {
+
+   const url = config.url.endsWith('/') ? config.url : `${config.url}/`;
+   const hostname = new URL(config.url).hostname;
+
+   const { durationMs, data: projects } = await measure(() =>
+      fetchAllProjects(url)
+   );
+
+   // exclude "All-Projects" and "All-Users" projects
+   delete projects['All-Projects'];
+   delete projects['All-Users'];
+
+   logger.debug(`Fetched ${Object.keys(projects).length} projects in ${durationMs}ms.`);
+
+   let repos: GitRepository[] = Object.keys(projects).map((projectName) => {
+      const project = projects[projectName];
+      let webUrl = "https://www.gerritcodereview.com/";
+      // Gerrit projects can have multiple web links; use the first one
+      if (project.web_links) {
+         const webLink = project.web_links[0];
+         if (webLink) {
+            webUrl = webLink.url;
+         }
+      }
+      const repoId = `${hostname}/${projectName}`;
+      const repoPath = path.resolve(path.join(ctx.reposPath, `${repoId}.git`));
+
+      const cloneUrl = `${url}${encodeURIComponent(projectName)}`;
+
+      return {
+         vcs: 'git',
+         codeHost: 'gerrit',
+         name: projectName,
+         id: repoId,
+         cloneUrl: cloneUrl,
+         path: repoPath,
+         isStale: false, // Gerrit projects are typically not stale
+         isFork: false, // Gerrit doesn't have forks in the same way as GitHub
+         isArchived: false,
+         gitConfigMetadata: {
+            // Gerrit uses Gitiles for web UI. This can sometimes be "browse" type in zoekt
+            'zoekt.web-url-type': 'gitiles',
+            'zoekt.web-url': webUrl,
+            'zoekt.name': repoId,
+            'zoekt.archived': marshalBool(false),
+            'zoekt.fork': marshalBool(false),
+            'zoekt.public': marshalBool(true), // Assuming projects are public; adjust as needed
+         },
+         branches: [],
+         tags: []
+      } satisfies GitRepository;
+   });
+
+   // include repos by glob if specified in config
+   if (config.projects) {
+      repos = includeReposByName(repos, config.projects);
+   }
+
+   if (config.exclude && config.exclude.projects) {
+      repos = excludeReposByName(repos, config.exclude.projects);
+   }
+
+   return repos;
+};
+
+const fetchAllProjects = async (url: string): Promise<GerritProjects> => {
+
+   const projectsEndpoint = `${url}projects/`;
+   logger.debug(`Fetching projects from Gerrit at ${projectsEndpoint}...`);
+   const response = await fetch(projectsEndpoint);
+
+   if (!response.ok) {
+      throw new Error(`Failed to fetch projects from Gerrit: ${response.statusText}`);
+   }
+
+   const text = await response.text();
+
+   // Gerrit prepends ")]}'\n" to prevent XSSI attacks; remove it
+   // https://gerrit-review.googlesource.com/Documentation/rest-api.html
+   const jsonText = text.replace(")]}'\n", '');
+   const data = JSON.parse(jsonText);
+   return data;
+};

packages/backend/src/main.ts

@@ -4,6 +4,7 @@ import { SourcebotConfigurationSchema } from "./schemas/v2.js";
 import { getGitHubReposFromConfig } from "./github.js";
 import { getGitLabReposFromConfig } from "./gitlab.js";
 import { getGiteaReposFromConfig } from "./gitea.js";
+import { getGerritReposFromConfig } from "./gerrit.js";
 import { AppContext, LocalRepository, GitRepository, Repository } from "./types.js";
 import { cloneRepository, fetchRepository } from "./git.js";
 import { createLogger } from "./logger.js";
@@ -139,6 +140,11 @@ const syncConfig = async (configPath: string, db: Database, signal: AbortSignal,
                 configRepos.push(...giteaRepos);
                 break;
             }
+            case 'gerrit': {
+                const gerritRepos = await getGerritReposFromConfig(repoConfig, ctx);
+                configRepos.push(...gerritRepos);
+                break;
+            }
             case 'local': {
                 const repo = getLocalRepoFromConfig(repoConfig, ctx);
                 configRepos.push(repo);

packages/backend/src/schemas/v2.ts

@@ -1,6 +1,6 @@
 // THIS IS A AUTO-GENERATED FILE. DO NOT MODIFY MANUALLY!
 
-export type Repos = GitHubConfig | GitLabConfig | GiteaConfig | LocalConfig;
+export type Repos = GitHubConfig | GitLabConfig | GiteaConfig | GerritConfig | LocalConfig;
 
 /**
  * A Sourcebot configuration file outlines which repositories Sourcebot should sync and index.
@@ -173,6 +173,27 @@ export interface GiteaConfig {
   };
   revisions?: GitRevisions;
 }
+export interface GerritConfig {
+  /**
+   * Gerrit Configuration
+   */
+  type: "gerrit";
+  /**
+   * The URL of the Gerrit host.
+   */
+  url: string;
+  /**
+   * List of specific projects to sync. If not specified, all projects will be synced. Glob patterns are supported
+   */
+  projects?: string[];
+  exclude?: {
+    /**
+     * List of specific projects to exclude from syncing.
+     */
+    projects?: string[];
+  };
+  revisions?: GitRevisions;
+}
 export interface LocalConfig {
   /**
    * Local Configuration

packages/backend/src/utils.ts

@@ -48,6 +48,16 @@ export const excludeReposByName = <T extends Repository>(repos: T[], excludedRep
     });
 }
 
+export const includeReposByName = <T extends Repository>(repos: T[], includedRepoNames: string[], logger?: Logger) => {
+    return repos.filter((repo) => {
+        if (micromatch.isMatch(repo.name, includedRepoNames)) {
+            logger?.debug(`Including repo ${repo.id}. Reason: repos contain ${repo.name}`);
+            return true;
+        }
+        return false;
+    });
+}
+
 export const getTokenFromConfig = (token: string | { env: string }, ctx: AppContext) => {
     if (typeof token === 'string') {
         return token;

packages/web/public/gerrit.svg

@@ -3,6 +3,7 @@ import { twMerge } from "tailwind-merge"
 import githubLogo from "../../public/github.svg";
 import gitlabLogo from "../../public/gitlab.svg";
 import giteaLogo from "../../public/gitea.svg";
+import gerritLogo from "../../public/gerrit.svg";
 import { ServiceError } from "./serviceError";
 import { Repository } from "./types";
 
@@ -31,7 +32,7 @@ export const createPathWithQueryParams = (path: string, ...queryParams: [string,
 }
 
 type CodeHostInfo = {
-    type: "github" | "gitlab" | "gitea";
+    type: "github" | "gitlab" | "gitea" | "gerrit";
     displayName: string;
     costHostName: string;
     repoLink: string;
@@ -44,15 +45,14 @@ export const getRepoCodeHostInfo = (repo?: Repository): CodeHostInfo | undefined
         return undefined;
     }
 
-    const hostType = repo.RawConfig ? repo.RawConfig['web-url-type'] : undefined;
-    if (!hostType) {
+    const webUrlType = repo.RawConfig ? repo.RawConfig['web-url-type'] : undefined;
+    if (!webUrlType) {
         return undefined;
     }
 
     const url = new URL(repo.URL);
     const displayName = url.pathname.slice(1);
-
-    switch (hostType) {
+    switch (webUrlType) {
         case 'github':
             return {
                 type: "github",
@@ -78,6 +78,14 @@ export const getRepoCodeHostInfo = (repo?: Repository): CodeHostInfo | undefined
                 repoLink: repo.URL,
                 icon: giteaLogo,
             }
+        case 'gitiles':
+            return {
+                type: "gerrit",
+                displayName: displayName,
+                costHostName: "Gerrit",
+                repoLink: repo.URL,
+                icon: gerritLogo,
+            }
     }
 }
 
@@ -113,4 +121,4 @@ export const base64Decode = (base64: string): string => {
 // @see: https://stackoverflow.com/a/65959350/23221295
 export const isDefined = <T>(arg: T | null | undefined): arg is T extends null | undefined ? never : T => {
     return arg !== null && arg !== undefined;
-}
+}

packages/web/src/lib/utils.ts

@@ -356,6 +356,61 @@
             ],
             "additionalProperties": false
         },
+        "GerritConfig": {
+            "type": "object",
+            "properties": {
+                "type": {
+                    "const": "gerrit",
+                    "description": "Gerrit Configuration"
+                },
+                "url": {
+                    "type": "string",
+                    "format": "url",
+                    "description": "The URL of the Gerrit host.",
+                    "examples": [
+                        "https://gerrit.example.com"
+                    ],
+                    "pattern": "^https?:\\/\\/[^\\s/$.?#].[^\\s]*$"
+                },
+                "projects": {
+                    "type": "array",
+                    "items": {
+                        "type": "string"
+                    },
+                    "description": "List of specific projects to sync. If not specified, all projects will be synced. Glob patterns are supported",
+                    "examples": [
+                        [
+                            "project1/repo1",
+                            "project2/**"
+                        ]
+                    ]
+                },
+                "exclude": {
+                    "type": "object",
+                    "properties": {
+                        "projects": {
+                            "type": "array",
+                            "items": {
+                                "type": "string"
+                            },
+                            "examples": [
+                                [
+                                    "project1/repo1",
+                                    "project2/**"
+                                ]
+                            ],
+                            "description": "List of specific projects to exclude from syncing."
+                        }
+                    },
+                    "additionalProperties": false
+                }
+            },
+            "required": [
+                "type",
+                "url"
+            ],
+            "additionalProperties": false
+        },
         "LocalConfig": {
             "type": "object",
             "properties": {
@@ -415,6 +470,9 @@
                 {
                     "$ref": "#/definitions/GiteaConfig"
                 },
+                {
+                    "$ref": "#/definitions/GerritConfig"
+                },
                 {
                     "$ref": "#/definitions/LocalConfig"
                 }