fix: make gerrit repo fetching paginated (#114)

* fix: make gerrit repo fetching paginated

In some cases gerrit will limit the number of projects returned by
/projects endpoint, forcing the client to paginate their request to get
all projects.
This fulfills this requirement to get all projects

* Add some more metadata repo projects to ignore list

---------

Co-authored-by: Brendan Kellam <[email protected]>

Author: KonradStanski

packages/backend/src/gerrit.ts

@@ -23,7 +23,7 @@ interface GerritWebLink {
 
 const logger = createLogger('Gerrit');
 
-export const getGerritReposFromConfig = async (config: GerritConfig, ctx: AppContext) => {
+export const getGerritReposFromConfig = async (config: GerritConfig, ctx: AppContext): Promise<GitRepository[]> => {
 
    const url = config.url.endsWith('/') ? config.url : `${config.url}/`;
    const hostname = new URL(config.url).hostname;
@@ -35,6 +35,8 @@ export const getGerritReposFromConfig = async (config: GerritConfig, ctx: AppCon
    // exclude "All-Projects" and "All-Users" projects
    delete projects['All-Projects'];
    delete projects['All-Users'];
+   delete projects['All-Avatars']
+   delete projects['All-Archived-Projects']
 
    logger.debug(`Fetched ${Object.keys(projects).length} projects in ${durationMs}ms.`);
 
@@ -90,20 +92,35 @@ export const getGerritReposFromConfig = async (config: GerritConfig, ctx: AppCon
 };
 
 const fetchAllProjects = async (url: string): Promise<GerritProjects> => {
-
    const projectsEndpoint = `${url}projects/`;
-   logger.debug(`Fetching projects from Gerrit at ${projectsEndpoint}...`);
-   const response = await fetch(projectsEndpoint);
+   let allProjects: GerritProjects = {};
+   let start = 0; // Start offset for pagination
+   let hasMoreProjects = true;
 
-   if (!response.ok) {
-      throw new Error(`Failed to fetch projects from Gerrit: ${response.statusText}`);
-   }
+   while (hasMoreProjects) {
+      const endpointWithParams = `${projectsEndpoint}?S=${start}`;
+      logger.debug(`Fetching projects from Gerrit at ${endpointWithParams}`);
+
+      const response = await fetch(endpointWithParams);
+      if (!response.ok) {
+         throw new Error(`Failed to fetch projects from Gerrit: ${response.statusText}`);
+      }
 
-   const text = await response.text();
+      const text = await response.text();
+      const jsonText = text.replace(")]}'\n", ''); // Remove XSSI protection prefix
+      const data: GerritProjects = JSON.parse(jsonText);
+
+      // Merge the current batch of projects with allProjects
+      Object.assign(allProjects, data);
+
+      // Check if there are more projects to fetch
+      hasMoreProjects = Object.values(data).some(
+         (project) => (project as any)._more_projects === true
+      );
+
+      // Update the offset based on the number of projects in the current response
+      start += Object.keys(data).length;
+   }
 
-   // Gerrit prepends ")]}'\n" to prevent XSSI attacks; remove it
-   // https://gerrit-review.googlesource.com/Documentation/rest-api.html
-   const jsonText = text.replace(")]}'\n", '');
-   const data = JSON.parse(jsonText);
-   return data;
+   return allProjects;
 };

packages/backend/src/schemas/v2.ts

@@ -192,7 +192,6 @@ export interface GerritConfig {
      */
     projects?: string[];
   };
-  revisions?: GitRevisions;
 }
 export interface LocalConfig {
   /**