feat: add config on source

api/package.json

@@ -74,6 +74,7 @@
     },
     "dependencies": {
         "@gitbeaker/core": "^43.4.0",
+        "@gitbeaker/requester-utils": "^43.8.0",
         "@gitbeaker/rest": "^43.4.0",
         "@octokit/rest": "^22.0.0",
         "@sentry/node": "^10.23.0",

api/scripts/seed.ts

@@ -39,7 +39,8 @@ const seed = async () => {
         description: undefined,
         url: "https://www.wikidata.org/",
         kind: "wikidata",
-        priority: 1
+        priority: 1,
+        configuration: undefined
     } satisfies Source;
     await db.insertInto("sources").values(source).execute();
 

api/src/core/adapters/CNLL/index.ts

@@ -2,20 +2,25 @@
 // SPDX-FileCopyrightText: 2024-2025 Université Grenoble Alpes
 // SPDX-License-Identifier: MIT
 
-import { SecondarySourceGateway } from "../../ports/SourceGateway";
+import { SourceGateway } from "../../ports/SourceGateway";
 import { getCNLLSoftwareExternalData } from "./getExternalData";
 import { getCnllPrestatairesSill } from "../getCnllPrestatairesSill";
 
-export const cnllSourceGateway: SecondarySourceGateway = {
-    sourceType: "ComptoirDuLibre",
-    sourceProfile: "Secondary",
-    softwareExternal: { getById: getCNLLSoftwareExternalData },
-    discoverSoftwareLinks: async () => {
-        const cnllProviders = await getCnllPrestatairesSill();
-        return cnllProviders.map(provider => ({
-            externalId: provider.sill_id.toString(),
-            softwareId: provider.sill_id,
-            softwareName: provider.nom
-        }));
+export type CNLLGateway = SourceGateway & {
+    softwareExtra: NonNullable<SourceGateway["softwareExtra"]>;
+};
+
+export const cnllSourceGateway: CNLLGateway = {
+    sourceType: "CNLL",
+    softwareExtra: {
+        getSoftwareExternal: getCNLLSoftwareExternalData,
+        getDiscoverSoftwareLinks: async () => {
+            const cnllProviders = await getCnllPrestatairesSill();
+            return cnllProviders.map(provider => ({
+                externalId: provider.sill_id.toString(),
+                softwareId: provider.sill_id,
+                softwareName: provider.nom
+            }));
+        }
     }
 };

api/src/core/adapters/GitHub/api/repo.ts

@@ -32,11 +32,8 @@ const parseURL = (repoUrl: string | URL): { repo: string; owner: string } => {
     };
 };
 
-export const repoGitHubEndpointMaker = (params: { githubPersonalAccessTokenForApiRateLimit?: string }) => {
-    const { githubPersonalAccessTokenForApiRateLimit } = params;
-    const octokit = new Octokit({
-        auth: githubPersonalAccessTokenForApiRateLimit
-    });
+export const gitHubEndpointMaker = (params?: { auth?: string }) => {
+    const octokit = new Octokit(params?.auth ? { auth: params.auth } : {});
 
     return {
         repo: {

api/src/core/adapters/GitHub/getExternalData.ts

@@ -8,15 +8,16 @@ import type { GetSoftwareExternal } from "../../ports/GetSoftwareExternal";
 import type { SoftwareExternal } from "../../types/SoftwareTypes";
 import { Source } from "../../usecases/readWriteSillData";
 import { identifersUtils } from "../../../tools/identifiersTools";
-import { repoGitHubEndpointMaker } from "./api/repo";
+import { gitHubEndpointMaker } from "./api/repo";
 
 export const getGitHubSoftwareExternalData: GetSoftwareExternal = memoize(
     async ({ externalId, source }: { externalId: string; source: Source }): Promise<SoftwareExternal | undefined> => {
         if (source.kind !== "GitHub") throw new Error("This source if not compatible with GitHub Adapter");
         if (source.url !== "https://github.com/")
             throw new Error("This source doesn't allow custom url, please set it properly.");
 
-        const gitHubApi = repoGitHubEndpointMaker({});
+        const configApi = source?.configuration?.auth ? { auth: source?.configuration?.auth } : {};
+        const gitHubApi = gitHubEndpointMaker(configApi);
         if (!gitHubApi) throw new Error("This GitHub url provided doesn't work.");
 
         const repoUrl = externalId.includes("https://github.com") ? externalId : `https://github.com/${externalId}`;

api/src/core/adapters/GitHub/getSofrwareFormData.ts

@@ -5,7 +5,7 @@
 import { GetSoftwareFormData } from "../../ports/GetSoftwareFormData";
 import { SoftwareFormData, Source } from "../../usecases/readWriteSillData";
 import { resolveOsAndPlatforms } from "../../utils";
-import { repoGitHubEndpointMaker } from "./api/repo";
+import { gitHubEndpointMaker } from "./api/repo";
 
 import memoize from "memoizee";
 
@@ -15,7 +15,8 @@ export const getGitHubSoftwareFOrm: GetSoftwareFormData = memoize(
         if (source.url !== "https://github.com/")
             throw new Error("This source doesn't allow custom url, please set it properly.");
 
-        const gitHubApi = repoGitHubEndpointMaker({});
+        const configApi = source?.configuration?.auth ? { auth: source?.configuration?.auth } : {};
+        const gitHubApi = gitHubEndpointMaker(configApi);
 
         const repoData = await gitHubApi.repo.get({ repoUrl: externalId });
         if (!repoData) throw new Error(`This GitHub url (${externalId}) provided doesn't work.`);

api/src/core/adapters/GitHub/getSoftwareOptions.ts

@@ -7,7 +7,7 @@ import type {
     SoftwareExternalDataOption
 } from "../../ports/GetSoftwareExternalDataOptions";
 import { Source } from "../../usecases/readWriteSillData";
-import { GitHubAPI, repoGitHubEndpointMaker } from "./api/repo";
+import { GitHubAPI, gitHubEndpointMaker } from "./api/repo";
 
 const gitHubSoftwareToExternalOption =
     ({ source }: { source: Source }) =>
@@ -29,7 +29,8 @@ export const getGitHubSoftwareOptions: GetSoftwareExternalDataOptions = async ({
     if (source.url !== "https://github.com/")
         throw new Error("This source doesn't allow custom url, please set it properly.");
 
-    const gitHubApi = repoGitHubEndpointMaker({});
+    const configApi = source?.configuration?.auth ? { auth: source?.configuration?.auth } : {};
+    const gitHubApi = gitHubEndpointMaker(configApi);
     const result = await gitHubApi.search.repo.searchByName(queryString);
 
     if (!result) return [];

api/src/core/adapters/GitHub/index.ts

@@ -2,15 +2,23 @@
 // SPDX-FileCopyrightText: 2024-2025 Université Grenoble Alpes
 // SPDX-License-Identifier: MIT
 
-import { PrimarySourceGateway } from "../../ports/SourceGateway";
+import { SourceGateway } from "../../ports/SourceGateway";
 import { getGitHubSoftwareExternalData } from "./getExternalData";
 import { getGitHubSoftwareFOrm } from "./getSofrwareFormData";
 import { getGitHubSoftwareOptions } from "./getSoftwareOptions";
 
-export const gitHubSourceGateway: PrimarySourceGateway = {
+export type GitHubGateway = SourceGateway & {
+    softwareExtra: NonNullable<SourceGateway["softwareExtra"]>;
+    software: NonNullable<SourceGateway["software"]>;
+};
+
+export const gitHubSourceGateway: GitHubGateway = {
     sourceType: "GitHub",
-    sourceProfile: "Primary",
-    softwareExternal: { getById: getGitHubSoftwareExternalData },
-    softwareOptions: { getById: getGitHubSoftwareOptions },
-    softwareForm: { getById: getGitHubSoftwareFOrm }
+    software: {
+        getSoftwareForm: getGitHubSoftwareFOrm,
+        getSoftwareOptions: getGitHubSoftwareOptions
+    },
+    softwareExtra: {
+        getSoftwareExternal: getGitHubSoftwareExternalData
+    }
 };

api/src/core/adapters/GitLab/getExternalData.ts

@@ -10,14 +10,13 @@ import type { SoftwareExternal } from "../../types/SoftwareTypes";
 import { Source } from "../../usecases/readWriteSillData";
 import { identifersUtils } from "../../../tools/identifiersTools";
 import { resolveExternalReferenceToProject } from "./api/utils";
+import { convertSourceConfigToBaseRequestOptions } from "../../../tools/sourceConfig";
 
 export const getGitLabSoftwareExternalData: GetSoftwareExternal = memoize(
     async ({ externalId, source }: { externalId: string; source: Source }): Promise<SoftwareExternal | undefined> => {
         if (source.kind !== "GitLab") throw new Error("This source if not compatible with GitLab Adapter");
 
-        const api = new Gitlab({
-            host: source.url
-        });
+        const api = new Gitlab(convertSourceConfigToBaseRequestOptions(source));
 
         const project = await resolveExternalReferenceToProject({ externalId, gitLabApi: api });
         if (!project) return;

api/src/core/adapters/GitLab/getSoftwareFormData.ts

@@ -9,14 +9,13 @@ import { GetSoftwareFormData } from "../../ports/GetSoftwareFormData";
 import { SoftwareFormData, Source } from "../../usecases/readWriteSillData";
 import { resolveOsAndPlatforms } from "../../utils";
 import { resolveExternalReferenceToProject } from "./api/utils";
+import { convertSourceConfigToBaseRequestOptions } from "../../../tools/sourceConfig";
 
 export const getGitLabSoftwareForm: GetSoftwareFormData = memoize(
     async ({ externalId, source }: { externalId: string; source: Source }): Promise<SoftwareFormData | undefined> => {
         if (source.kind !== "GitLab") throw new Error("This source if not compatible with GitLab Adapter");
 
-        const api = new Gitlab({
-            host: source.url
-        });
+        const api = new Gitlab(convertSourceConfigToBaseRequestOptions(source));
 
         const project = await resolveExternalReferenceToProject({ externalId, gitLabApi: api });
         if (!project) return;

api/src/core/adapters/GitLab/getSoftwareOptions.ts

@@ -8,6 +8,7 @@ import type {
     SoftwareExternalDataOption
 } from "../../ports/GetSoftwareExternalDataOptions";
 import { Source } from "../../usecases/readWriteSillData";
+import { convertSourceConfigToBaseRequestOptions } from "../../../tools/sourceConfig";
 
 const gitLabSoftwareToExternalOption =
     ({ source }: { source: Source }) =>
@@ -26,9 +27,7 @@ export const getGitLabSoftwareOptions: GetSoftwareExternalDataOptions = async ({
 
     console.info(`GitLab doesn't support multi languages, ${language} not used`);
 
-    const api = new Gitlab({
-        host: source.url
-    });
+    const api = new Gitlab(convertSourceConfigToBaseRequestOptions(source));
 
     const result = await api.Projects.all({ search: queryString });
 

api/src/core/adapters/GitLab/index.ts

@@ -2,15 +2,23 @@
 // SPDX-FileCopyrightText: 2024-2025 Université Grenoble Alpes
 // SPDX-License-Identifier: MIT
 
-import { PrimarySourceGateway } from "../../ports/SourceGateway";
+import { SourceGateway } from "../../ports/SourceGateway";
 import { getGitLabSoftwareExternalData } from "./getExternalData";
 import { getGitLabSoftwareForm } from "./getSoftwareFormData";
 import { getGitLabSoftwareOptions } from "./getSoftwareOptions";
 
-export const gitLabSourceGateway: PrimarySourceGateway = {
+export const gitLabSourceGateway: GitLabGateway = {
     sourceType: "GitLab",
-    sourceProfile: "Primary",
-    softwareExternal: { getById: getGitLabSoftwareExternalData },
-    softwareOptions: { getById: getGitLabSoftwareOptions },
-    softwareForm: { getById: getGitLabSoftwareForm }
+    software: {
+        getSoftwareForm: getGitLabSoftwareForm,
+        getSoftwareOptions: getGitLabSoftwareOptions
+    },
+    softwareExtra: {
+        getSoftwareExternal: getGitLabSoftwareExternalData
+    }
+};
+
+export type GitLabGateway = SourceGateway & {
+    softwareExtra: NonNullable<SourceGateway["softwareExtra"]>;
+    software: NonNullable<SourceGateway["software"]>;
 };

api/src/core/adapters/comptoirDuLibre/index.ts

@@ -5,26 +5,34 @@
 import { getCDLSoftwareOptions } from "./getCDLSoftwareOptions";
 import { getCDLSoftwareExternalData } from "./getCDLExternalData";
 import { getCDLFormData } from "./getCDLFormData";
-import { PrimarySourceGateway } from "../../ports/SourceGateway";
+import { SourceGateway } from "../../ports/SourceGateway";
 import { comptoirDuLibreApi } from "../comptoirDuLibreApi";
 
-export const comptoirDuLibreSourceGateway: PrimarySourceGateway = {
+export type CDLGateway = SourceGateway & {
+    softwareExtra: NonNullable<SourceGateway["softwareExtra"]>;
+    software: NonNullable<SourceGateway["software"]>;
+};
+
+export const comptoirDuLibreSourceGateway: CDLGateway = {
     sourceType: "ComptoirDuLibre",
-    sourceProfile: "Primary",
-    softwareExternal: { getById: getCDLSoftwareExternalData },
-    softwareOptions: { getById: getCDLSoftwareOptions },
-    softwareForm: { getById: getCDLFormData },
-    discoverSoftwareLinks: async () => {
-        const cdlData = await comptoirDuLibreApi.getComptoirDuLibre();
-        return cdlData.softwares
-            .filter(software => !Array.isArray(software.external_resources.sill))
-            .map(software => {
-                const sill = software.external_resources.sill as { id: number };
-                return {
-                    externalId: software.id.toString(),
-                    softwareId: sill.id,
-                    softwareName: software.name
-                };
-            });
+    software: {
+        getSoftwareForm: getCDLFormData,
+        getSoftwareOptions: getCDLSoftwareOptions
+    },
+    softwareExtra: {
+        getSoftwareExternal: getCDLSoftwareExternalData,
+        getDiscoverSoftwareLinks: async () => {
+            const cdlData = await comptoirDuLibreApi.getComptoirDuLibre();
+            return cdlData.softwares
+                .filter(software => !Array.isArray(software.external_resources.sill))
+                .map(software => {
+                    const sill = software.external_resources.sill as { id: number };
+                    return {
+                        externalId: software.id.toString(),
+                        softwareId: sill.id,
+                        softwareName: software.name
+                    };
+                });
+        }
     }
 };

api/src/core/adapters/comptoirDuLibreApi.ts

@@ -2,7 +2,6 @@
 // SPDX-FileCopyrightText: 2024-2025 Université Grenoble Alpes
 // SPDX-License-Identifier: MIT
 
-import fetch from "node-fetch";
 import { type ComptoirDuLibreApi, zComptoirDuLibre } from "../ports/ComptoirDuLibreApi";
 import cheerio, { type CheerioAPI, type Cheerio, type Element } from "cheerio";
 import memoize from "memoizee";

api/src/core/adapters/dbApi/kysely/kysely.database.ts

@@ -140,12 +140,19 @@ type SimilarExternalSoftwareExternalDataTable = {
     sourceSlug: string;
 };
 
+export type SourceConfig = {
+    queryTimeout?: number;
+    auth?: string;
+    rateLimitRetryDuration?: number;
+};
+
 type SourcesTable = {
     slug: string;
     kind: ExternalDataOriginKind;
     url: string;
     priority: number;
     description: JSONColumnType<LocalizedString> | null;
+    configuration: JSONColumnType<SourceConfig> | null;
 };
 
 type SoftwareAttributeDefinitionsTable = {

api/src/core/adapters/dbApi/kysely/migrations/1774540759933_add-config-on-source.ts

@@ -0,0 +1,9 @@
+import type { Kysely } from "kysely";
+
+export async function up(db: Kysely<any>): Promise<void> {
+    await db.schema.alterTable("sources").addColumn("configuration", "jsonb").execute();
+}
+
+export async function down(db: Kysely<any>): Promise<void> {
+    await db.schema.alterTable("sources").dropColumn("configuration").execute();
+}

api/src/core/adapters/getCnllPrestatairesSill.ts

@@ -2,9 +2,7 @@
 // SPDX-FileCopyrightText: 2024-2025 Université Grenoble Alpes
 // SPDX-License-Identifier: MIT
 
-import fetch from "node-fetch";
 import memoize from "memoizee";
-import * as https from "https";
 import { z } from "zod";
 import { zCnllPrestatairesSill, type GetCnllPrestatairesSill } from "../ports/GetCnllPrestatairesSill";
 
@@ -14,7 +12,7 @@ export const getCnllPrestatairesSill: GetCnllPrestatairesSill = memoize(
     async () => {
         try {
             console.info("Fetching cnll prestataires sill");
-            const res = await fetch(url, { "agent": new https.Agent({ "rejectUnauthorized": false }) });
+            const res = await fetch(url);
 
             if (res.status !== 200) {
                 throw new Error(`Failed to fetch ${url}`);