diff --git a/CHANGELOG.md b/CHANGELOG.md index 59264a06f..ae9d14b13 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,6 +28,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Changed navbar indexing indicator to only report progress for first time indexing jobs. [#563](https://github.com/sourcebot-dev/sourcebot/pull/563) - Improved repo indexing job stability and robustness. [#563](https://github.com/sourcebot-dev/sourcebot/pull/563) - Improved repositories table. [#572](https://github.com/sourcebot-dev/sourcebot/pull/572) +- Improved connections table. [#579](https://github.com/sourcebot-dev/sourcebot/pull/579) ### Removed - Removed spam "login page loaded" log. [#552](https://github.com/sourcebot-dev/sourcebot/pull/552) diff --git a/docs/snippets/schemas/v3/app.schema.mdx b/docs/snippets/schemas/v3/app.schema.mdx index 7eabc79fe..6d5e5a212 100644 --- a/docs/snippets/schemas/v3/app.schema.mdx +++ b/docs/snippets/schemas/v3/app.schema.mdx @@ -3,11 +3,9 @@ { "$schema": "http://json-schema.org/draft-07/schema#", "title": "AppConfig", - "oneOf": [ - { - "$schema": "http://json-schema.org/draft-07/schema#", + "definitions": { + "GitHubAppConfig": { "type": "object", - "title": "GithubAppConfig", "properties": { "type": { "const": "githubApp", @@ -61,19 +59,70 @@ }, "required": [ "type", - "id" + "id", + "privateKey" ], - "oneOf": [ - { - "required": [ - "privateKey" + "additionalProperties": false + } + }, + "oneOf": [ + { + "type": "object", + "properties": { + "type": { + "const": "githubApp", + "description": "GitHub App Configuration" + }, + "deploymentHostname": { + "type": "string", + "format": "hostname", + "default": "github.com", + "description": "The hostname of the GitHub App deployment.", + "examples": [ + "github.com", + "github.example.com" ] }, - { - "required": [ - "privateKeyPath" + "id": { + "type": "string", + "description": "The ID of the GitHub App." + }, + "privateKey": { + "description": "The private key of the GitHub App.", + "anyOf": [ + { + "type": "object", + "properties": { + "secret": { + "type": "string", + "description": "The name of the secret that contains the token." + } + }, + "required": [ + "secret" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "env": { + "type": "string", + "description": "The name of the environment variable that contains the token. Only supported in declarative connection configs." + } + }, + "required": [ + "env" + ], + "additionalProperties": false + } ] } + }, + "required": [ + "type", + "id", + "privateKey" ], "additionalProperties": false } diff --git a/docs/snippets/schemas/v3/index.schema.mdx b/docs/snippets/schemas/v3/index.schema.mdx index c79cd2b37..84c73bc52 100644 --- a/docs/snippets/schemas/v3/index.schema.mdx +++ b/docs/snippets/schemas/v3/index.schema.mdx @@ -4280,11 +4280,9 @@ "items": { "$schema": "http://json-schema.org/draft-07/schema#", "title": "AppConfig", - "oneOf": [ - { - "$schema": "http://json-schema.org/draft-07/schema#", + "definitions": { + "GitHubAppConfig": { "type": "object", - "title": "GithubAppConfig", "properties": { "type": { "const": "githubApp", @@ -4338,19 +4336,70 @@ }, "required": [ "type", - "id" + "id", + "privateKey" ], - "oneOf": [ - { - "required": [ - "privateKey" - ] + "additionalProperties": false + } + }, + "oneOf": [ + { + "type": "object", + "properties": { + "type": { + "const": "githubApp", + "description": "GitHub App Configuration" }, - { - "required": [ - "privateKeyPath" + "deploymentHostname": { + "type": "string", + "format": "hostname", + "default": "github.com", + "description": "The hostname of the GitHub App deployment.", + "examples": [ + "github.com", + "github.example.com" ] + }, + "id": { + "type": "string", + "description": "The ID of the GitHub App." + }, + "privateKey": { + "anyOf": [ + { + "type": "object", + "properties": { + "secret": { + "type": "string", + "description": "The name of the secret that contains the token." + } + }, + "required": [ + "secret" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "env": { + "type": "string", + "description": "The name of the environment variable that contains the token. Only supported in declarative connection configs." + } + }, + "required": [ + "env" + ], + "additionalProperties": false + } + ], + "description": "The private key of the GitHub App." } + }, + "required": [ + "type", + "id", + "privateKey" ], "additionalProperties": false } diff --git a/packages/backend/package.json b/packages/backend/package.json index 8329fbbb7..f1466c0e7 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -40,6 +40,7 @@ "argparse": "^2.0.1", "azure-devops-node-api": "^15.1.1", "bullmq": "^5.34.10", + "chokidar": "^4.0.3", "cross-fetch": "^4.0.0", "dotenv": "^16.4.5", "express": "^4.21.2", diff --git a/packages/backend/src/azuredevops.ts b/packages/backend/src/azuredevops.ts index a06b9c090..de5879357 100644 --- a/packages/backend/src/azuredevops.ts +++ b/packages/backend/src/azuredevops.ts @@ -1,6 +1,6 @@ import { AzureDevOpsConnectionConfig } from "@sourcebot/schemas/v3/azuredevops.type"; import { createLogger } from "@sourcebot/logger"; -import { getTokenFromConfig, measure, fetchWithRetry } from "./utils.js"; +import { measure, fetchWithRetry } from "./utils.js"; import micromatch from "micromatch"; import { PrismaClient } from "@sourcebot/db"; import { BackendException, BackendError } from "@sourcebot/error"; @@ -8,6 +8,7 @@ import { processPromiseResults, throwIfAnyFailed } from "./connectionUtils.js"; import * as Sentry from "@sentry/node"; import * as azdev from "azure-devops-node-api"; import { GitRepository } from "azure-devops-node-api/interfaces/GitInterfaces.js"; +import { getTokenFromConfig } from "@sourcebot/crypto"; const logger = createLogger('azuredevops'); const AZUREDEVOPS_CLOUD_HOSTNAME = "dev.azure.com"; @@ -34,7 +35,7 @@ export const getAzureDevOpsReposFromConfig = async ( const baseUrl = config.url || `https://${AZUREDEVOPS_CLOUD_HOSTNAME}`; const token = config.token ? - await getTokenFromConfig(config.token, orgId, db, logger) : + await getTokenFromConfig(config.token, orgId, db) : undefined; if (!token) { @@ -47,47 +48,39 @@ export const getAzureDevOpsReposFromConfig = async ( const useTfsPath = config.useTfsPath || false; let allRepos: GitRepository[] = []; - let notFound: { - users: string[], - orgs: string[], - repos: string[], - } = { - users: [], - orgs: [], - repos: [], - }; + let allWarnings: string[] = []; if (config.orgs) { - const { validRepos, notFoundOrgs } = await getReposForOrganizations( + const { repos, warnings } = await getReposForOrganizations( config.orgs, baseUrl, token, useTfsPath ); - allRepos = allRepos.concat(validRepos); - notFound.orgs = notFoundOrgs; + allRepos = allRepos.concat(repos); + allWarnings = allWarnings.concat(warnings); } if (config.projects) { - const { validRepos, notFoundProjects } = await getReposForProjects( + const { repos, warnings } = await getReposForProjects( config.projects, baseUrl, token, useTfsPath ); - allRepos = allRepos.concat(validRepos); - notFound.repos = notFound.repos.concat(notFoundProjects); + allRepos = allRepos.concat(repos); + allWarnings = allWarnings.concat(warnings); } if (config.repos) { - const { validRepos, notFoundRepos } = await getRepos( + const { repos, warnings } = await getRepos( config.repos, baseUrl, token, useTfsPath ); - allRepos = allRepos.concat(validRepos); - notFound.repos = notFound.repos.concat(notFoundRepos); + allRepos = allRepos.concat(repos); + allWarnings = allWarnings.concat(warnings); } let repos = allRepos @@ -103,8 +96,8 @@ export const getAzureDevOpsReposFromConfig = async ( logger.debug(`Found ${repos.length} total repositories.`); return { - validRepos: repos, - notFound, + repos, + warnings: allWarnings, }; }; @@ -221,10 +214,11 @@ async function getReposForOrganizations( // Check if it's a 404-like error (organization not found) if (error && typeof error === 'object' && 'statusCode' in error && error.statusCode === 404) { - logger.error(`Organization ${org} not found or no access`); + const warning = `Organization ${org} not found or no access`; + logger.warn(warning); return { - type: 'notFound' as const, - value: org + type: 'warning' as const, + warning }; } throw error; @@ -232,11 +226,11 @@ async function getReposForOrganizations( })); throwIfAnyFailed(results); - const { validItems: validRepos, notFoundItems: notFoundOrgs } = processPromiseResults(results); + const { validItems: repos, warnings } = processPromiseResults(results); return { - validRepos, - notFoundOrgs, + repos, + warnings, }; } @@ -274,10 +268,11 @@ async function getReposForProjects( logger.error(`Failed to fetch repositories for project ${project}.`, error); if (error && typeof error === 'object' && 'statusCode' in error && error.statusCode === 404) { - logger.error(`Project ${project} not found or no access`); + const warning = `Project ${project} not found or no access`; + logger.warn(warning); return { - type: 'notFound' as const, - value: project + type: 'warning' as const, + warning }; } throw error; @@ -285,11 +280,11 @@ async function getReposForProjects( })); throwIfAnyFailed(results); - const { validItems: validRepos, notFoundItems: notFoundProjects } = processPromiseResults(results); + const { validItems: repos, warnings } = processPromiseResults(results); return { - validRepos, - notFoundProjects, + repos, + warnings, }; } @@ -328,10 +323,11 @@ async function getRepos( logger.error(`Failed to fetch repository ${repo}.`, error); if (error && typeof error === 'object' && 'statusCode' in error && error.statusCode === 404) { - logger.error(`Repository ${repo} not found or no access`); + const warning = `Repository ${repo} not found or no access`; + logger.warn(warning); return { - type: 'notFound' as const, - value: repo + type: 'warning' as const, + warning }; } throw error; @@ -339,10 +335,10 @@ async function getRepos( })); throwIfAnyFailed(results); - const { validItems: validRepos, notFoundItems: notFoundRepos } = processPromiseResults(results); + const { validItems: repos, warnings } = processPromiseResults(results); return { - validRepos, - notFoundRepos, + repos, + warnings, }; } \ No newline at end of file diff --git a/packages/backend/src/bitbucket.ts b/packages/backend/src/bitbucket.ts index cfa591cc3..75adc311b 100644 --- a/packages/backend/src/bitbucket.ts +++ b/packages/backend/src/bitbucket.ts @@ -4,7 +4,7 @@ import { BitbucketConnectionConfig } from "@sourcebot/schemas/v3/bitbucket.type" import type { ClientOptions, ClientPathsWithMethod } from "openapi-fetch"; import { createLogger } from "@sourcebot/logger"; import { PrismaClient } from "@sourcebot/db"; -import { getTokenFromConfig, measure, fetchWithRetry } from "./utils.js"; +import { measure, fetchWithRetry } from "./utils.js"; import * as Sentry from "@sentry/node"; import { SchemaRepository as CloudRepository, @@ -12,6 +12,7 @@ import { import { SchemaRestRepository as ServerRepository } from "@coderabbitai/bitbucket/server/openapi"; import { processPromiseResults } from "./connectionUtils.js"; import { throwIfAnyFailed } from "./connectionUtils.js"; +import { getTokenFromConfig } from "@sourcebot/crypto"; const logger = createLogger('bitbucket'); const BITBUCKET_CLOUD_GIT = 'https://bitbucket.org'; @@ -27,9 +28,9 @@ interface BitbucketClient { apiClient: any; baseUrl: string; gitUrl: string; - getReposForWorkspace: (client: BitbucketClient, workspaces: string[]) => Promise<{validRepos: BitbucketRepository[], notFoundWorkspaces: string[]}>; - getReposForProjects: (client: BitbucketClient, projects: string[]) => Promise<{validRepos: BitbucketRepository[], notFoundProjects: string[]}>; - getRepos: (client: BitbucketClient, repos: string[]) => Promise<{validRepos: BitbucketRepository[], notFoundRepos: string[]}>; + getReposForWorkspace: (client: BitbucketClient, workspaces: string[]) => Promise<{repos: BitbucketRepository[], warnings: string[]}>; + getReposForProjects: (client: BitbucketClient, projects: string[]) => Promise<{repos: BitbucketRepository[], warnings: string[]}>; + getRepos: (client: BitbucketClient, repos: string[]) => Promise<{repos: BitbucketRepository[], warnings: string[]}>; shouldExcludeRepo: (repo: BitbucketRepository, config: BitbucketConnectionConfig) => boolean; } @@ -59,7 +60,7 @@ type ServerPaginatedResponse = { export const getBitbucketReposFromConfig = async (config: BitbucketConnectionConfig, orgId: number, db: PrismaClient) => { const token = config.token ? - await getTokenFromConfig(config.token, orgId, db, logger) : + await getTokenFromConfig(config.token, orgId, db) : undefined; if (config.deploymentType === 'server' && !config.url) { @@ -71,32 +72,24 @@ export const getBitbucketReposFromConfig = async (config: BitbucketConnectionCon cloudClient(config.user, token); let allRepos: BitbucketRepository[] = []; - let notFound: { - orgs: string[], - users: string[], - repos: string[], - } = { - orgs: [], - users: [], - repos: [], - }; + let allWarnings: string[] = []; if (config.workspaces) { - const { validRepos, notFoundWorkspaces } = await client.getReposForWorkspace(client, config.workspaces); - allRepos = allRepos.concat(validRepos); - notFound.orgs = notFoundWorkspaces; + const { repos, warnings } = await client.getReposForWorkspace(client, config.workspaces); + allRepos = allRepos.concat(repos); + allWarnings = allWarnings.concat(warnings); } if (config.projects) { - const { validRepos, notFoundProjects } = await client.getReposForProjects(client, config.projects); - allRepos = allRepos.concat(validRepos); - notFound.orgs = notFoundProjects; + const { repos, warnings } = await client.getReposForProjects(client, config.projects); + allRepos = allRepos.concat(repos); + allWarnings = allWarnings.concat(warnings); } if (config.repos) { - const { validRepos, notFoundRepos } = await client.getRepos(client, config.repos); - allRepos = allRepos.concat(validRepos); - notFound.repos = notFoundRepos; + const { repos, warnings } = await client.getRepos(client, config.repos); + allRepos = allRepos.concat(repos); + allWarnings = allWarnings.concat(warnings); } const filteredRepos = allRepos.filter((repo) => { @@ -104,8 +97,8 @@ export const getBitbucketReposFromConfig = async (config: BitbucketConnectionCon }); return { - validRepos: filteredRepos, - notFound, + repos: filteredRepos, + warnings: allWarnings, }; } @@ -186,7 +179,7 @@ function parseUrl(url: string): { path: string; query: Record; } } -async function cloudGetReposForWorkspace(client: BitbucketClient, workspaces: string[]): Promise<{validRepos: CloudRepository[], notFoundWorkspaces: string[]}> { +async function cloudGetReposForWorkspace(client: BitbucketClient, workspaces: string[]): Promise<{repos: CloudRepository[], warnings: string[]}> { const results = await Promise.allSettled(workspaces.map(async (workspace) => { try { logger.debug(`Fetching all repos for workspace ${workspace}...`); @@ -221,10 +214,11 @@ async function cloudGetReposForWorkspace(client: BitbucketClient, workspaces: st const status = e?.cause?.response?.status; if (status == 404) { - logger.error(`Workspace ${workspace} not found or invalid access`) + const warning = `Workspace ${workspace} not found or invalid access`; + logger.warn(warning); return { - type: 'notFound' as const, - value: workspace + type: 'warning' as const, + warning } } throw e; @@ -232,21 +226,22 @@ async function cloudGetReposForWorkspace(client: BitbucketClient, workspaces: st })); throwIfAnyFailed(results); - const { validItems: validRepos, notFoundItems: notFoundWorkspaces } = processPromiseResults(results); + const { validItems: repos, warnings } = processPromiseResults(results); return { - validRepos, - notFoundWorkspaces, + repos, + warnings, }; } -async function cloudGetReposForProjects(client: BitbucketClient, projects: string[]): Promise<{validRepos: CloudRepository[], notFoundProjects: string[]}> { +async function cloudGetReposForProjects(client: BitbucketClient, projects: string[]): Promise<{repos: CloudRepository[], warnings: string[]}> { const results = await Promise.allSettled(projects.map(async (project) => { const [workspace, project_name] = project.split('/'); if (!workspace || !project_name) { - logger.error(`Invalid project ${project}`); + const warning = `Invalid project ${project}`; + logger.warn(warning); return { - type: 'notFound' as const, - value: project + type: 'warning' as const, + warning } } @@ -282,10 +277,11 @@ async function cloudGetReposForProjects(client: BitbucketClient, projects: strin const status = e?.cause?.response?.status; if (status == 404) { - logger.error(`Project ${project_name} not found in ${workspace} or invalid access`) + const warning = `Project ${project_name} not found in ${workspace} or invalid access`; + logger.warn(warning); return { - type: 'notFound' as const, - value: project + type: 'warning' as const, + warning } } throw e; @@ -293,21 +289,22 @@ async function cloudGetReposForProjects(client: BitbucketClient, projects: strin })); throwIfAnyFailed(results); - const { validItems: validRepos, notFoundItems: notFoundProjects } = processPromiseResults(results); + const { validItems: repos, warnings } = processPromiseResults(results); return { - validRepos, - notFoundProjects + repos, + warnings } } -async function cloudGetRepos(client: BitbucketClient, repos: string[]): Promise<{validRepos: CloudRepository[], notFoundRepos: string[]}> { - const results = await Promise.allSettled(repos.map(async (repo) => { +async function cloudGetRepos(client: BitbucketClient, repoList: string[]): Promise<{repos: CloudRepository[], warnings: string[]}> { + const results = await Promise.allSettled(repoList.map(async (repo) => { const [workspace, repo_slug] = repo.split('/'); if (!workspace || !repo_slug) { - logger.error(`Invalid repo ${repo}`); + const warning = `Invalid repo ${repo}`; + logger.warn(warning); return { - type: 'notFound' as const, - value: repo + type: 'warning' as const, + warning }; } @@ -329,10 +326,11 @@ async function cloudGetRepos(client: BitbucketClient, repos: string[]): Promise< const status = e?.cause?.response?.status; if (status === 404) { - logger.error(`Repo ${repo} not found in ${workspace} or invalid access`); + const warning = `Repo ${repo} not found in ${workspace} or invalid access`; + logger.warn(warning); return { - type: 'notFound' as const, - value: repo + type: 'warning' as const, + warning }; } throw e; @@ -340,10 +338,10 @@ async function cloudGetRepos(client: BitbucketClient, repos: string[]): Promise< })); throwIfAnyFailed(results); - const { validItems: validRepos, notFoundItems: notFoundRepos } = processPromiseResults(results); + const { validItems: repos, warnings } = processPromiseResults(results); return { - validRepos, - notFoundRepos + repos, + warnings }; } @@ -434,15 +432,16 @@ const getPaginatedServer = async ( return results; } -async function serverGetReposForWorkspace(client: BitbucketClient, workspaces: string[]): Promise<{validRepos: ServerRepository[], notFoundWorkspaces: string[]}> { +async function serverGetReposForWorkspace(client: BitbucketClient, workspaces: string[]): Promise<{repos: ServerRepository[], warnings: string[]}> { + const warnings = workspaces.map(workspace => `Workspaces are not supported in Bitbucket Server: ${workspace}`); logger.debug('Workspaces are not supported in Bitbucket Server'); return { - validRepos: [], - notFoundWorkspaces: workspaces + repos: [], + warnings }; } -async function serverGetReposForProjects(client: BitbucketClient, projects: string[]): Promise<{validRepos: ServerRepository[], notFoundProjects: string[]}> { +async function serverGetReposForProjects(client: BitbucketClient, projects: string[]): Promise<{repos: ServerRepository[], warnings: string[]}> { const results = await Promise.allSettled(projects.map(async (project) => { try { logger.debug(`Fetching all repos for project ${project}...`); @@ -477,10 +476,11 @@ async function serverGetReposForProjects(client: BitbucketClient, projects: stri const status = e?.cause?.response?.status; if (status == 404) { - logger.error(`Project ${project} not found or invalid access`); + const warning = `Project ${project} not found or invalid access`; + logger.warn(warning); return { - type: 'notFound' as const, - value: project + type: 'warning' as const, + warning }; } throw e; @@ -488,21 +488,22 @@ async function serverGetReposForProjects(client: BitbucketClient, projects: stri })); throwIfAnyFailed(results); - const { validItems: validRepos, notFoundItems: notFoundProjects } = processPromiseResults(results); + const { validItems: repos, warnings } = processPromiseResults(results); return { - validRepos, - notFoundProjects + repos, + warnings }; } -async function serverGetRepos(client: BitbucketClient, repos: string[]): Promise<{validRepos: ServerRepository[], notFoundRepos: string[]}> { - const results = await Promise.allSettled(repos.map(async (repo) => { +async function serverGetRepos(client: BitbucketClient, repoList: string[]): Promise<{repos: ServerRepository[], warnings: string[]}> { + const results = await Promise.allSettled(repoList.map(async (repo) => { const [project, repo_slug] = repo.split('/'); if (!project || !repo_slug) { - logger.error(`Invalid repo ${repo}`); + const warning = `Invalid repo ${repo}`; + logger.warn(warning); return { - type: 'notFound' as const, - value: repo + type: 'warning' as const, + warning }; } @@ -524,10 +525,11 @@ async function serverGetRepos(client: BitbucketClient, repos: string[]): Promise const status = e?.cause?.response?.status; if (status === 404) { - logger.error(`Repo ${repo} not found in project ${project} or invalid access`); + const warning = `Repo ${repo} not found in project ${project} or invalid access`; + logger.warn(warning); return { - type: 'notFound' as const, - value: repo + type: 'warning' as const, + warning }; } throw e; @@ -535,10 +537,10 @@ async function serverGetRepos(client: BitbucketClient, repos: string[]): Promise })); throwIfAnyFailed(results); - const { validItems: validRepos, notFoundItems: notFoundRepos } = processPromiseResults(results); + const { validItems: repos, warnings } = processPromiseResults(results); return { - validRepos, - notFoundRepos + repos, + warnings }; } diff --git a/packages/backend/src/configManager.ts b/packages/backend/src/configManager.ts new file mode 100644 index 000000000..14c6b0deb --- /dev/null +++ b/packages/backend/src/configManager.ts @@ -0,0 +1,126 @@ +import { Prisma, PrismaClient } from "@sourcebot/db"; +import { createLogger } from "@sourcebot/logger"; +import { ConnectionConfig } from "@sourcebot/schemas/v3/connection.type"; +import { loadConfig } from "@sourcebot/shared"; +import chokidar, { FSWatcher } from 'chokidar'; +import { ConnectionManager } from "./connectionManager.js"; +import { SINGLE_TENANT_ORG_ID } from "./constants.js"; +import { syncSearchContexts } from "./ee/syncSearchContexts.js"; + +const logger = createLogger('config-manager'); + +export class ConfigManager { + private watcher: FSWatcher; + + constructor( + private db: PrismaClient, + private connectionManager: ConnectionManager, + configPath: string, + ) { + this.watcher = chokidar.watch(configPath, { + ignoreInitial: true, // Don't fire events for existing files + awaitWriteFinish: { + stabilityThreshold: 100, // File size stable for 100ms + pollInterval: 100 // Check every 100ms + }, + atomic: true // Handle atomic writes (temp file + rename) + }); + + this.watcher.on('change', async () => { + logger.info(`Config file ${configPath} changed. Syncing config.`); + try { + await this.syncConfig(configPath); + } catch (error) { + logger.error(`Failed to sync config: ${error}`); + } + }); + + this.syncConfig(configPath); + } + + private syncConfig = async (configPath: string) => { + const config = await loadConfig(configPath); + + await this.syncConnections(config.connections); + await syncSearchContexts({ + contexts: config.contexts, + orgId: SINGLE_TENANT_ORG_ID, + db: this.db, + }); + } + + private syncConnections = async (connections?: { [key: string]: ConnectionConfig }) => { + if (connections) { + for (const [key, newConnectionConfig] of Object.entries(connections)) { + const existingConnection = await this.db.connection.findUnique({ + where: { + name_orgId: { + name: key, + orgId: SINGLE_TENANT_ORG_ID, + } + } + }); + + + const existingConnectionConfig = existingConnection ? existingConnection.config as unknown as ConnectionConfig : undefined; + const connectionNeedsSyncing = + !existingConnection || + (JSON.stringify(existingConnectionConfig) !== JSON.stringify(newConnectionConfig)); + + // Either update the existing connection or create a new one. + const connection = existingConnection ? + await this.db.connection.update({ + where: { + id: existingConnection.id, + }, + data: { + config: newConnectionConfig as unknown as Prisma.InputJsonValue, + isDeclarative: true, + } + }) : + await this.db.connection.create({ + data: { + name: key, + config: newConnectionConfig as unknown as Prisma.InputJsonValue, + connectionType: newConnectionConfig.type, + isDeclarative: true, + org: { + connect: { + id: SINGLE_TENANT_ORG_ID, + } + } + } + }); + + if (connectionNeedsSyncing) { + const [jobId] = await this.connectionManager.createJobs([connection]); + logger.info(`Change detected for connection '${key}' (id: ${connection.id}). Created sync job ${jobId}.`); + } + } + } + + // Delete any connections that are no longer in the config. + const deletedConnections = await this.db.connection.findMany({ + where: { + isDeclarative: true, + name: { + notIn: Object.keys(connections ?? {}), + }, + orgId: SINGLE_TENANT_ORG_ID, + } + }); + + for (const connection of deletedConnections) { + logger.info(`Deleting connection with name '${connection.name}'. Connection ID: ${connection.id}`); + await this.db.connection.delete({ + where: { + id: connection.id, + } + }) + } + } + + public dispose = async () => { + await this.watcher.close(); + } +} \ No newline at end of file diff --git a/packages/backend/src/connectionManager.ts b/packages/backend/src/connectionManager.ts index ce023fc50..40637ff0c 100644 --- a/packages/backend/src/connectionManager.ts +++ b/packages/backend/src/connectionManager.ts @@ -1,212 +1,219 @@ -import { Connection, ConnectionSyncStatus, PrismaClient, Prisma } from "@sourcebot/db"; -import { Job, Queue, Worker } from 'bullmq'; -import { Settings } from "./types.js"; -import { ConnectionConfig } from "@sourcebot/schemas/v3/connection.type"; +import * as Sentry from "@sentry/node"; +import { Connection, ConnectionSyncJobStatus, PrismaClient } from "@sourcebot/db"; import { createLogger } from "@sourcebot/logger"; +import { ConnectionConfig } from "@sourcebot/schemas/v3/connection.type"; +import { loadConfig } from "@sourcebot/shared"; +import { Job, Queue, ReservedJob, Worker } from "groupmq"; import { Redis } from 'ioredis'; -import { RepoData, compileGithubConfig, compileGitlabConfig, compileGiteaConfig, compileGerritConfig, compileBitbucketConfig, compileAzureDevOpsConfig, compileGenericGitHostConfig } from "./repoCompileUtils.js"; -import { BackendError, BackendException } from "@sourcebot/error"; -import { captureEvent } from "./posthog.js"; import { env } from "./env.js"; -import * as Sentry from "@sentry/node"; -import { loadConfig, syncSearchContexts } from "@sourcebot/shared"; +import { compileAzureDevOpsConfig, compileBitbucketConfig, compileGenericGitHostConfig, compileGerritConfig, compileGiteaConfig, compileGithubConfig, compileGitlabConfig } from "./repoCompileUtils.js"; +import { Settings } from "./types.js"; +import { groupmqLifecycleExceptionWrapper } from "./utils.js"; +import { syncSearchContexts } from "./ee/syncSearchContexts.js"; +import { captureEvent } from "./posthog.js"; +import { PromClient } from "./promClient.js"; -const QUEUE_NAME = 'connectionSyncQueue'; +const LOG_TAG = 'connection-manager'; +const logger = createLogger(LOG_TAG); +const createJobLogger = (jobId: string) => createLogger(`${LOG_TAG}:job:${jobId}`); type JobPayload = { + jobId: string, connectionId: number, connectionName: string, orgId: number, - config: ConnectionConfig, }; type JobResult = { repoCount: number, } +const JOB_TIMEOUT_MS = 1000 * 60 * 60 * 2; // 2 hour timeout + export class ConnectionManager { private worker: Worker; private queue: Queue; - private logger = createLogger('connection-manager'); private interval?: NodeJS.Timeout; constructor( private db: PrismaClient, private settings: Settings, redis: Redis, + private promClient: PromClient, ) { - this.queue = new Queue(QUEUE_NAME, { - connection: redis, + this.queue = new Queue({ + redis, + namespace: 'connection-sync-queue', + jobTimeoutMs: JOB_TIMEOUT_MS, + maxAttempts: 3, + logger: env.DEBUG_ENABLE_GROUPMQ_LOGGING === 'true', }); - this.worker = new Worker(QUEUE_NAME, this.runSyncJob.bind(this), { - connection: redis, + + this.worker = new Worker({ + queue: this.queue, + maxStalledCount: 1, + handler: this.runJob.bind(this), concurrency: this.settings.maxConnectionSyncJobConcurrency, + ...(env.DEBUG_ENABLE_GROUPMQ_LOGGING === 'true' ? { + logger: true, + } : {}), }); - this.worker.on('completed', this.onSyncJobCompleted.bind(this)); - this.worker.on('failed', this.onSyncJobFailed.bind(this)); - } - - public async scheduleConnectionSync(connection: Connection) { - await this.db.$transaction(async (tx) => { - await tx.connection.update({ - where: { id: connection.id }, - data: { syncStatus: ConnectionSyncStatus.IN_SYNC_QUEUE }, - }); - - const connectionConfig = connection.config as unknown as ConnectionConfig; - await this.queue.add('connectionSyncJob', { - connectionId: connection.id, - connectionName: connection.name, - orgId: connection.orgId, - config: connectionConfig, - }, { - removeOnComplete: env.REDIS_REMOVE_ON_COMPLETE, - removeOnFail: env.REDIS_REMOVE_ON_FAIL, - }); - this.logger.info(`Added job to queue for connection ${connection.name} (id: ${connection.id})`); - }).catch((err: unknown) => { - this.logger.error(`Failed to add job to queue for connection ${connection.name} (id: ${connection.id}): ${err}`); - }); + this.worker.on('completed', this.onJobCompleted.bind(this)); + this.worker.on('failed', this.onJobFailed.bind(this)); + this.worker.on('stalled', this.onJobStalled.bind(this)); + this.worker.on('error', this.onWorkerError.bind(this)); } public startScheduler() { - this.logger.debug('Starting scheduler'); + logger.debug('Starting scheduler'); this.interval = setInterval(async () => { const thresholdDate = new Date(Date.now() - this.settings.resyncConnectionIntervalMs); + const timeoutDate = new Date(Date.now() - JOB_TIMEOUT_MS); + const connections = await this.db.connection.findMany({ where: { - OR: [ - // When the connection needs to be synced, we want to sync it immediately. + AND: [ { - syncStatus: ConnectionSyncStatus.SYNC_NEEDED, + OR: [ + { syncedAt: null }, + { syncedAt: { lt: thresholdDate } }, + ] }, - // When the connection has already been synced, we only want to re-sync if the re-sync interval has elapsed - // (or if the date isn't set for some reason). { - AND: [ - { - OR: [ - { syncStatus: ConnectionSyncStatus.SYNCED }, - { syncStatus: ConnectionSyncStatus.SYNCED_WITH_WARNINGS }, - ] - }, - { - OR: [ - { syncedAt: null }, - { syncedAt: { lt: thresholdDate } }, - ] + NOT: { + syncJobs: { + some: { + OR: [ + // Don't schedule if there are active jobs that were created within the threshold date. + // This handles the case where a job is stuck in a pending state and will never be scheduled. + { + AND: [ + { status: { in: [ConnectionSyncJobStatus.PENDING, ConnectionSyncJobStatus.IN_PROGRESS] } }, + { createdAt: { gt: timeoutDate } }, + ] + }, + // Don't schedule if there are recent failed jobs (within the threshold date). + { + AND: [ + { status: ConnectionSyncJobStatus.FAILED }, + { completedAt: { gt: thresholdDate } }, + ] + } + ] + } } - ] + } } ] } }); - for (const connection of connections) { - await this.scheduleConnectionSync(connection); + + if (connections.length > 0) { + await this.createJobs(connections); } }, this.settings.resyncConnectionPollingIntervalMs); + + this.worker.run(); } - private async runSyncJob(job: Job): Promise { - const { config, orgId, connectionName } = job.data; - // @note: We aren't actually doing anything with this atm. - const abortController = new AbortController(); - const connection = await this.db.connection.findUnique({ - where: { - id: job.data.connectionId, - }, + public async createJobs(connections: Connection[]) { + const jobs = await this.db.connectionSyncJob.createManyAndReturn({ + data: connections.map(connection => ({ + connectionId: connection.id, + })), + include: { + connection: true, + } }); - if (!connection) { - const e = new BackendException(BackendError.CONNECTION_SYNC_CONNECTION_NOT_FOUND, { - message: `Connection ${job.data.connectionId} not found`, + for (const job of jobs) { + await this.queue.add({ + groupId: `connection:${job.connectionId}`, + data: { + jobId: job.id, + connectionId: job.connectionId, + connectionName: job.connection.name, + orgId: job.connection.orgId, + }, + jobId: job.id, }); - Sentry.captureException(e); - throw e; + + this.promClient.pendingConnectionSyncJobs.inc({ connection: job.connection.name }); } - // Reset the syncStatusMetadata to an empty object at the start of the sync job - await this.db.connection.update({ - where: { - id: job.data.connectionId, - }, - data: { - syncStatus: ConnectionSyncStatus.SYNCING, - syncStatusMetadata: {} - } - }) + return jobs.map(job => job.id); + } + private async runJob(job: ReservedJob): Promise { + const { jobId, connectionName } = job.data; + const logger = createJobLogger(jobId); + logger.info(`Running connection sync job ${jobId} for connection ${connectionName} (id: ${job.data.connectionId}) (attempt ${job.attempts + 1} / ${job.maxAttempts})`); - let result: { - repoData: RepoData[], - notFound: { - users: string[], - orgs: string[], - repos: string[], - } - } = { - repoData: [], - notFound: { - users: [], - orgs: [], - repos: [], - } - }; + this.promClient.pendingConnectionSyncJobs.dec({ connection: connectionName }); + this.promClient.activeConnectionSyncJobs.inc({ connection: connectionName }); - try { - result = await (async () => { - switch (config.type) { - case 'github': { - return await compileGithubConfig(config, job.data.connectionId, orgId, this.db, abortController); - } - case 'gitlab': { - return await compileGitlabConfig(config, job.data.connectionId, orgId, this.db); - } - case 'gitea': { - return await compileGiteaConfig(config, job.data.connectionId, orgId, this.db); - } - case 'gerrit': { - return await compileGerritConfig(config, job.data.connectionId, orgId); - } - case 'bitbucket': { - return await compileBitbucketConfig(config, job.data.connectionId, orgId, this.db); - } - case 'azuredevops': { - return await compileAzureDevOpsConfig(config, job.data.connectionId, orgId, this.db, abortController); - } - case 'git': { - return await compileGenericGitHostConfig(config, job.data.connectionId, orgId); + // @note: We aren't actually doing anything with this atm. + const abortController = new AbortController(); + + const { connection: { config: rawConnectionConfig, orgId } } = await this.db.connectionSyncJob.update({ + where: { + id: jobId, + }, + data: { + status: ConnectionSyncJobStatus.IN_PROGRESS, + }, + select: { + connection: { + select: { + config: true, + orgId: true, } } - })(); - } catch (err) { - this.logger.error(`Failed to compile repo data for connection ${job.data.connectionId} (${connectionName}): ${err}`); - Sentry.captureException(err); + }, + }); - if (err instanceof BackendException) { - throw err; - } else { - throw new BackendException(BackendError.CONNECTION_SYNC_SYSTEM_ERROR, { - message: `Failed to compile repo data for connection ${job.data.connectionId}`, - }); - } - } + const config = rawConnectionConfig as unknown as ConnectionConfig; - let { repoData, notFound } = result; + const result = await (async () => { + switch (config.type) { + case 'github': { + return await compileGithubConfig(config, job.data.connectionId, orgId, this.db, abortController); + } + case 'gitlab': { + return await compileGitlabConfig(config, job.data.connectionId, orgId, this.db); + } + case 'gitea': { + return await compileGiteaConfig(config, job.data.connectionId, orgId, this.db); + } + case 'gerrit': { + return await compileGerritConfig(config, job.data.connectionId, orgId); + } + case 'bitbucket': { + return await compileBitbucketConfig(config, job.data.connectionId, orgId, this.db); + } + case 'azuredevops': { + return await compileAzureDevOpsConfig(config, job.data.connectionId, orgId, this.db); + } + case 'git': { + return await compileGenericGitHostConfig(config, job.data.connectionId, orgId); + } + } + })(); + + let { repoData, warnings } = result; - // Push the information regarding not found users, orgs, and repos to the connection's syncStatusMetadata. Note that - // this won't be overwritten even if the connection job fails - await this.db.connection.update({ + await this.db.connectionSyncJob.update({ where: { - id: job.data.connectionId, + id: jobId, }, data: { - syncStatusMetadata: { notFound } - } + warningMessages: warnings, + }, }); + // Filter out any duplicates by external_id and external_codeHostUrl. repoData = repoData.filter((repo, index, self) => { return index === self.findIndex(r => @@ -233,7 +240,7 @@ export class ConnectionManager { } }); const deleteDuration = performance.now() - deleteStart; - this.logger.info(`Deleted all RepoToConnection records for connection ${connectionName} (id: ${job.data.connectionId}) in ${deleteDuration}ms`); + logger.info(`Deleted all RepoToConnection records for connection ${connectionName} (id: ${job.data.connectionId}) in ${deleteDuration}ms`); const totalUpsertStart = performance.now(); for (const repo of repoData) { @@ -250,10 +257,10 @@ export class ConnectionManager { create: repo, }) const upsertDuration = performance.now() - upsertStart; - this.logger.debug(`Upserted repo ${repo.displayName} (id: ${repo.external_id}) in ${upsertDuration}ms`); + logger.debug(`Upserted repo ${repo.displayName} (id: ${repo.external_id}) in ${upsertDuration}ms`); } const totalUpsertDuration = performance.now() - totalUpsertStart; - this.logger.info(`Upserted ${repoData.length} repos for connection ${connectionName} (id: ${job.data.connectionId}) in ${totalUpsertDuration}ms`); + logger.info(`Upserted ${repoData.length} repos for connection ${connectionName} (id: ${job.data.connectionId}) in ${totalUpsertDuration}ms`); }, { timeout: env.CONNECTION_MANAGER_UPSERT_TIMEOUT_MS }); return { @@ -262,106 +269,124 @@ export class ConnectionManager { } - private async onSyncJobCompleted(job: Job, result: JobResult) { - this.logger.info(`Connection sync job for connection ${job.data.connectionName} (id: ${job.data.connectionId}, jobId: ${job.id}) completed`); - const { connectionId, orgId } = job.data; + private onJobCompleted = async (job: Job) => + groupmqLifecycleExceptionWrapper('onJobCompleted', logger, async () => { + const logger = createJobLogger(job.id); + const { connectionId, connectionName, orgId } = job.data; - let syncStatusMetadata: Record = (await this.db.connection.findUnique({ - where: { id: connectionId }, - select: { syncStatusMetadata: true } - }))?.syncStatusMetadata as Record ?? {}; - const { notFound } = syncStatusMetadata as { - notFound: { - users: string[], - orgs: string[], - repos: string[], - } - }; + await this.db.connectionSyncJob.update({ + where: { + id: job.id, + }, + data: { + status: ConnectionSyncJobStatus.COMPLETED, + completedAt: new Date(), + connection: { + update: { + syncedAt: new Date(), + } + } + } + }); - await this.db.connection.update({ - where: { - id: connectionId, - }, - data: { - syncStatus: - notFound.users.length > 0 || - notFound.orgs.length > 0 || - notFound.repos.length > 0 ? ConnectionSyncStatus.SYNCED_WITH_WARNINGS : ConnectionSyncStatus.SYNCED, - syncedAt: new Date() + // After a connection has synced, we need to re-sync the org's search contexts as + // there may be new repos that match the search context's include/exclude patterns. + if (env.CONFIG_PATH) { + try { + const config = await loadConfig(env.CONFIG_PATH); + + await syncSearchContexts({ + db: this.db, + orgId, + contexts: config.contexts, + }); + } catch (err) { + logger.error(`Failed to sync search contexts for connection ${connectionId}: ${err}`); + Sentry.captureException(err); + } } + + logger.info(`Connection sync job ${job.id} for connection ${job.data.connectionName} (id: ${job.data.connectionId}) completed`); + + this.promClient.activeConnectionSyncJobs.dec({ connection: connectionName }); + this.promClient.connectionSyncJobSuccessTotal.inc({ connection: connectionName }); + + const result = job.returnvalue as JobResult; + captureEvent('backend_connection_sync_job_completed', { + connectionId: connectionId, + repoCount: result.repoCount, + }); }); - // After a connection has synced, we need to re-sync the org's search contexts as - // there may be new repos that match the search context's include/exclude patterns. - if (env.CONFIG_PATH) { - try { - const config = await loadConfig(env.CONFIG_PATH); + private onJobFailed = async (job: Job) => + groupmqLifecycleExceptionWrapper('onJobFailed', logger, async () => { + const logger = createJobLogger(job.id); + + const attempt = job.attemptsMade + 1; + const wasLastAttempt = attempt >= job.opts.attempts; - await syncSearchContexts({ - db: this.db, - orgId, - contexts: config.contexts, + if (wasLastAttempt) { + const { connection } = await this.db.connectionSyncJob.update({ + where: { id: job.id }, + data: { + status: ConnectionSyncJobStatus.FAILED, + completedAt: new Date(), + errorMessage: job.failedReason, + }, + select: { + connection: true, + } }); - } catch (err) { - this.logger.error(`Failed to sync search contexts for connection ${connectionId}: ${err}`); - Sentry.captureException(err); - } - } + this.promClient.activeConnectionSyncJobs.dec({ connection: connection.name }); + this.promClient.connectionSyncJobFailTotal.inc({ connection: connection.name }); - captureEvent('backend_connection_sync_job_completed', { - connectionId: connectionId, - repoCount: result.repoCount, - }); - } + logger.error(`Failed job ${job.id} for connection ${connection.name} (id: ${connection.id}). Attempt ${attempt} / ${job.opts.attempts}. Failing job.`); + } else { + const connection = await this.db.connection.findUniqueOrThrow({ + where: { id: job.data.connectionId }, + }); - private async onSyncJobFailed(job: Job | undefined, err: unknown) { - this.logger.info(`Connection sync job for connection ${job?.data.connectionName} (id: ${job?.data.connectionId}, jobId: ${job?.id}) failed with error: ${err}`); - Sentry.captureException(err, { - tags: { - connectionid: job?.data.connectionId, - jobId: job?.id, - queue: QUEUE_NAME, - } - }); + this.promClient.connectionSyncJobReattemptsTotal.inc({ connection: connection.name }); - if (job) { - const { connectionId } = job.data; + logger.warn(`Failed job ${job.id} for connection ${connection.name} (id: ${connection.id}). Attempt ${attempt} / ${job.opts.attempts}. Retrying.`); + } captureEvent('backend_connection_sync_job_failed', { - connectionId: connectionId, - error: err instanceof BackendException ? err.code : 'UNKNOWN', + connectionId: job.data.connectionId, + error: job.failedReason, }); + }); - // We may have pushed some metadata during the execution of the job, so we make sure to not overwrite the metadata here - let syncStatusMetadata: Record = (await this.db.connection.findUnique({ - where: { id: connectionId }, - select: { syncStatusMetadata: true } - }))?.syncStatusMetadata as Record ?? {}; - - if (err instanceof BackendException) { - syncStatusMetadata = { - ...syncStatusMetadata, - error: err.code, - ...err.metadata, - } - } else { - syncStatusMetadata = { - ...syncStatusMetadata, - error: 'UNKNOWN', - } - } - - await this.db.connection.update({ - where: { - id: connectionId, - }, + private onJobStalled = async (jobId: string) => + groupmqLifecycleExceptionWrapper('onJobStalled', logger, async () => { + const logger = createJobLogger(jobId); + const { connection } = await this.db.connectionSyncJob.update({ + where: { id: jobId }, data: { - syncStatus: ConnectionSyncStatus.FAILED, - syncStatusMetadata: syncStatusMetadata as Prisma.InputJsonValue, + status: ConnectionSyncJobStatus.FAILED, + completedAt: new Date(), + errorMessage: 'Job stalled', + }, + select: { + connection: true, } }); - } + + this.promClient.activeConnectionSyncJobs.dec({ connection: connection.name }); + this.promClient.connectionSyncJobFailTotal.inc({ connection: connection.name }); + + logger.error(`Job ${jobId} stalled for connection ${connection.name} (id: ${connection.id})`); + + captureEvent('backend_connection_sync_job_failed', { + connectionId: connection.id, + error: 'Job stalled', + }); + }); + + private async onWorkerError(error: Error) { + Sentry.captureException(error); + logger.error(`Connection syncer worker error.`, error); } public async dispose() { diff --git a/packages/backend/src/connectionUtils.ts b/packages/backend/src/connectionUtils.ts index ca19fa598..074dfe7fe 100644 --- a/packages/backend/src/connectionUtils.ts +++ b/packages/backend/src/connectionUtils.ts @@ -5,21 +5,21 @@ type ValidResult = { data: T[]; }; -type NotFoundResult = { - type: 'notFound'; - value: string; +type WarningResult = { + type: 'warning'; + warning: string; }; -type CustomResult = ValidResult | NotFoundResult; +type CustomResult = ValidResult | WarningResult; export function processPromiseResults( results: PromiseSettledResult>[], ): { validItems: T[]; - notFoundItems: string[]; + warnings: string[]; } { const validItems: T[] = []; - const notFoundItems: string[] = []; + const warnings: string[] = []; results.forEach(result => { if (result.status === 'fulfilled') { @@ -27,14 +27,14 @@ export function processPromiseResults( if (value.type === 'valid') { validItems.push(...value.data); } else { - notFoundItems.push(value.value); + warnings.push(value.warning); } } }); return { validItems, - notFoundItems, + warnings, }; } diff --git a/packages/backend/src/constants.ts b/packages/backend/src/constants.ts index 3150187e2..d6db3bece 100644 --- a/packages/backend/src/constants.ts +++ b/packages/backend/src/constants.ts @@ -1,6 +1,8 @@ import { env } from "./env.js"; import path from "path"; +export const SINGLE_TENANT_ORG_ID = 1; + export const PERMISSION_SYNC_SUPPORTED_CODE_HOST_TYPES = [ 'github', ]; diff --git a/packages/backend/src/ee/githubAppManager.ts b/packages/backend/src/ee/githubAppManager.ts index 1a76d117e..ad4aa2478 100644 --- a/packages/backend/src/ee/githubAppManager.ts +++ b/packages/backend/src/ee/githubAppManager.ts @@ -1,10 +1,10 @@ -import { GithubAppConfig, SourcebotConfig } from "@sourcebot/schemas/v3/index.type"; import { loadConfig } from "@sourcebot/shared"; import { env } from "../env.js"; import { createLogger } from "@sourcebot/logger"; -import { getTokenFromConfig } from "../utils.js"; +import { getTokenFromConfig } from "@sourcebot/crypto"; import { PrismaClient } from "@sourcebot/db"; import { App } from "@octokit/app"; +import { GitHubAppConfig } from "@sourcebot/schemas/v3/index.type"; const logger = createLogger('githubAppManager'); const GITHUB_DEFAULT_DEPLOYMENT_HOSTNAME = 'github.com'; @@ -53,7 +53,7 @@ export class GithubAppManager { return; } - const githubApps = config.apps.filter(app => app.type === 'githubApp') as GithubAppConfig[]; + const githubApps = config.apps.filter(app => app.type === 'githubApp') as GitHubAppConfig[]; logger.info(`Found ${githubApps.length} GitHub apps in config`); for (const app of githubApps) { @@ -62,7 +62,7 @@ export class GithubAppManager { // @todo: we should move SINGLE_TENANT_ORG_ID to shared package or just remove the need to pass this in // when resolving tokens const SINGLE_TENANT_ORG_ID = 1; - const privateKey = await getTokenFromConfig(app.privateKey, SINGLE_TENANT_ORG_ID, this.db!); + const privateKey = await getTokenFromConfig(app.privateKey, SINGLE_TENANT_ORG_ID, this.db); const octokitApp = new App({ appId: Number(app.id), diff --git a/packages/shared/src/ee/syncSearchContexts.ts b/packages/backend/src/ee/syncSearchContexts.ts similarity index 98% rename from packages/shared/src/ee/syncSearchContexts.ts rename to packages/backend/src/ee/syncSearchContexts.ts index 8dd932aeb..e53679aee 100644 --- a/packages/shared/src/ee/syncSearchContexts.ts +++ b/packages/backend/src/ee/syncSearchContexts.ts @@ -1,8 +1,7 @@ import micromatch from "micromatch"; import { createLogger } from "@sourcebot/logger"; import { PrismaClient } from "@sourcebot/db"; -import { getPlan, hasEntitlement } from "../entitlements.js"; -import { SOURCEBOT_SUPPORT_EMAIL } from "../constants.js"; +import { getPlan, hasEntitlement, SOURCEBOT_SUPPORT_EMAIL } from "@sourcebot/shared"; import { SearchContext } from "@sourcebot/schemas/v3/index.type"; const logger = createLogger('sync-search-contexts'); diff --git a/packages/backend/src/env.ts b/packages/backend/src/env.ts index 5e6e1e1d9..841caf982 100644 --- a/packages/backend/src/env.ts +++ b/packages/backend/src/env.ts @@ -47,7 +47,7 @@ export const env = createEnv({ DEBUG_ENABLE_GROUPMQ_LOGGING: booleanSchema.default('false'), DATABASE_URL: z.string().url().default("postgresql://postgres:postgres@localhost:5432/postgres"), - CONFIG_PATH: z.string().optional(), + CONFIG_PATH: z.string(), CONNECTION_MANAGER_UPSERT_TIMEOUT_MS: numberSchema.default(300000), REPO_SYNC_RETRY_BASE_SLEEP_SECONDS: numberSchema.default(60), diff --git a/packages/backend/src/gerrit.ts b/packages/backend/src/gerrit.ts index 25e3cfa7b..ef149648b 100644 --- a/packages/backend/src/gerrit.ts +++ b/packages/backend/src/gerrit.ts @@ -37,7 +37,6 @@ const logger = createLogger('gerrit'); export const getGerritReposFromConfig = async (config: GerritConnectionConfig): Promise => { const url = config.url.endsWith('/') ? config.url : `${config.url}/`; - const hostname = new URL(config.url).hostname; let { durationMs, data: projects } = await measure(async () => { try { diff --git a/packages/backend/src/git.ts b/packages/backend/src/git.ts index 80d34cefb..dbb602e1c 100644 --- a/packages/backend/src/git.ts +++ b/packages/backend/src/git.ts @@ -35,6 +35,11 @@ const createGitClientForPath = (path: string, onProgress?: onProgressFn, signal? * parent directory. */ GIT_CEILING_DIRECTORIES: parentPath, + /** + * Disable git credential prompts. This ensures that git operations will fail + * immediately if credentials are not available, rather than prompting for input. + */ + GIT_TERMINAL_PROMPT: '0', }) .cwd({ path, diff --git a/packages/backend/src/gitea.ts b/packages/backend/src/gitea.ts index aefe1c24d..ab3eee3fd 100644 --- a/packages/backend/src/gitea.ts +++ b/packages/backend/src/gitea.ts @@ -1,6 +1,6 @@ import { Api, giteaApi, HttpResponse, Repository as GiteaRepository } from 'gitea-js'; import { GiteaConnectionConfig } from '@sourcebot/schemas/v3/gitea.type'; -import { getTokenFromConfig, measure } from './utils.js'; +import { measure } from './utils.js'; import fetch from 'cross-fetch'; import { createLogger } from '@sourcebot/logger'; import micromatch from 'micromatch'; @@ -8,6 +8,7 @@ import { PrismaClient } from '@sourcebot/db'; import { processPromiseResults, throwIfAnyFailed } from './connectionUtils.js'; import * as Sentry from "@sentry/node"; import { env } from './env.js'; +import { getTokenFromConfig } from "@sourcebot/crypto"; const logger = createLogger('gitea'); const GITEA_CLOUD_HOSTNAME = "gitea.com"; @@ -18,7 +19,7 @@ export const getGiteaReposFromConfig = async (config: GiteaConnectionConfig, org GITEA_CLOUD_HOSTNAME; const token = config.token ? - await getTokenFromConfig(config.token, orgId, db, logger) : + await getTokenFromConfig(config.token, orgId, db) : hostname === GITEA_CLOUD_HOSTNAME ? env.FALLBACK_GITEA_CLOUD_TOKEN : undefined; @@ -29,32 +30,24 @@ export const getGiteaReposFromConfig = async (config: GiteaConnectionConfig, org }); let allRepos: GiteaRepository[] = []; - let notFound: { - users: string[], - orgs: string[], - repos: string[], - } = { - users: [], - orgs: [], - repos: [], - }; + let allWarnings: string[] = []; if (config.orgs) { - const { validRepos, notFoundOrgs } = await getReposForOrgs(config.orgs, api); - allRepos = allRepos.concat(validRepos); - notFound.orgs = notFoundOrgs; + const { repos, warnings } = await getReposForOrgs(config.orgs, api); + allRepos = allRepos.concat(repos); + allWarnings = allWarnings.concat(warnings); } if (config.repos) { - const { validRepos, notFoundRepos } = await getRepos(config.repos, api); - allRepos = allRepos.concat(validRepos); - notFound.repos = notFoundRepos; + const { repos, warnings } = await getRepos(config.repos, api); + allRepos = allRepos.concat(repos); + allWarnings = allWarnings.concat(warnings); } if (config.users) { - const { validRepos, notFoundUsers } = await getReposOwnedByUsers(config.users, api); - allRepos = allRepos.concat(validRepos); - notFound.users = notFoundUsers; + const { repos, warnings } = await getReposOwnedByUsers(config.users, api); + allRepos = allRepos.concat(repos); + allWarnings = allWarnings.concat(warnings); } allRepos = allRepos.filter(repo => repo.full_name !== undefined); @@ -78,8 +71,8 @@ export const getGiteaReposFromConfig = async (config: GiteaConnectionConfig, org logger.debug(`Found ${repos.length} total repositories.`); return { - validRepos: repos, - notFound, + repos, + warnings: allWarnings, }; } @@ -145,10 +138,11 @@ const getReposOwnedByUsers = async (users: string[], api: Api) => { Sentry.captureException(e); if (e?.status === 404) { - logger.error(`User ${user} not found or no access`); + const warning = `User ${user} not found or no access`; + logger.warn(warning); return { - type: 'notFound' as const, - value: user + type: 'warning' as const, + warning }; } throw e; @@ -156,11 +150,11 @@ const getReposOwnedByUsers = async (users: string[], api: Api) => { })); throwIfAnyFailed(results); - const { validItems: validRepos, notFoundItems: notFoundUsers } = processPromiseResults(results); + const { validItems: repos, warnings } = processPromiseResults(results); return { - validRepos, - notFoundUsers, + repos, + warnings, }; } @@ -185,10 +179,11 @@ const getReposForOrgs = async (orgs: string[], api: Api) => { Sentry.captureException(e); if (e?.status === 404) { - logger.error(`Organization ${org} not found or no access`); + const warning = `Organization ${org} not found or no access`; + logger.warn(warning); return { - type: 'notFound' as const, - value: org + type: 'warning' as const, + warning }; } throw e; @@ -196,16 +191,16 @@ const getReposForOrgs = async (orgs: string[], api: Api) => { })); throwIfAnyFailed(results); - const { validItems: validRepos, notFoundItems: notFoundOrgs } = processPromiseResults(results); + const { validItems: repos, warnings } = processPromiseResults(results); return { - validRepos, - notFoundOrgs, + repos, + warnings, }; } -const getRepos = async (repos: string[], api: Api) => { - const results = await Promise.allSettled(repos.map(async (repo) => { +const getRepos = async (repoList: string[], api: Api) => { + const results = await Promise.allSettled(repoList.map(async (repo) => { try { logger.debug(`Fetching repository info for ${repo}...`); @@ -223,10 +218,11 @@ const getRepos = async (repos: string[], api: Api) => { Sentry.captureException(e); if (e?.status === 404) { - logger.error(`Repository ${repo} not found or no access`); + const warning = `Repository ${repo} not found or no access`; + logger.warn(warning); return { - type: 'notFound' as const, - value: repo + type: 'warning' as const, + warning }; } throw e; @@ -234,11 +230,11 @@ const getRepos = async (repos: string[], api: Api) => { })); throwIfAnyFailed(results); - const { validItems: validRepos, notFoundItems: notFoundRepos } = processPromiseResults(results); + const { validItems: repos, warnings } = processPromiseResults(results); return { - validRepos, - notFoundRepos, + repos, + warnings, }; } diff --git a/packages/backend/src/github.ts b/packages/backend/src/github.ts index ee26a0245..550d259d5 100644 --- a/packages/backend/src/github.ts +++ b/packages/backend/src/github.ts @@ -1,15 +1,15 @@ import { Octokit } from "@octokit/rest"; -import { GithubConnectionConfig } from "@sourcebot/schemas/v3/github.type"; +import * as Sentry from "@sentry/node"; +import { PrismaClient } from "@sourcebot/db"; import { createLogger } from "@sourcebot/logger"; -import { getTokenFromConfig, measure, fetchWithRetry } from "./utils.js"; +import { GithubConnectionConfig } from "@sourcebot/schemas/v3/github.type"; +import { hasEntitlement } from "@sourcebot/shared"; import micromatch from "micromatch"; -import { PrismaClient } from "@sourcebot/db"; -import { BackendException, BackendError } from "@sourcebot/error"; import { processPromiseResults, throwIfAnyFailed } from "./connectionUtils.js"; -import * as Sentry from "@sentry/node"; -import { env } from "./env.js"; import { GithubAppManager } from "./ee/githubAppManager.js"; -import { hasEntitlement } from "@sourcebot/shared"; +import { env } from "./env.js"; +import { fetchWithRetry, measure } from "./utils.js"; +import { getTokenFromConfig } from "@sourcebot/crypto"; export const GITHUB_CLOUD_HOSTNAME = "github.com"; const logger = createLogger('github'); @@ -92,13 +92,13 @@ const getOctokitWithGithubApp = async ( } } -export const getGitHubReposFromConfig = async (config: GithubConnectionConfig, orgId: number, db: PrismaClient, signal: AbortSignal) => { +export const getGitHubReposFromConfig = async (config: GithubConnectionConfig, orgId: number, db: PrismaClient, signal: AbortSignal): Promise<{ repos: OctokitRepository[], warnings: string[] }> => { const hostname = config.url ? new URL(config.url).hostname : GITHUB_CLOUD_HOSTNAME; const token = config.token ? - await getTokenFromConfig(config.token, orgId, db, logger) : + await getTokenFromConfig(config.token, orgId, db) : hostname === GITHUB_CLOUD_HOSTNAME ? env.FALLBACK_GITHUB_CLOUD_TOKEN : undefined; @@ -108,57 +108,36 @@ export const getGitHubReposFromConfig = async (config: GithubConnectionConfig, o url: config.url, }); + if (isAuthenticated) { try { await octokit.rest.users.getAuthenticated(); } catch (error) { Sentry.captureException(error); - - if (isHttpError(error, 401)) { - const e = new BackendException(BackendError.CONNECTION_SYNC_INVALID_TOKEN, { - ...(config.token && 'secret' in config.token ? { - secretKey: config.token.secret, - } : {}), - }); - Sentry.captureException(e); - throw e; - } - - const e = new BackendException(BackendError.CONNECTION_SYNC_SYSTEM_ERROR, { - message: `Failed to authenticate with GitHub`, - }); - Sentry.captureException(e); - throw e; + logger.error(`Failed to authenticate with GitHub`, error); + throw error; } } let allRepos: OctokitRepository[] = []; - let notFound: { - users: string[], - orgs: string[], - repos: string[], - } = { - users: [], - orgs: [], - repos: [], - }; + let allWarnings: string[] = []; if (config.orgs) { - const { validRepos, notFoundOrgs } = await getReposForOrgs(config.orgs, octokit, signal, config.url); - allRepos = allRepos.concat(validRepos); - notFound.orgs = notFoundOrgs; + const { repos, warnings } = await getReposForOrgs(config.orgs, octokit, signal, config.url); + allRepos = allRepos.concat(repos); + allWarnings = allWarnings.concat(warnings); } if (config.repos) { - const { validRepos, notFoundRepos } = await getRepos(config.repos, octokit, signal, config.url); - allRepos = allRepos.concat(validRepos); - notFound.repos = notFoundRepos; + const { repos, warnings } = await getRepos(config.repos, octokit, signal, config.url); + allRepos = allRepos.concat(repos); + allWarnings = allWarnings.concat(warnings); } if (config.users) { - const { validRepos, notFoundUsers } = await getReposOwnedByUsers(config.users, octokit, signal, config.url); - allRepos = allRepos.concat(validRepos); - notFound.users = notFoundUsers; + const { repos, warnings } = await getReposOwnedByUsers(config.users, octokit, signal, config.url); + allRepos = allRepos.concat(repos); + allWarnings = allWarnings.concat(warnings); } let repos = allRepos @@ -177,8 +156,8 @@ export const getGitHubReposFromConfig = async (config: GithubConnectionConfig, o logger.debug(`Found ${repos.length} total repositories.`); return { - validRepos: repos, - notFound, + repos, + warnings: allWarnings, }; } @@ -256,10 +235,11 @@ const getReposOwnedByUsers = async (users: string[], octokit: Octokit, signal: A logger.error(`Failed to fetch repositories for user ${user}.`, error); if (isHttpError(error, 404)) { - logger.error(`User ${user} not found or no access`); + const warning = `User ${user} not found or no access`; + logger.warn(warning); return { - type: 'notFound' as const, - value: user + type: 'warning' as const, + warning }; } throw error; @@ -267,18 +247,18 @@ const getReposOwnedByUsers = async (users: string[], octokit: Octokit, signal: A })); throwIfAnyFailed(results); - const { validItems: validRepos, notFoundItems: notFoundUsers } = processPromiseResults(results); + const { validItems: repos, warnings } = processPromiseResults(results); return { - validRepos, - notFoundUsers, + repos, + warnings, }; } const getReposForOrgs = async (orgs: string[], octokit: Octokit, signal: AbortSignal, url?: string) => { const results = await Promise.allSettled(orgs.map(async (org) => { try { - logger.info(`Fetching repository info for org ${org}...`); + logger.debug(`Fetching repository info for org ${org}...`); const octokitToUse = await getOctokitWithGithubApp(octokit, org, url, `org ${org}`); const { durationMs, data } = await measure(async () => { @@ -293,7 +273,7 @@ const getReposForOrgs = async (orgs: string[], octokit: Octokit, signal: AbortSi return fetchWithRetry(fetchFn, `org ${org}`, logger); }); - logger.info(`Found ${data.length} in org ${org} in ${durationMs}ms.`); + logger.debug(`Found ${data.length} in org ${org} in ${durationMs}ms.`); return { type: 'valid' as const, data @@ -303,10 +283,11 @@ const getReposForOrgs = async (orgs: string[], octokit: Octokit, signal: AbortSi logger.error(`Failed to fetch repositories for org ${org}.`, error); if (isHttpError(error, 404)) { - logger.error(`Organization ${org} not found or no access`); + const warning = `Organization ${org} not found or no access`; + logger.warn(warning); return { - type: 'notFound' as const, - value: org + type: 'warning' as const, + warning }; } throw error; @@ -314,11 +295,11 @@ const getReposForOrgs = async (orgs: string[], octokit: Octokit, signal: AbortSi })); throwIfAnyFailed(results); - const { validItems: validRepos, notFoundItems: notFoundOrgs } = processPromiseResults(results); + const { validItems: repos, warnings } = processPromiseResults(results); return { - validRepos, - notFoundOrgs, + repos, + warnings, }; } @@ -326,7 +307,7 @@ const getRepos = async (repoList: string[], octokit: Octokit, signal: AbortSigna const results = await Promise.allSettled(repoList.map(async (repo) => { try { const [owner, repoName] = repo.split('/'); - logger.info(`Fetching repository info for ${repo}...`); + logger.debug(`Fetching repository info for ${repo}...`); const octokitToUse = await getOctokitWithGithubApp(octokit, owner, url, `repo ${repo}`); const { durationMs, data: result } = await measure(async () => { @@ -341,7 +322,7 @@ const getRepos = async (repoList: string[], octokit: Octokit, signal: AbortSigna return fetchWithRetry(fetchFn, repo, logger); }); - logger.info(`Found info for repository ${repo} in ${durationMs}ms`); + logger.debug(`Found info for repository ${repo} in ${durationMs}ms`); return { type: 'valid' as const, data: [result.data] @@ -352,10 +333,11 @@ const getRepos = async (repoList: string[], octokit: Octokit, signal: AbortSigna logger.error(`Failed to fetch repository ${repo}.`, error); if (isHttpError(error, 404)) { - logger.error(`Repository ${repo} not found or no access`); + const warning = `Repository ${repo} not found or no access`; + logger.warn(warning); return { - type: 'notFound' as const, - value: repo + type: 'warning' as const, + warning }; } throw error; @@ -363,11 +345,11 @@ const getRepos = async (repoList: string[], octokit: Octokit, signal: AbortSigna })); throwIfAnyFailed(results); - const { validItems: validRepos, notFoundItems: notFoundRepos } = processPromiseResults(results); + const { validItems: repos, warnings } = processPromiseResults(results); return { - validRepos, - notFoundRepos, + repos, + warnings, }; } diff --git a/packages/backend/src/gitlab.ts b/packages/backend/src/gitlab.ts index d13692d3d..e4954b349 100644 --- a/packages/backend/src/gitlab.ts +++ b/packages/backend/src/gitlab.ts @@ -2,11 +2,12 @@ import { Gitlab, ProjectSchema } from "@gitbeaker/rest"; import micromatch from "micromatch"; import { createLogger } from "@sourcebot/logger"; import { GitlabConnectionConfig } from "@sourcebot/schemas/v3/gitlab.type" -import { getTokenFromConfig, measure, fetchWithRetry } from "./utils.js"; +import { measure, fetchWithRetry } from "./utils.js"; import { PrismaClient } from "@sourcebot/db"; import { processPromiseResults, throwIfAnyFailed } from "./connectionUtils.js"; import * as Sentry from "@sentry/node"; import { env } from "./env.js"; +import { getTokenFromConfig } from "@sourcebot/crypto"; const logger = createLogger('gitlab'); export const GITLAB_CLOUD_HOSTNAME = "gitlab.com"; @@ -17,7 +18,7 @@ export const getGitLabReposFromConfig = async (config: GitlabConnectionConfig, o GITLAB_CLOUD_HOSTNAME; const token = config.token ? - await getTokenFromConfig(config.token, orgId, db, logger) : + await getTokenFromConfig(config.token, orgId, db) : hostname === GITLAB_CLOUD_HOSTNAME ? env.FALLBACK_GITLAB_CLOUD_TOKEN : undefined; @@ -33,15 +34,7 @@ export const getGitLabReposFromConfig = async (config: GitlabConnectionConfig, o }); let allRepos: ProjectSchema[] = []; - let notFound: { - orgs: string[], - users: string[], - repos: string[], - } = { - orgs: [], - users: [], - repos: [], - }; + let allWarnings: string[] = []; if (config.all === true) { if (hostname !== GITLAB_CLOUD_HOSTNAME) { @@ -61,7 +54,9 @@ export const getGitLabReposFromConfig = async (config: GitlabConnectionConfig, o throw e; } } else { - logger.warn(`Ignoring option all:true in config : host is ${GITLAB_CLOUD_HOSTNAME}`); + const warning = `Ignoring option all:true in config : host is ${GITLAB_CLOUD_HOSTNAME}`; + logger.warn(warning); + allWarnings = allWarnings.concat(warning); } } @@ -87,10 +82,11 @@ export const getGitLabReposFromConfig = async (config: GitlabConnectionConfig, o const status = e?.cause?.response?.status; if (status === 404) { - logger.error(`Group ${group} not found or no access`); + const warning = `Group ${group} not found or no access`; + logger.warn(warning); return { - type: 'notFound' as const, - value: group + type: 'warning' as const, + warning }; } throw e; @@ -98,9 +94,9 @@ export const getGitLabReposFromConfig = async (config: GitlabConnectionConfig, o })); throwIfAnyFailed(results); - const { validItems: validRepos, notFoundItems: notFoundOrgs } = processPromiseResults(results); + const { validItems: validRepos, warnings } = processPromiseResults(results); allRepos = allRepos.concat(validRepos); - notFound.orgs = notFoundOrgs; + allWarnings = allWarnings.concat(warnings); } if (config.users) { @@ -124,10 +120,11 @@ export const getGitLabReposFromConfig = async (config: GitlabConnectionConfig, o const status = e?.cause?.response?.status; if (status === 404) { - logger.error(`User ${user} not found or no access`); + const warning = `User ${user} not found or no access`; + logger.warn(warning); return { - type: 'notFound' as const, - value: user + type: 'warning' as const, + warning }; } throw e; @@ -135,9 +132,9 @@ export const getGitLabReposFromConfig = async (config: GitlabConnectionConfig, o })); throwIfAnyFailed(results); - const { validItems: validRepos, notFoundItems: notFoundUsers } = processPromiseResults(results); + const { validItems: validRepos, warnings } = processPromiseResults(results); allRepos = allRepos.concat(validRepos); - notFound.users = notFoundUsers; + allWarnings = allWarnings.concat(warnings); } if (config.projects) { @@ -160,10 +157,11 @@ export const getGitLabReposFromConfig = async (config: GitlabConnectionConfig, o const status = e?.cause?.response?.status; if (status === 404) { - logger.error(`Project ${project} not found or no access`); + const warning = `Project ${project} not found or no access`; + logger.warn(warning); return { - type: 'notFound' as const, - value: project + type: 'warning' as const, + warning }; } throw e; @@ -171,9 +169,9 @@ export const getGitLabReposFromConfig = async (config: GitlabConnectionConfig, o })); throwIfAnyFailed(results); - const { validItems: validRepos, notFoundItems: notFoundRepos } = processPromiseResults(results); + const { validItems: validRepos, warnings } = processPromiseResults(results); allRepos = allRepos.concat(validRepos); - notFound.repos = notFoundRepos; + allWarnings = allWarnings.concat(warnings); } let repos = allRepos @@ -192,8 +190,8 @@ export const getGitLabReposFromConfig = async (config: GitlabConnectionConfig, o logger.debug(`Found ${repos.length} total repositories.`); return { - validRepos: repos, - notFound, + repos, + warnings: allWarnings, }; } diff --git a/packages/backend/src/index.ts b/packages/backend/src/index.ts index 2e2ec569c..fd7847415 100644 --- a/packages/backend/src/index.ts +++ b/packages/backend/src/index.ts @@ -6,14 +6,15 @@ import { getConfigSettings, hasEntitlement } from '@sourcebot/shared'; import { existsSync } from 'fs'; import { mkdir } from 'fs/promises'; import { Redis } from 'ioredis'; +import { ConfigManager } from "./configManager.js"; import { ConnectionManager } from './connectionManager.js'; import { INDEX_CACHE_DIR, REPOS_CACHE_DIR } from './constants.js'; +import { GithubAppManager } from "./ee/githubAppManager.js"; import { RepoPermissionSyncer } from './ee/repoPermissionSyncer.js'; import { UserPermissionSyncer } from "./ee/userPermissionSyncer.js"; -import { GithubAppManager } from "./ee/githubAppManager.js"; import { env } from "./env.js"; -import { RepoIndexManager } from "./repoIndexManager.js"; import { PromClient } from './promClient.js'; +import { RepoIndexManager } from "./repoIndexManager.js"; const logger = createLogger('backend-entrypoint'); @@ -49,10 +50,11 @@ if (hasEntitlement('github-app')) { await GithubAppManager.getInstance().init(prisma); } -const connectionManager = new ConnectionManager(prisma, settings, redis); +const connectionManager = new ConnectionManager(prisma, settings, redis, promClient); const repoPermissionSyncer = new RepoPermissionSyncer(prisma, settings, redis); const userPermissionSyncer = new UserPermissionSyncer(prisma, settings, redis); const repoIndexManager = new RepoIndexManager(prisma, settings, redis, promClient); +const configManager = new ConfigManager(prisma, connectionManager, env.CONFIG_PATH); connectionManager.startScheduler(); repoIndexManager.startScheduler(); @@ -66,11 +68,13 @@ else if (env.EXPERIMENT_EE_PERMISSION_SYNC_ENABLED === 'true' && hasEntitlement( userPermissionSyncer.startScheduler(); } +logger.info('Worker started.'); + const cleanup = async (signal: string) => { logger.info(`Received ${signal}, cleaning up...`); const shutdownTimeout = 30000; // 30 seconds - + try { await Promise.race([ Promise.all([ @@ -79,8 +83,9 @@ const cleanup = async (signal: string) => { repoPermissionSyncer.dispose(), userPermissionSyncer.dispose(), promClient.dispose(), + configManager.dispose(), ]), - new Promise((_, reject) => + new Promise((_, reject) => setTimeout(() => reject(new Error('Shutdown timeout')), shutdownTimeout) ) ]); diff --git a/packages/backend/src/promClient.ts b/packages/backend/src/promClient.ts index c4eb7f06b..67ed6f3fb 100644 --- a/packages/backend/src/promClient.ts +++ b/packages/backend/src/promClient.ts @@ -16,6 +16,12 @@ export class PromClient { public repoIndexJobFailTotal: Counter; public repoIndexJobSuccessTotal: Counter; + public activeConnectionSyncJobs: Gauge; + public pendingConnectionSyncJobs: Gauge; + public connectionSyncJobReattemptsTotal: Counter; + public connectionSyncJobFailTotal: Counter; + public connectionSyncJobSuccessTotal: Counter; + public readonly PORT = 3060; constructor() { @@ -56,6 +62,41 @@ export class PromClient { }); this.registry.registerMetric(this.repoIndexJobSuccessTotal); + this.activeConnectionSyncJobs = new Gauge({ + name: 'active_connection_sync_jobs', + help: 'The number of connection sync jobs in progress', + labelNames: ['connection'], + }); + this.registry.registerMetric(this.activeConnectionSyncJobs); + + this.pendingConnectionSyncJobs = new Gauge({ + name: 'pending_connection_sync_jobs', + help: 'The number of connection sync jobs waiting in queue', + labelNames: ['connection'], + }); + this.registry.registerMetric(this.pendingConnectionSyncJobs); + + this.connectionSyncJobReattemptsTotal = new Counter({ + name: 'connection_sync_job_reattempts', + help: 'The number of connection sync job reattempts', + labelNames: ['connection'], + }); + this.registry.registerMetric(this.connectionSyncJobReattemptsTotal); + + this.connectionSyncJobFailTotal = new Counter({ + name: 'connection_sync_job_fails', + help: 'The number of connection sync job fails', + labelNames: ['connection'], + }); + this.registry.registerMetric(this.connectionSyncJobFailTotal); + + this.connectionSyncJobSuccessTotal = new Counter({ + name: 'connection_sync_job_successes', + help: 'The number of connection sync job successes', + labelNames: ['connection'], + }); + this.registry.registerMetric(this.connectionSyncJobSuccessTotal); + client.collectDefaultMetrics({ register: this.registry, }); diff --git a/packages/backend/src/repoCompileUtils.ts b/packages/backend/src/repoCompileUtils.ts index f5edb4f6c..d78455e00 100644 --- a/packages/backend/src/repoCompileUtils.ts +++ b/packages/backend/src/repoCompileUtils.ts @@ -24,22 +24,20 @@ export type RepoData = WithRequired; const logger = createLogger('repo-compile-utils'); +type CompileResult = { + repoData: RepoData[], + warnings: string[], +} + export const compileGithubConfig = async ( config: GithubConnectionConfig, connectionId: number, orgId: number, db: PrismaClient, - abortController: AbortController): Promise<{ - repoData: RepoData[], - notFound: { - users: string[], - orgs: string[], - repos: string[], - } - }> => { + abortController: AbortController): Promise => { const gitHubReposResult = await getGitHubReposFromConfig(config, orgId, db, abortController.signal); - const gitHubRepos = gitHubReposResult.validRepos; - const notFound = gitHubReposResult.notFound; + const gitHubRepos = gitHubReposResult.repos; + const warnings = gitHubReposResult.warnings; const hostUrl = config.url ?? 'https://github.com'; const repoNameRoot = new URL(hostUrl) @@ -100,7 +98,7 @@ export const compileGithubConfig = async ( return { repoData: repos, - notFound, + warnings, }; } @@ -108,11 +106,11 @@ export const compileGitlabConfig = async ( config: GitlabConnectionConfig, connectionId: number, orgId: number, - db: PrismaClient) => { + db: PrismaClient): Promise => { const gitlabReposResult = await getGitLabReposFromConfig(config, orgId, db); - const gitlabRepos = gitlabReposResult.validRepos; - const notFound = gitlabReposResult.notFound; + const gitlabRepos = gitlabReposResult.repos; + const warnings = gitlabReposResult.warnings; const hostUrl = config.url ?? 'https://gitlab.com'; const repoNameRoot = new URL(hostUrl) @@ -177,7 +175,7 @@ export const compileGitlabConfig = async ( return { repoData: repos, - notFound, + warnings, }; } @@ -185,11 +183,11 @@ export const compileGiteaConfig = async ( config: GiteaConnectionConfig, connectionId: number, orgId: number, - db: PrismaClient) => { + db: PrismaClient): Promise => { const giteaReposResult = await getGiteaReposFromConfig(config, orgId, db); - const giteaRepos = giteaReposResult.validRepos; - const notFound = giteaReposResult.notFound; + const giteaRepos = giteaReposResult.repos; + const warnings = giteaReposResult.warnings; const hostUrl = config.url ?? 'https://gitea.com'; const repoNameRoot = new URL(hostUrl) @@ -248,14 +246,14 @@ export const compileGiteaConfig = async ( return { repoData: repos, - notFound, + warnings, }; } export const compileGerritConfig = async ( config: GerritConnectionConfig, connectionId: number, - orgId: number) => { + orgId: number): Promise => { const gerritRepos = await getGerritReposFromConfig(config); const hostUrl = config.url; @@ -329,11 +327,7 @@ export const compileGerritConfig = async ( return { repoData: repos, - notFound: { - users: [], - orgs: [], - repos: [], - } + warnings: [], }; } @@ -341,11 +335,11 @@ export const compileBitbucketConfig = async ( config: BitbucketConnectionConfig, connectionId: number, orgId: number, - db: PrismaClient) => { + db: PrismaClient): Promise => { const bitbucketReposResult = await getBitbucketReposFromConfig(config, orgId, db); - const bitbucketRepos = bitbucketReposResult.validRepos; - const notFound = bitbucketReposResult.notFound; + const bitbucketRepos = bitbucketReposResult.repos; + const warnings = bitbucketReposResult.warnings; const hostUrl = config.url ?? 'https://bitbucket.org'; const repoNameRoot = new URL(hostUrl) @@ -450,7 +444,7 @@ export const compileBitbucketConfig = async ( return { repoData: repos, - notFound, + warnings, }; } @@ -458,7 +452,7 @@ export const compileGenericGitHostConfig = async ( config: GenericGitHostConnectionConfig, connectionId: number, orgId: number, -) => { +): Promise => { const configUrl = new URL(config.url); if (configUrl.protocol === 'file:') { return compileGenericGitHostConfig_file(config, orgId, connectionId); @@ -476,7 +470,7 @@ export const compileGenericGitHostConfig_file = async ( config: GenericGitHostConnectionConfig, orgId: number, connectionId: number, -) => { +): Promise => { const configUrl = new URL(config.url); assert(configUrl.protocol === 'file:', 'config.url must be a file:// URL'); @@ -486,30 +480,24 @@ export const compileGenericGitHostConfig_file = async ( }); const repos: RepoData[] = []; - const notFound: { - users: string[], - orgs: string[], - repos: string[], - } = { - users: [], - orgs: [], - repos: [], - }; + const warnings: string[] = []; await Promise.all(repoPaths.map(async (repoPath) => { const isGitRepo = await isPathAValidGitRepoRoot({ path: repoPath, }); if (!isGitRepo) { - logger.warn(`Skipping ${repoPath} - not a git repository.`); - notFound.repos.push(repoPath); + const warning = `Skipping ${repoPath} - not a git repository.`; + logger.warn(warning); + warnings.push(warning); return; } const origin = await getOriginUrl(repoPath); if (!origin) { - logger.warn(`Skipping ${repoPath} - remote.origin.url not found in git config.`); - notFound.repos.push(repoPath); + const warning = `Skipping ${repoPath} - remote.origin.url not found in git config.`; + logger.warn(warning); + warnings.push(warning); return; } @@ -552,7 +540,7 @@ export const compileGenericGitHostConfig_file = async ( return { repoData: repos, - notFound, + warnings, } } @@ -561,27 +549,21 @@ export const compileGenericGitHostConfig_url = async ( config: GenericGitHostConnectionConfig, orgId: number, connectionId: number, -) => { +): Promise => { const remoteUrl = new URL(config.url); assert(remoteUrl.protocol === 'http:' || remoteUrl.protocol === 'https:', 'config.url must be a http:// or https:// URL'); - const notFound: { - users: string[], - orgs: string[], - repos: string[], - } = { - users: [], - orgs: [], - repos: [], - }; + const warnings: string[] = []; // Validate that we are dealing with a valid git repo. const isGitRepo = await isUrlAValidGitRepo(remoteUrl.toString()); if (!isGitRepo) { - notFound.repos.push(remoteUrl.toString()); + const warning = `Skipping ${remoteUrl.toString()} - not a git repository.`; + logger.warn(warning); + warnings.push(warning); return { repoData: [], - notFound, + warnings, } } @@ -616,7 +598,7 @@ export const compileGenericGitHostConfig_url = async ( return { repoData: [repo], - notFound, + warnings, } } @@ -624,12 +606,11 @@ export const compileAzureDevOpsConfig = async ( config: AzureDevOpsConnectionConfig, connectionId: number, orgId: number, - db: PrismaClient, - abortController: AbortController) => { + db: PrismaClient): Promise => { const azureDevOpsReposResult = await getAzureDevOpsReposFromConfig(config, orgId, db); - const azureDevOpsRepos = azureDevOpsReposResult.validRepos; - const notFound = azureDevOpsReposResult.notFound; + const azureDevOpsRepos = azureDevOpsReposResult.repos; + const warnings = azureDevOpsReposResult.warnings; const hostUrl = config.url ?? 'https://dev.azure.com'; const repoNameRoot = new URL(hostUrl) @@ -699,6 +680,6 @@ export const compileAzureDevOpsConfig = async ( return { repoData: repos, - notFound, + warnings, }; } diff --git a/packages/backend/src/repoIndexManager.ts b/packages/backend/src/repoIndexManager.ts index 98258b0b8..042ba8c7e 100644 --- a/packages/backend/src/repoIndexManager.ts +++ b/packages/backend/src/repoIndexManager.ts @@ -149,7 +149,8 @@ export class RepoIndexManager { } private async scheduleCleanupJobs() { - const thresholdDate = new Date(Date.now() - this.settings.repoGarbageCollectionGracePeriodMs); + const gcGracePeriodMs = new Date(Date.now() - this.settings.repoGarbageCollectionGracePeriodMs); + const timeoutDate = new Date(Date.now() - this.settings.repoIndexTimeoutMs); const reposToCleanup = await this.db.repo.findMany({ where: { @@ -158,9 +159,8 @@ export class RepoIndexManager { }, OR: [ { indexedAt: null }, - { indexedAt: { lt: thresholdDate } }, + { indexedAt: { lt: gcGracePeriodMs } }, ], - // Don't schedule if there are active jobs that were created within the threshold date. NOT: { jobs: { some: { @@ -178,7 +178,7 @@ export class RepoIndexManager { }, { createdAt: { - gt: thresholdDate, + gt: timeoutDate, } } ] diff --git a/packages/backend/src/utils.ts b/packages/backend/src/utils.ts index 8a1d0cc6c..4bb18549f 100644 --- a/packages/backend/src/utils.ts +++ b/packages/backend/src/utils.ts @@ -2,8 +2,7 @@ import { Logger } from "winston"; import { RepoAuthCredentials, RepoWithConnections } from "./types.js"; import path from 'path'; import { PrismaClient, Repo } from "@sourcebot/db"; -import { getTokenFromConfig as getTokenFromConfigBase } from "@sourcebot/crypto"; -import { BackendException, BackendError } from "@sourcebot/error"; +import { getTokenFromConfig } from "@sourcebot/crypto"; import * as Sentry from "@sentry/node"; import { GithubConnectionConfig, GitlabConnectionConfig, GiteaConnectionConfig, BitbucketConnectionConfig, AzureDevOpsConnectionConfig } from '@sourcebot/schemas/v3/connection.type'; import { GithubAppManager } from "./ee/githubAppManager.js"; @@ -24,22 +23,6 @@ export const marshalBool = (value?: boolean) => { return !!value ? '1' : '0'; } -export const getTokenFromConfig = async (token: any, orgId: number, db: PrismaClient, logger?: Logger) => { - try { - return await getTokenFromConfigBase(token, orgId, db); - } catch (error: unknown) { - if (error instanceof Error) { - const e = new BackendException(BackendError.CONNECTION_SYNC_SECRET_DNE, { - message: error.message, - }); - Sentry.captureException(e); - logger?.error(error.message); - throw e; - } - throw error; - } -}; - export const resolvePathRelativeToConfig = (localPath: string, configPath: string) => { let absolutePath = localPath; if (!path.isAbsolute(absolutePath)) { @@ -156,7 +139,7 @@ export const getAuthCredentialsForRepo = async (repo: RepoWithConnections, db: P if (connection.connectionType === 'github') { const config = connection.config as unknown as GithubConnectionConfig; if (config.token) { - const token = await getTokenFromConfig(config.token, connection.orgId, db, logger); + const token = await getTokenFromConfig(config.token, connection.orgId, db); return { hostUrl: config.url, token, @@ -171,7 +154,7 @@ export const getAuthCredentialsForRepo = async (repo: RepoWithConnections, db: P } else if (connection.connectionType === 'gitlab') { const config = connection.config as unknown as GitlabConnectionConfig; if (config.token) { - const token = await getTokenFromConfig(config.token, connection.orgId, db, logger); + const token = await getTokenFromConfig(config.token, connection.orgId, db); return { hostUrl: config.url, token, @@ -187,7 +170,7 @@ export const getAuthCredentialsForRepo = async (repo: RepoWithConnections, db: P } else if (connection.connectionType === 'gitea') { const config = connection.config as unknown as GiteaConnectionConfig; if (config.token) { - const token = await getTokenFromConfig(config.token, connection.orgId, db, logger); + const token = await getTokenFromConfig(config.token, connection.orgId, db); return { hostUrl: config.url, token, @@ -202,7 +185,7 @@ export const getAuthCredentialsForRepo = async (repo: RepoWithConnections, db: P } else if (connection.connectionType === 'bitbucket') { const config = connection.config as unknown as BitbucketConnectionConfig; if (config.token) { - const token = await getTokenFromConfig(config.token, connection.orgId, db, logger); + const token = await getTokenFromConfig(config.token, connection.orgId, db); const username = config.user ?? 'x-token-auth'; return { hostUrl: config.url, @@ -219,7 +202,7 @@ export const getAuthCredentialsForRepo = async (repo: RepoWithConnections, db: P } else if (connection.connectionType === 'azuredevops') { const config = connection.config as unknown as AzureDevOpsConnectionConfig; if (config.token) { - const token = await getTokenFromConfig(config.token, connection.orgId, db, logger); + const token = await getTokenFromConfig(config.token, connection.orgId, db); // For ADO server, multiple auth schemes may be supported. If the ADO deployment supports NTLM, the git clone will default // to this over basic auth. As a result, we cannot embed the token in the clone URL and must force basic auth by passing in the token diff --git a/packages/db/prisma/migrations/20251026194617_add_connection_job_table/migration.sql b/packages/db/prisma/migrations/20251026194617_add_connection_job_table/migration.sql new file mode 100644 index 000000000..60ba68019 --- /dev/null +++ b/packages/db/prisma/migrations/20251026194617_add_connection_job_table/migration.sql @@ -0,0 +1,30 @@ +/* + Warnings: + + - You are about to drop the column `syncStatus` on the `Connection` table. All the data in the column will be lost. + - You are about to drop the column `syncStatusMetadata` on the `Connection` table. All the data in the column will be lost. + +*/ +-- CreateEnum +CREATE TYPE "ConnectionSyncJobStatus" AS ENUM ('PENDING', 'IN_PROGRESS', 'COMPLETED', 'FAILED'); + +-- AlterTable +ALTER TABLE "Connection" DROP COLUMN "syncStatus", +DROP COLUMN "syncStatusMetadata"; + +-- CreateTable +CREATE TABLE "ConnectionSyncJob" ( + "id" TEXT NOT NULL, + "status" "ConnectionSyncJobStatus" NOT NULL DEFAULT 'PENDING', + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" TIMESTAMP(3) NOT NULL, + "completedAt" TIMESTAMP(3), + "warningMessages" TEXT[], + "errorMessage" TEXT, + "connectionId" INTEGER NOT NULL, + + CONSTRAINT "ConnectionSyncJob_pkey" PRIMARY KEY ("id") +); + +-- AddForeignKey +ALTER TABLE "ConnectionSyncJob" ADD CONSTRAINT "ConnectionSyncJob_connectionId_fkey" FOREIGN KEY ("connectionId") REFERENCES "Connection"("id") ON DELETE CASCADE ON UPDATE CASCADE; diff --git a/packages/db/prisma/migrations/20251026194628_ensure_single_tenant_org/migration.sql b/packages/db/prisma/migrations/20251026194628_ensure_single_tenant_org/migration.sql new file mode 100644 index 000000000..8d2fd114f --- /dev/null +++ b/packages/db/prisma/migrations/20251026194628_ensure_single_tenant_org/migration.sql @@ -0,0 +1,13 @@ +-- Installs the pgcrypto extension. Required for the gen_random_uuid() function. +-- @see: https://www.prisma.io/docs/orm/prisma-migrate/workflows/native-database-functions#how-to-install-a-postgresql-extension-as-part-of-a-migration +CREATE EXTENSION IF NOT EXISTS pgcrypto; + +-- Ensure single tenant organization exists +INSERT INTO "Org" (id, name, domain, "inviteLinkId", "createdAt", "updatedAt") +VALUES (1, 'default', '~', gen_random_uuid(), NOW(), NOW()) +ON CONFLICT (id) DO NOTHING; + +-- Backfill inviteLinkId for any existing orgs that don't have one +UPDATE "Org" +SET "inviteLinkId" = gen_random_uuid() +WHERE "inviteLinkId" IS NULL; \ No newline at end of file diff --git a/packages/db/prisma/schema.prisma b/packages/db/prisma/schema.prisma index f04d293c6..93adb7171 100644 --- a/packages/db/prisma/schema.prisma +++ b/packages/db/prisma/schema.prisma @@ -132,15 +132,15 @@ model Connection { isDeclarative Boolean @default(false) createdAt DateTime @default(now()) updatedAt DateTime @updatedAt - /// When the connection was last synced successfully. - syncedAt DateTime? repos RepoToConnection[] - syncStatus ConnectionSyncStatus @default(SYNC_NEEDED) - syncStatusMetadata Json? // The type of connection (e.g., github, gitlab, etc.) connectionType String + syncJobs ConnectionSyncJob[] + /// When the connection was last synced successfully. + syncedAt DateTime? + // The organization that owns this connection org Org @relation(fields: [orgId], references: [id], onDelete: Cascade) orgId Int @@ -148,6 +148,27 @@ model Connection { @@unique([name, orgId]) } +enum ConnectionSyncJobStatus { + PENDING + IN_PROGRESS + COMPLETED + FAILED +} + +model ConnectionSyncJob { + id String @id @default(cuid()) + status ConnectionSyncJobStatus @default(PENDING) + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + completedAt DateTime? + + warningMessages String[] + errorMessage String? + + connection Connection @relation(fields: [connectionId], references: [id], onDelete: Cascade) + connectionId Int +} + model RepoToConnection { addedAt DateTime @default(now()) diff --git a/packages/schemas/src/v3/app.schema.ts b/packages/schemas/src/v3/app.schema.ts index 5b4b96f75..87a7eb27f 100644 --- a/packages/schemas/src/v3/app.schema.ts +++ b/packages/schemas/src/v3/app.schema.ts @@ -2,11 +2,9 @@ const schema = { "$schema": "http://json-schema.org/draft-07/schema#", "title": "AppConfig", - "oneOf": [ - { - "$schema": "http://json-schema.org/draft-07/schema#", + "definitions": { + "GitHubAppConfig": { "type": "object", - "title": "GithubAppConfig", "properties": { "type": { "const": "githubApp", @@ -60,19 +58,70 @@ const schema = { }, "required": [ "type", - "id" + "id", + "privateKey" ], - "oneOf": [ - { - "required": [ - "privateKey" + "additionalProperties": false + } + }, + "oneOf": [ + { + "type": "object", + "properties": { + "type": { + "const": "githubApp", + "description": "GitHub App Configuration" + }, + "deploymentHostname": { + "type": "string", + "format": "hostname", + "default": "github.com", + "description": "The hostname of the GitHub App deployment.", + "examples": [ + "github.com", + "github.example.com" ] }, - { - "required": [ - "privateKeyPath" + "id": { + "type": "string", + "description": "The ID of the GitHub App." + }, + "privateKey": { + "description": "The private key of the GitHub App.", + "anyOf": [ + { + "type": "object", + "properties": { + "secret": { + "type": "string", + "description": "The name of the secret that contains the token." + } + }, + "required": [ + "secret" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "env": { + "type": "string", + "description": "The name of the environment variable that contains the token. Only supported in declarative connection configs." + } + }, + "required": [ + "env" + ], + "additionalProperties": false + } ] } + }, + "required": [ + "type", + "id", + "privateKey" ], "additionalProperties": false } diff --git a/packages/schemas/src/v3/app.type.ts b/packages/schemas/src/v3/app.type.ts index 255ef033d..c087a7594 100644 --- a/packages/schemas/src/v3/app.type.ts +++ b/packages/schemas/src/v3/app.type.ts @@ -1,6 +1,34 @@ // THIS IS A AUTO-GENERATED FILE. DO NOT MODIFY MANUALLY! -export type AppConfig = GithubAppConfig; -export type GithubAppConfig = { - [k: string]: unknown; -}; +export type AppConfig = GitHubAppConfig; + +export interface GitHubAppConfig { + /** + * GitHub App Configuration + */ + type: "githubApp"; + /** + * The hostname of the GitHub App deployment. + */ + deploymentHostname?: string; + /** + * The ID of the GitHub App. + */ + id: string; + /** + * The private key of the GitHub App. + */ + privateKey: + | { + /** + * The name of the secret that contains the token. + */ + secret: string; + } + | { + /** + * The name of the environment variable that contains the token. Only supported in declarative connection configs. + */ + env: string; + }; +} diff --git a/packages/schemas/src/v3/githubApp.schema.ts b/packages/schemas/src/v3/githubApp.schema.ts deleted file mode 100644 index aab0ef20a..000000000 --- a/packages/schemas/src/v3/githubApp.schema.ts +++ /dev/null @@ -1,75 +0,0 @@ -// THIS IS A AUTO-GENERATED FILE. DO NOT MODIFY MANUALLY! -const schema = { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "title": "GithubAppConfig", - "properties": { - "type": { - "const": "githubApp", - "description": "GitHub App Configuration" - }, - "deploymentHostname": { - "type": "string", - "format": "hostname", - "default": "github.com", - "description": "The hostname of the GitHub App deployment.", - "examples": [ - "github.com", - "github.example.com" - ] - }, - "id": { - "type": "string", - "description": "The ID of the GitHub App." - }, - "privateKey": { - "description": "The private key of the GitHub App.", - "anyOf": [ - { - "type": "object", - "properties": { - "secret": { - "type": "string", - "description": "The name of the secret that contains the token." - } - }, - "required": [ - "secret" - ], - "additionalProperties": false - }, - { - "type": "object", - "properties": { - "env": { - "type": "string", - "description": "The name of the environment variable that contains the token. Only supported in declarative connection configs." - } - }, - "required": [ - "env" - ], - "additionalProperties": false - } - ] - } - }, - "required": [ - "type", - "id" - ], - "oneOf": [ - { - "required": [ - "privateKey" - ] - }, - { - "required": [ - "privateKeyPath" - ] - } - ], - "additionalProperties": false -} as const; -export { schema as githubAppSchema }; \ No newline at end of file diff --git a/packages/schemas/src/v3/githubApp.type.ts b/packages/schemas/src/v3/githubApp.type.ts deleted file mode 100644 index cd5af6af1..000000000 --- a/packages/schemas/src/v3/githubApp.type.ts +++ /dev/null @@ -1,34 +0,0 @@ -// THIS IS A AUTO-GENERATED FILE. DO NOT MODIFY MANUALLY! - -export type GithubAppConfig = { - /** - * GitHub App Configuration - */ - type: "githubApp"; - /** - * The hostname of the GitHub App deployment. - */ - deploymentHostname?: string; - /** - * The ID of the GitHub App. - */ - id: string; - /** - * The private key of the GitHub App. - */ - privateKey?: - | { - /** - * The name of the secret that contains the token. - */ - secret: string; - } - | { - /** - * The name of the environment variable that contains the token. Only supported in declarative connection configs. - */ - env: string; - }; -} & { - [k: string]: unknown; -}; diff --git a/packages/schemas/src/v3/index.schema.ts b/packages/schemas/src/v3/index.schema.ts index c326e7b65..d5125c697 100644 --- a/packages/schemas/src/v3/index.schema.ts +++ b/packages/schemas/src/v3/index.schema.ts @@ -4279,11 +4279,9 @@ const schema = { "items": { "$schema": "http://json-schema.org/draft-07/schema#", "title": "AppConfig", - "oneOf": [ - { - "$schema": "http://json-schema.org/draft-07/schema#", + "definitions": { + "GitHubAppConfig": { "type": "object", - "title": "GithubAppConfig", "properties": { "type": { "const": "githubApp", @@ -4337,19 +4335,70 @@ const schema = { }, "required": [ "type", - "id" + "id", + "privateKey" ], - "oneOf": [ - { - "required": [ - "privateKey" - ] + "additionalProperties": false + } + }, + "oneOf": [ + { + "type": "object", + "properties": { + "type": { + "const": "githubApp", + "description": "GitHub App Configuration" }, - { - "required": [ - "privateKeyPath" + "deploymentHostname": { + "type": "string", + "format": "hostname", + "default": "github.com", + "description": "The hostname of the GitHub App deployment.", + "examples": [ + "github.com", + "github.example.com" ] + }, + "id": { + "type": "string", + "description": "The ID of the GitHub App." + }, + "privateKey": { + "anyOf": [ + { + "type": "object", + "properties": { + "secret": { + "type": "string", + "description": "The name of the secret that contains the token." + } + }, + "required": [ + "secret" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "env": { + "type": "string", + "description": "The name of the environment variable that contains the token. Only supported in declarative connection configs." + } + }, + "required": [ + "env" + ], + "additionalProperties": false + } + ], + "description": "The private key of the GitHub App." } + }, + "required": [ + "type", + "id", + "privateKey" ], "additionalProperties": false } diff --git a/packages/schemas/src/v3/index.type.ts b/packages/schemas/src/v3/index.type.ts index 3d0b19478..53ae9533c 100644 --- a/packages/schemas/src/v3/index.type.ts +++ b/packages/schemas/src/v3/index.type.ts @@ -25,10 +25,7 @@ export type LanguageModel = | OpenAICompatibleLanguageModel | OpenRouterLanguageModel | XaiLanguageModel; -export type AppConfig = GithubAppConfig; -export type GithubAppConfig = { - [k: string]: unknown; -}; +export type AppConfig = GitHubAppConfig; export interface SourcebotConfig { $schema?: string; @@ -1073,3 +1070,33 @@ export interface XaiLanguageModel { baseUrl?: string; headers?: LanguageModelHeaders; } +export interface GitHubAppConfig { + /** + * GitHub App Configuration + */ + type: "githubApp"; + /** + * The hostname of the GitHub App deployment. + */ + deploymentHostname?: string; + /** + * The ID of the GitHub App. + */ + id: string; + /** + * The private key of the GitHub App. + */ + privateKey: + | { + /** + * The name of the secret that contains the token. + */ + secret: string; + } + | { + /** + * The name of the environment variable that contains the token. Only supported in declarative connection configs. + */ + env: string; + }; +} diff --git a/packages/shared/src/index.server.ts b/packages/shared/src/index.server.ts index bdaea067a..f3303c149 100644 --- a/packages/shared/src/index.server.ts +++ b/packages/shared/src/index.server.ts @@ -24,7 +24,4 @@ export { isRemotePath, getConfigSettings, } from "./utils.js"; -export { - syncSearchContexts, -} from "./ee/syncSearchContexts.js"; export * from "./constants.js"; \ No newline at end of file diff --git a/packages/web/package.json b/packages/web/package.json index 709d0e474..7818853c6 100644 --- a/packages/web/package.json +++ b/packages/web/package.json @@ -113,7 +113,6 @@ "ai": "^5.0.45", "ajv": "^8.17.1", "bcryptjs": "^3.0.2", - "chokidar": "^4.0.3", "class-variance-authority": "^0.7.0", "client-only": "^0.0.1", "clsx": "^2.1.1", diff --git a/packages/web/src/actions.ts b/packages/web/src/actions.ts index 6cacac68e..7c3a472ab 100644 --- a/packages/web/src/actions.ts +++ b/packages/web/src/actions.ts @@ -10,7 +10,7 @@ import { prisma } from "@/prisma"; import { render } from "@react-email/components"; import * as Sentry from '@sentry/nextjs'; import { encrypt, generateApiKey, getTokenFromConfig, hashSecret } from "@sourcebot/crypto"; -import { ApiKey, Org, OrgRole, Prisma, RepoIndexingJobStatus, RepoIndexingJobType, StripeSubscriptionStatus } from "@sourcebot/db"; +import { ApiKey, ConnectionSyncJobStatus, Org, OrgRole, Prisma, RepoIndexingJobStatus, RepoIndexingJobType, StripeSubscriptionStatus } from "@sourcebot/db"; import { createLogger } from "@sourcebot/logger"; import { GiteaConnectionConfig } from "@sourcebot/schemas/v3/gitea.type"; import { GithubConnectionConfig } from "@sourcebot/schemas/v3/github.type"; @@ -30,7 +30,7 @@ import JoinRequestSubmittedEmail from "./emails/joinRequestSubmittedEmail"; import { AGENTIC_SEARCH_TUTORIAL_DISMISSED_COOKIE_NAME, MOBILE_UNSUPPORTED_SPLASH_SCREEN_DISMISSED_COOKIE_NAME, SINGLE_TENANT_ORG_DOMAIN, SOURCEBOT_GUEST_USER_ID, SOURCEBOT_SUPPORT_EMAIL } from "./lib/constants"; import { orgDomainSchema, orgNameSchema, repositoryQuerySchema } from "./lib/schemas"; import { ApiKeyPayload, TenancyMode } from "./lib/types"; -import { withOptionalAuthV2 } from "./withAuthV2"; +import { withAuthV2, withOptionalAuthV2 } from "./withAuthV2"; const logger = createLogger('web-actions'); const auditService = getAuditService(); @@ -593,6 +593,7 @@ export const getReposStats = async () => sew(() => prisma.repo.count({ where: { orgId: org.id, + indexedAt: null, jobs: { some: { type: RepoIndexingJobType.INDEX, @@ -604,7 +605,6 @@ export const getReposStats = async () => sew(() => } }, }, - indexedAt: null, } }), prisma.repo.count({ @@ -625,6 +625,42 @@ export const getReposStats = async () => sew(() => }) ) +export const getConnectionStats = async () => sew(() => + withAuthV2(async ({ org, prisma }) => { + const [ + numberOfConnections, + numberOfConnectionsWithFirstTimeSyncJobsInProgress, + ] = await Promise.all([ + prisma.connection.count({ + where: { + orgId: org.id, + } + }), + prisma.connection.count({ + where: { + orgId: org.id, + syncedAt: null, + syncJobs: { + some: { + status: { + in: [ + ConnectionSyncJobStatus.PENDING, + ConnectionSyncJobStatus.IN_PROGRESS, + ] + } + } + } + } + }) + ]); + + return { + numberOfConnections, + numberOfConnectionsWithFirstTimeSyncJobsInProgress, + }; + }) +); + export const getRepoInfoByName = async (repoName: string) => sew(() => withOptionalAuthV2(async ({ org, prisma }) => { // @note: repo names are represented by their remote url diff --git a/packages/web/src/app/[domain]/chat/components/demoCards.tsx b/packages/web/src/app/[domain]/chat/components/demoCards.tsx index 016d9605f..5c8d4e5c4 100644 --- a/packages/web/src/app/[domain]/chat/components/demoCards.tsx +++ b/packages/web/src/app/[domain]/chat/components/demoCards.tsx @@ -7,7 +7,7 @@ import { Badge } from "@/components/ui/badge"; import { Card } from "@/components/ui/card"; import { CardContent } from "@/components/ui/card"; import { DemoExamples, DemoSearchExample, DemoSearchScope } from "@/types"; -import { cn, getCodeHostIcon } from "@/lib/utils"; +import { cn, CodeHostType, getCodeHostIcon } from "@/lib/utils"; import useCaptureEvent from "@/hooks/useCaptureEvent"; import { SearchScopeInfoCard } from "@/features/chat/components/chatBox/searchScopeInfoCard"; @@ -41,25 +41,23 @@ export const DemoCards = ({ } if (searchScope.codeHostType) { - const codeHostIcon = getCodeHostIcon(searchScope.codeHostType); - if (codeHostIcon) { - // When selected, icons need to match the inverted badge colors - // In light mode selected: light icon on dark bg (invert) - // In dark mode selected: dark icon on light bg (no invert, override dark:invert) - const selectedIconClass = isSelected - ? "invert dark:invert-0" - : codeHostIcon.className; + const codeHostIcon = getCodeHostIcon(searchScope.codeHostType as CodeHostType); + // When selected, icons need to match the inverted badge colors + // In light mode selected: light icon on dark bg (invert) + // In dark mode selected: dark icon on light bg (no invert, override dark:invert) + const selectedIconClass = isSelected + ? "invert dark:invert-0" + : codeHostIcon.className; - return ( - {`${searchScope.codeHostType} - ); - } + return ( + {`${searchScope.codeHostType} + ); } return ; diff --git a/packages/web/src/app/[domain]/components/backButton.tsx b/packages/web/src/app/[domain]/components/backButton.tsx new file mode 100644 index 000000000..191715c29 --- /dev/null +++ b/packages/web/src/app/[domain]/components/backButton.tsx @@ -0,0 +1,20 @@ +import { cn } from "@/lib/utils"; +import { ArrowLeft } from "lucide-react" +import Link from "next/link" + +interface BackButtonProps { + href: string; + name: string; + className?: string; +} + +export function BackButton({ href, name, className }: BackButtonProps) { + return ( + + + + {name} + + + ) +} diff --git a/packages/web/src/app/[domain]/components/navigationMenu/index.tsx b/packages/web/src/app/[domain]/components/navigationMenu/index.tsx index 548427310..f350fe92a 100644 --- a/packages/web/src/app/[domain]/components/navigationMenu/index.tsx +++ b/packages/web/src/app/[domain]/components/navigationMenu/index.tsx @@ -1,4 +1,4 @@ -import { getRepos, getReposStats } from "@/actions"; +import { getConnectionStats, getRepos, getReposStats } from "@/actions"; import { SourcebotLogo } from "@/app/components/sourcebotLogo"; import { auth } from "@/auth"; import { Button } from "@/components/ui/button"; @@ -39,6 +39,11 @@ export const NavigationMenu = async ({ throw new ServiceErrorException(repoStats); } + const connectionStats = isAuthenticated ? await getConnectionStats() : null; + if (isServiceError(connectionStats)) { + throw new ServiceErrorException(connectionStats); + } + const sampleRepos = await getRepos({ where: { jobs: { @@ -93,7 +98,12 @@ export const NavigationMenu = async ({ 0} + isSettingsButtonNotificationDotVisible={ + connectionStats ? + connectionStats.numberOfConnectionsWithFirstTimeSyncJobsInProgress > 0 : + false + } isAuthenticated={isAuthenticated} /> diff --git a/packages/web/src/app/[domain]/components/navigationMenu/navigationItems.tsx b/packages/web/src/app/[domain]/components/navigationMenu/navigationItems.tsx index 52db3315a..b5a314837 100644 --- a/packages/web/src/app/[domain]/components/navigationMenu/navigationItems.tsx +++ b/packages/web/src/app/[domain]/components/navigationMenu/navigationItems.tsx @@ -3,20 +3,23 @@ import { NavigationMenuItem, NavigationMenuLink, NavigationMenuList, navigationMenuTriggerStyle } from "@/components/ui/navigation-menu"; import { Badge } from "@/components/ui/badge"; import { cn, getShortenedNumberDisplayString } from "@/lib/utils"; -import { SearchIcon, MessageCircleIcon, BookMarkedIcon, SettingsIcon, CircleIcon } from "lucide-react"; +import { SearchIcon, MessageCircleIcon, BookMarkedIcon, SettingsIcon } from "lucide-react"; import { usePathname } from "next/navigation"; +import { NotificationDot } from "../notificationDot"; interface NavigationItemsProps { domain: string; numberOfRepos: number; - numberOfReposWithFirstTimeIndexingJobsInProgress: number; + isReposButtonNotificationDotVisible: boolean; + isSettingsButtonNotificationDotVisible: boolean; isAuthenticated: boolean; } export const NavigationItems = ({ domain, numberOfRepos, - numberOfReposWithFirstTimeIndexingJobsInProgress, + isReposButtonNotificationDotVisible, + isSettingsButtonNotificationDotVisible, isAuthenticated, }: NavigationItemsProps) => { const pathname = usePathname(); @@ -59,9 +62,7 @@ export const NavigationItems = ({ Repositories {getShortenedNumberDisplayString(numberOfRepos)} - {numberOfReposWithFirstTimeIndexingJobsInProgress > 0 && ( - - )} + {isReposButtonNotificationDotVisible && } {isActive(`/${domain}/repos`) && } @@ -74,6 +75,7 @@ export const NavigationItems = ({ > Settings + {isSettingsButtonNotificationDotVisible && } {isActive(`/${domain}/settings`) && } @@ -86,4 +88,4 @@ const ActiveIndicator = () => { return (
); -}; \ No newline at end of file +}; diff --git a/packages/web/src/app/[domain]/components/navigationMenu/progressIndicator.tsx b/packages/web/src/app/[domain]/components/navigationMenu/progressIndicator.tsx index 8e1889df0..2584f11c1 100644 --- a/packages/web/src/app/[domain]/components/navigationMenu/progressIndicator.tsx +++ b/packages/web/src/app/[domain]/components/navigationMenu/progressIndicator.tsx @@ -6,6 +6,7 @@ import { Button } from "@/components/ui/button"; import { Separator } from "@/components/ui/separator"; import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip"; import { useDomain } from "@/hooks/useDomain"; +import { SINGLE_TENANT_ORG_DOMAIN } from "@/lib/constants"; import { RepositoryQuery } from "@/lib/types"; import { getCodeHostInfoForRepo, getShortenedNumberDisplayString } from "@/lib/utils"; import clsx from "clsx"; @@ -110,13 +111,14 @@ const RepoItem = ({ repo }: { repo: RepositoryQuery }) => { return ( -
{repoIcon} {displayName} -
+ ) } \ No newline at end of file diff --git a/packages/web/src/app/[domain]/components/notificationDot.tsx b/packages/web/src/app/[domain]/components/notificationDot.tsx new file mode 100644 index 000000000..796247053 --- /dev/null +++ b/packages/web/src/app/[domain]/components/notificationDot.tsx @@ -0,0 +1,9 @@ +import { cn } from "@/lib/utils" + +interface NotificationDotProps { + className?: string +} + +export const NotificationDot = ({ className }: NotificationDotProps) => { + return
+} \ No newline at end of file diff --git a/packages/web/src/app/[domain]/components/repositoryCarousel.tsx b/packages/web/src/app/[domain]/components/repositoryCarousel.tsx index bd4f2d1aa..a9d1239f3 100644 --- a/packages/web/src/app/[domain]/components/repositoryCarousel.tsx +++ b/packages/web/src/app/[domain]/components/repositoryCarousel.tsx @@ -37,7 +37,7 @@ export function RepositoryCarousel({ <> Create a{" "} - + connection {" "} to start indexing repositories diff --git a/packages/web/src/app/[domain]/repos/[id]/page.tsx b/packages/web/src/app/[domain]/repos/[id]/page.tsx index 715afa5cc..db14e50c1 100644 --- a/packages/web/src/app/[domain]/repos/[id]/page.tsx +++ b/packages/web/src/app/[domain]/repos/[id]/page.tsx @@ -4,21 +4,21 @@ import { Button } from "@/components/ui/button" import { Card, CardContent, CardDescription, CardHeader, CardTitle } from "@/components/ui/card" import { Skeleton } from "@/components/ui/skeleton" import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip" +import { env } from "@/env.mjs" import { SINGLE_TENANT_ORG_DOMAIN } from "@/lib/constants" import { ServiceErrorException } from "@/lib/serviceError" import { cn, getCodeHostInfoForRepo, isServiceError } from "@/lib/utils" import { withOptionalAuthV2 } from "@/withAuthV2" -import { ChevronLeft, ExternalLink, Info } from "lucide-react" +import { getConfigSettings, repoMetadataSchema } from "@sourcebot/shared" +import { ExternalLink, Info } from "lucide-react" import Image from "next/image" import Link from "next/link" import { notFound } from "next/navigation" import { Suspense } from "react" -import { RepoJobsTable } from "../components/repoJobsTable" -import { getConfigSettings } from "@sourcebot/shared" -import { env } from "@/env.mjs" +import { BackButton } from "../../components/backButton" import { DisplayDate } from "../../components/DisplayDate" import { RepoBranchesTable } from "../components/repoBranchesTable" -import { repoMetadataSchema } from "@sourcebot/shared" +import { RepoJobsTable } from "../components/repoJobsTable" export default async function RepoDetailPage({ params }: { params: Promise<{ id: string }> }) { const { id } = await params @@ -52,14 +52,13 @@ export default async function RepoDetailPage({ params }: { params: Promise<{ id: const repoMetadata = repoMetadataSchema.parse(repo.metadata); return ( -
+ <>
- +
@@ -103,7 +102,7 @@ export default async function RepoDetailPage({ params }: { params: Promise<{ id: - + @@ -122,7 +121,7 @@ export default async function RepoDetailPage({ params }: { params: Promise<{ id: - {repo.indexedAt ? : "Never"} + {repo.indexedAt ? : "Never"} @@ -141,7 +140,7 @@ export default async function RepoDetailPage({ params }: { params: Promise<{ id: - {nextIndexAttempt ? : "-"} + {nextIndexAttempt ? : "-"}
@@ -168,7 +167,7 @@ export default async function RepoDetailPage({ params }: { params: Promise<{ id: - Indexing Jobs + Indexing History History of all indexing and cleanup jobs for this repository. @@ -177,16 +176,17 @@ export default async function RepoDetailPage({ params }: { params: Promise<{ id: -
+ ) } const getRepoWithJobs = async (repoId: number) => sew(() => - withOptionalAuthV2(async ({ prisma }) => { + withOptionalAuthV2(async ({ prisma, org }) => { const repo = await prisma.repo.findUnique({ where: { id: repoId, + orgId: org.id, }, include: { jobs: { diff --git a/packages/web/src/app/[domain]/repos/components/reposTable.tsx b/packages/web/src/app/[domain]/repos/components/reposTable.tsx index 755d638a1..730a25856 100644 --- a/packages/web/src/app/[domain]/repos/components/reposTable.tsx +++ b/packages/web/src/app/[domain]/repos/components/reposTable.tsx @@ -37,6 +37,7 @@ import { useRouter } from "next/navigation" import { useToast } from "@/components/hooks/use-toast"; import { DisplayDate } from "../../components/DisplayDate" import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip" +import { NotificationDot } from "../../components/notificationDot" // @see: https://v0.app/chat/repo-indexing-status-uhjdDim8OUS @@ -53,6 +54,7 @@ export type Repo = { imageUrl: string | null indexedCommitHash: string | null latestJobStatus: "PENDING" | "IN_PROGRESS" | "COMPLETED" | "FAILED" | null + isFirstTimeIndex: boolean } const statusBadgeVariants = cva("", { @@ -111,14 +113,32 @@ export const columns: ColumnDef[] = [ {repo.displayName?.charAt(0) ?? repo.name.charAt(0)}
)} - + + {/* Link to the details page (instead of browse) when the repo is indexing + as the code will not be available yet */} + {repo.displayName || repo.name} + {repo.isFirstTimeIndex && ( + + + + + + + + This is the first time Sourcebot is indexing this repository. It may take a few minutes to complete. + + + )}
) }, @@ -150,7 +170,7 @@ export const columns: ColumnDef[] = [ } return ( - + ) } }, @@ -177,11 +197,11 @@ export const columns: ColumnDef[] = [ const HashComponent = commitUrl ? ( - {smallHash} - + href={commitUrl} + className="font-mono text-sm text-link hover:underline" + > + {smallHash} + ) : ( {smallHash} @@ -331,13 +351,13 @@ export const ReposTable = ({ data }: { data: Repo[] }) => {
- +
{table.getHeaderGroups().map((headerGroup) => ( {headerGroup.headers.map((header) => { return ( - @@ -353,7 +373,7 @@ export const ReposTable = ({ data }: { data: Repo[] }) => { table.getRowModel().rows.map((row) => ( {row.getVisibleCells().map((cell) => ( - diff --git a/packages/web/src/app/[domain]/repos/layout.tsx b/packages/web/src/app/[domain]/repos/layout.tsx index 6ff10adff..675fdf949 100644 --- a/packages/web/src/app/[domain]/repos/layout.tsx +++ b/packages/web/src/app/[domain]/repos/layout.tsx @@ -1,4 +1,11 @@ +import { InfoIcon } from "lucide-react"; import { NavigationMenu } from "../components/navigationMenu"; +import { SINGLE_TENANT_ORG_DOMAIN } from "@/lib/constants"; +import Link from "next/link"; +import { getCurrentUserRole, getReposStats } from "@/actions"; +import { isServiceError } from "@/lib/utils"; +import { ServiceErrorException } from "@/lib/serviceError"; +import { OrgRole } from "@sourcebot/db"; interface LayoutProps { children: React.ReactNode; @@ -12,11 +19,28 @@ export default async function Layout( const { domain } = params; const { children } = props; + const repoStats = await getReposStats(); + if (isServiceError(repoStats)) { + throw new ServiceErrorException(repoStats); + } + + const userRoleInOrg = await getCurrentUserRole(domain); + return (
+ {(repoStats.numberOfRepos === 0 && userRoleInOrg === OrgRole.OWNER) && ( +
+ + No repositories configured. Create a connection to get started. +
+ )}
-
{children}
+
+
+ {children} +
+
) diff --git a/packages/web/src/app/[domain]/repos/page.tsx b/packages/web/src/app/[domain]/repos/page.tsx index 3e6120819..79c4497bc 100644 --- a/packages/web/src/app/[domain]/repos/page.tsx +++ b/packages/web/src/app/[domain]/repos/page.tsx @@ -3,16 +3,33 @@ import { ServiceErrorException } from "@/lib/serviceError"; import { isServiceError } from "@/lib/utils"; import { withOptionalAuthV2 } from "@/withAuthV2"; import { ReposTable } from "./components/reposTable"; +import { RepoIndexingJobStatus } from "@sourcebot/db"; export default async function ReposPage() { - const repos = await getReposWithLatestJob(); - if (isServiceError(repos)) { - throw new ServiceErrorException(repos); + const _repos = await getReposWithLatestJob(); + if (isServiceError(_repos)) { + throw new ServiceErrorException(_repos); } + const repos = _repos + .map((repo) => ({ + ...repo, + latestJobStatus: repo.jobs.length > 0 ? repo.jobs[0].status : null, + isFirstTimeIndex: repo.indexedAt === null && repo.jobs.filter((job) => job.status === RepoIndexingJobStatus.PENDING || job.status === RepoIndexingJobStatus.IN_PROGRESS).length > 0, + })) + .sort((a, b) => { + if (a.isFirstTimeIndex && !b.isFirstTimeIndex) { + return -1; + } + if (!a.isFirstTimeIndex && b.isFirstTimeIndex) { + return 1; + } + return a.name.localeCompare(b.name); + }); + return ( -
+ <>

Repositories

View and manage your code repositories and their indexing status.

@@ -27,16 +44,17 @@ export default async function ReposPage() { createdAt: repo.createdAt, webUrl: repo.webUrl, imageUrl: repo.imageUrl, - latestJobStatus: repo.jobs.length > 0 ? repo.jobs[0].status : null, + latestJobStatus: repo.latestJobStatus, + isFirstTimeIndex: repo.isFirstTimeIndex, codeHostType: repo.external_codeHostType, indexedCommitHash: repo.indexedCommitHash, }))} /> -
+ ) } const getReposWithLatestJob = async () => sew(() => - withOptionalAuthV2(async ({ prisma }) => { + withOptionalAuthV2(async ({ prisma, org }) => { const repos = await prisma.repo.findMany({ include: { jobs: { @@ -48,6 +66,9 @@ const getReposWithLatestJob = async () => sew(() => }, orderBy: { name: 'asc' + }, + where: { + orgId: org.id, } }); return repos; diff --git a/packages/web/src/app/[domain]/settings/components/header.tsx b/packages/web/src/app/[domain]/settings/components/header.tsx deleted file mode 100644 index 79a24ee4c..000000000 --- a/packages/web/src/app/[domain]/settings/components/header.tsx +++ /dev/null @@ -1,22 +0,0 @@ -import { Separator } from "@/components/ui/separator"; -import { cn } from "@/lib/utils"; -import clsx from "clsx"; - -interface HeaderProps { - children: React.ReactNode; - withTopMargin?: boolean; - className?: string; -} - -export const Header = ({ - children, - withTopMargin = true, - className, -}: HeaderProps) => { - return ( -
- {children} - -
- ) -} \ No newline at end of file diff --git a/packages/web/src/app/[domain]/settings/components/sidebar-nav.tsx b/packages/web/src/app/[domain]/settings/components/sidebar-nav.tsx index ddf37adbd..78b5a7ccc 100644 --- a/packages/web/src/app/[domain]/settings/components/sidebar-nav.tsx +++ b/packages/web/src/app/[domain]/settings/components/sidebar-nav.tsx @@ -1,44 +1,54 @@ "use client" -import React from "react" +import { buttonVariants } from "@/components/ui/button" +import { NotificationDot } from "@/app/[domain]/components/notificationDot" +import { cn } from "@/lib/utils" import Link from "next/link" import { usePathname } from "next/navigation" -import { cn } from "@/lib/utils" -import { buttonVariants } from "@/components/ui/button" +import React from "react" -interface SidebarNavProps extends React.HTMLAttributes { - items: { +export type SidebarNavItem = { href: string + hrefRegex?: string title: React.ReactNode - }[] + isNotificationDotVisible?: boolean +} + +interface SidebarNavProps extends React.HTMLAttributes { + items: SidebarNavItem[] } export function SidebarNav({ className, items, ...props }: SidebarNavProps) { - const pathname = usePathname() + const pathname = usePathname() - return ( - - ) + {items.map((item) => { + const isActive = item.hrefRegex ? new RegExp(item.hrefRegex).test(pathname) : pathname === item.href; + + return ( + + {item.title} + {item.isNotificationDotVisible && } + + ) + })} + + ) } \ No newline at end of file diff --git a/packages/web/src/app/[domain]/settings/connections/[id]/page.tsx b/packages/web/src/app/[domain]/settings/connections/[id]/page.tsx new file mode 100644 index 000000000..5db0d28c3 --- /dev/null +++ b/packages/web/src/app/[domain]/settings/connections/[id]/page.tsx @@ -0,0 +1,204 @@ +import { sew } from "@/actions"; +import { BackButton } from "@/app/[domain]/components/backButton"; +import { DisplayDate } from "@/app/[domain]/components/DisplayDate"; +import { Card, CardContent, CardDescription, CardHeader, CardTitle } from "@/components/ui/card"; +import { Skeleton } from "@/components/ui/skeleton"; +import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip"; +import { env } from "@/env.mjs"; +import { SINGLE_TENANT_ORG_DOMAIN } from "@/lib/constants"; +import { notFound, ServiceErrorException } from "@/lib/serviceError"; +import { CodeHostType, isServiceError } from "@/lib/utils"; +import { withAuthV2 } from "@/withAuthV2"; +import { AzureDevOpsConnectionConfig, BitbucketConnectionConfig, GenericGitHostConnectionConfig, GerritConnectionConfig, GiteaConnectionConfig, GithubConnectionConfig, GitlabConnectionConfig } from "@sourcebot/schemas/v3/index.type"; +import { getConfigSettings } from "@sourcebot/shared"; +import { Info } from "lucide-react"; +import Link from "next/link"; +import { Suspense } from "react"; +import { ConnectionJobsTable } from "../components/connectionJobsTable"; + +interface ConnectionDetailPageProps { + params: Promise<{ + id: string + }> +} + + +export default async function ConnectionDetailPage(props: ConnectionDetailPageProps) { + const params = await props.params; + const { id } = params; + + const connection = await getConnectionWithJobs(Number.parseInt(id)); + if (isServiceError(connection)) { + throw new ServiceErrorException(connection); + } + + const configSettings = await getConfigSettings(env.CONFIG_PATH); + + const nextSyncAttempt = (() => { + const latestJob = connection.syncJobs.length > 0 ? connection.syncJobs[0] : null; + if (!latestJob) { + return undefined; + } + + if (latestJob.completedAt) { + return new Date(latestJob.completedAt.getTime() + configSettings.resyncConnectionIntervalMs); + } + + return undefined; + })(); + + const codeHostUrl = (() => { + const connectionType = connection.connectionType as CodeHostType; + switch (connectionType) { + case 'github': { + const config = connection.config as unknown as GithubConnectionConfig; + return config.url ?? 'https://github.com'; + } + case 'gitlab': { + const config = connection.config as unknown as GitlabConnectionConfig; + return config.url ?? 'https://gitlab.com'; + } + case 'gitea': { + const config = connection.config as unknown as GiteaConnectionConfig; + return config.url ?? 'https://gitea.com'; + } + case 'gerrit': { + const config = connection.config as unknown as GerritConnectionConfig; + return config.url; + } + case 'bitbucket-server': { + const config = connection.config as unknown as BitbucketConnectionConfig; + return config.url!; + } + case 'bitbucket-cloud': { + const config = connection.config as unknown as BitbucketConnectionConfig; + return config.url ?? 'https://bitbucket.org'; + } + case 'azuredevops': { + const config = connection.config as unknown as AzureDevOpsConnectionConfig; + return config.url ?? 'https://dev.azure.com'; + } + case 'generic-git-host': { + const config = connection.config as unknown as GenericGitHostConnectionConfig; + return config.url; + } + } + })(); + + return ( +
+ +
+

{connection.name}

+ + + {codeHostUrl} + +
+ +
+ + + + Created + + + + + +

When this connection was first added to Sourcebot

+
+
+
+
+ + + +
+ + + + + Last synced + + + + + +

The last time this connection was successfully synced

+
+
+
+
+ + {connection.syncedAt ? : "Never"} + +
+ + + + + Scheduled + + + + + +

When the connection will be resynced next. Modifying the config will also trigger a resync.

+
+
+
+
+ + {nextSyncAttempt ? : "-"} + +
+
+ + + + Sync History + History of all sync jobs for this connection. + + + }> + + + + +
+ ) +} + +const getConnectionWithJobs = async (id: number) => sew(() => + withAuthV2(async ({ prisma, org }) => { + const connection = await prisma.connection.findUnique({ + where: { + id, + orgId: org.id, + }, + include: { + syncJobs: { + orderBy: { + createdAt: 'desc', + }, + }, + }, + }); + + if (!connection) { + return notFound(); + } + + return connection; + }) +) \ No newline at end of file diff --git a/packages/web/src/app/[domain]/settings/connections/components/connectionJobsTable.tsx b/packages/web/src/app/[domain]/settings/connections/components/connectionJobsTable.tsx new file mode 100644 index 000000000..fec991cb4 --- /dev/null +++ b/packages/web/src/app/[domain]/settings/connections/components/connectionJobsTable.tsx @@ -0,0 +1,311 @@ +"use client" + +import { Badge } from "@/components/ui/badge" +import { Button } from "@/components/ui/button" +import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select" +import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from "@/components/ui/table" +import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from "@/components/ui/tooltip" +import { + type ColumnDef, + type ColumnFiltersState, + type SortingState, + type VisibilityState, + flexRender, + getCoreRowModel, + getFilteredRowModel, + getPaginationRowModel, + getSortedRowModel, + useReactTable, +} from "@tanstack/react-table" +import { cva } from "class-variance-authority" +import { AlertCircle, AlertTriangle, ArrowUpDown, RefreshCwIcon } from "lucide-react" +import * as React from "react" +import { CopyIconButton } from "@/app/[domain]/components/copyIconButton" +import { useMemo } from "react" +import { LightweightCodeHighlighter } from "@/app/[domain]/components/lightweightCodeHighlighter" +import { useRouter } from "next/navigation" +import { useToast } from "@/components/hooks/use-toast" +import { DisplayDate } from "@/app/[domain]/components/DisplayDate" + + +export type ConnectionSyncJob = { + id: string + status: "PENDING" | "IN_PROGRESS" | "COMPLETED" | "FAILED" + createdAt: Date + updatedAt: Date + completedAt: Date | null + errorMessage: string | null + warningMessages: string[] +} + +const statusBadgeVariants = cva("", { + variants: { + status: { + PENDING: "bg-secondary text-secondary-foreground hover:bg-secondary/80", + IN_PROGRESS: "bg-primary text-primary-foreground hover:bg-primary/90", + COMPLETED: "bg-green-600 text-white hover:bg-green-700", + FAILED: "bg-destructive text-destructive-foreground hover:bg-destructive/90", + }, + }, +}) + +const getStatusBadge = (status: ConnectionSyncJob["status"]) => { + const labels = { + PENDING: "Pending", + IN_PROGRESS: "In Progress", + COMPLETED: "Completed", + FAILED: "Failed", + } + + return {labels[status]} +} + +const getDuration = (start: Date, end: Date | null) => { + if (!end) return "-" + const diff = end.getTime() - start.getTime() + const minutes = Math.floor(diff / 60000) + const seconds = Math.floor((diff % 60000) / 1000) + return `${minutes}m ${seconds}s` +} + +export const columns: ColumnDef[] = [ + { + accessorKey: "status", + header: "Status", + cell: ({ row }) => { + const job = row.original + return ( +
+ {getStatusBadge(row.getValue("status"))} + {job.errorMessage ? ( + + + + + + + + {job.errorMessage} + + + + + ) : job.warningMessages.length > 0 ? ( + + + + + + +

{job.warningMessages.length} warning(s) while syncing:

+
+ {job.warningMessages.map((warning, index) => ( +
+ {index + 1}. + {warning} +
+ ))} +
+
+
+
+ ) : null} +
+ ) + }, + filterFn: (row, id, value) => { + return value.includes(row.getValue(id)) + }, + }, + { + accessorKey: "createdAt", + header: ({ column }) => { + return ( + + ) + }, + cell: ({ row }) => , + }, + { + accessorKey: "completedAt", + header: ({ column }) => { + return ( + + ) + }, + cell: ({ row }) => { + const completedAt = row.getValue("completedAt") as Date | null; + if (!completedAt) { + return "-"; + } + + return + }, + }, + { + id: "duration", + header: "Duration", + cell: ({ row }) => { + const job = row.original + return getDuration(job.createdAt, job.completedAt) + }, + }, + { + accessorKey: "id", + header: "Job ID", + cell: ({ row }) => { + const id = row.getValue("id") as string + return ( +
+ {id} + { + navigator.clipboard.writeText(id); + return true; + }} /> +
+ ) + }, + }, +] + +export const ConnectionJobsTable = ({ data }: { data: ConnectionSyncJob[] }) => { + const [sorting, setSorting] = React.useState([{ id: "createdAt", desc: true }]) + const [columnFilters, setColumnFilters] = React.useState([]) + const [columnVisibility, setColumnVisibility] = React.useState({}) + const router = useRouter(); + const { toast } = useToast(); + + const table = useReactTable({ + data, + columns, + onSortingChange: setSorting, + onColumnFiltersChange: setColumnFilters, + getCoreRowModel: getCoreRowModel(), + getPaginationRowModel: getPaginationRowModel(), + getSortedRowModel: getSortedRowModel(), + getFilteredRowModel: getFilteredRowModel(), + onColumnVisibilityChange: setColumnVisibility, + state: { + sorting, + columnFilters, + columnVisibility, + }, + }) + + const { + numCompleted, + numInProgress, + numPending, + numFailed, + } = useMemo(() => { + return { + numCompleted: data.filter((job) => job.status === "COMPLETED").length, + numInProgress: data.filter((job) => job.status === "IN_PROGRESS").length, + numPending: data.filter((job) => job.status === "PENDING").length, + numFailed: data.filter((job) => job.status === "FAILED").length, + }; + }, [data]); + + return ( +
+
+ + + +
+ +
+
+ + {table.getHeaderGroups().map((headerGroup) => ( + + {headerGroup.headers.map((header) => { + return ( + + {header.isPlaceholder ? null : flexRender(header.column.columnDef.header, header.getContext())} + + ) + })} + + ))} + + + {table.getRowModel().rows?.length ? ( + table.getRowModel().rows.map((row) => ( + + {row.getVisibleCells().map((cell) => ( + {flexRender(cell.column.columnDef.cell, cell.getContext())} + ))} + + )) + ) : ( + + + No sync jobs found. + + + )} + +
+
+ +
+
+ {table.getFilteredRowModel().rows.length} job(s) total +
+
+ + +
+
+
+ ) +} diff --git a/packages/web/src/app/[domain]/settings/connections/components/connectionsTable.tsx b/packages/web/src/app/[domain]/settings/connections/components/connectionsTable.tsx new file mode 100644 index 000000000..db161124f --- /dev/null +++ b/packages/web/src/app/[domain]/settings/connections/components/connectionsTable.tsx @@ -0,0 +1,295 @@ +"use client" + +import { DisplayDate } from "@/app/[domain]/components/DisplayDate" +import { NotificationDot } from "@/app/[domain]/components/notificationDot" +import { useToast } from "@/components/hooks/use-toast" +import { Badge } from "@/components/ui/badge" +import { Button } from "@/components/ui/button" +import { Input } from "@/components/ui/input" +import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select" +import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from "@/components/ui/table" +import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip" +import { SINGLE_TENANT_ORG_DOMAIN } from "@/lib/constants" +import { CodeHostType, getCodeHostIcon } from "@/lib/utils" +import { + type ColumnDef, + type ColumnFiltersState, + type SortingState, + type VisibilityState, + flexRender, + getCoreRowModel, + getFilteredRowModel, + getPaginationRowModel, + getSortedRowModel, + useReactTable, +} from "@tanstack/react-table" +import { cva } from "class-variance-authority" +import { ArrowUpDown, RefreshCwIcon } from "lucide-react" +import Image from "next/image" +import Link from "next/link" +import { useRouter } from "next/navigation" +import { useMemo, useState } from "react" + + +export type Connection = { + id: number + name: string + syncedAt: Date | null + codeHostType: CodeHostType + latestJobStatus: "PENDING" | "IN_PROGRESS" | "COMPLETED" | "FAILED" | null + isFirstTimeSync: boolean +} + +const statusBadgeVariants = cva("", { + variants: { + status: { + PENDING: "bg-secondary text-secondary-foreground hover:bg-secondary/80", + IN_PROGRESS: "bg-primary text-primary-foreground hover:bg-primary/90", + COMPLETED: "bg-green-600 text-white hover:bg-green-700", + FAILED: "bg-destructive text-destructive-foreground hover:bg-destructive/90", + NO_JOBS: "bg-secondary text-secondary-foreground hover:bg-secondary/80", + }, + }, +}) + +const getStatusBadge = (status: Connection["latestJobStatus"]) => { + if (!status) { + return No Jobs + } + + const labels = { + PENDING: "Pending", + IN_PROGRESS: "In Progress", + COMPLETED: "Completed", + FAILED: "Failed", + } + + return {labels[status]} +} + +export const columns: ColumnDef[] = [ + { + accessorKey: "name", + size: 400, + header: ({ column }) => { + return ( + + ) + }, + cell: ({ row }) => { + const connection = row.original; + const codeHostIcon = getCodeHostIcon(connection.codeHostType); + + return ( +
+ {`${connection.codeHostType} + + {connection.name} + + {connection.isFirstTimeSync && ( + + + + + + + + This is the first time Sourcebot is syncing this connection. It may take a few minutes to complete. + + + )} +
+ ) + }, + }, + { + accessorKey: "latestJobStatus", + size: 150, + header: "Lastest status", + cell: ({ row }) => getStatusBadge(row.getValue("latestJobStatus")), + }, + { + accessorKey: "syncedAt", + size: 200, + header: ({ column }) => { + return ( + + ) + }, + cell: ({ row }) => { + const syncedAt = row.getValue("syncedAt") as Date | null; + if (!syncedAt) { + return "-"; + } + + return ( + + ) + } + }, +] + +export const ConnectionsTable = ({ data }: { data: Connection[] }) => { + const [sorting, setSorting] = useState([]) + const [columnFilters, setColumnFilters] = useState([]) + const [columnVisibility, setColumnVisibility] = useState({}) + const [rowSelection, setRowSelection] = useState({}) + const router = useRouter(); + const { toast } = useToast(); + + const { + numCompleted, + numInProgress, + numPending, + numFailed, + numNoJobs, + } = useMemo(() => { + return { + numCompleted: data.filter((connection) => connection.latestJobStatus === "COMPLETED").length, + numInProgress: data.filter((connection) => connection.latestJobStatus === "IN_PROGRESS").length, + numPending: data.filter((connection) => connection.latestJobStatus === "PENDING").length, + numFailed: data.filter((connection) => connection.latestJobStatus === "FAILED").length, + numNoJobs: data.filter((connection) => connection.latestJobStatus === null).length, + } + }, [data]); + + const table = useReactTable({ + data, + columns, + onSortingChange: setSorting, + onColumnFiltersChange: setColumnFilters, + getCoreRowModel: getCoreRowModel(), + getPaginationRowModel: getPaginationRowModel(), + getSortedRowModel: getSortedRowModel(), + getFilteredRowModel: getFilteredRowModel(), + onColumnVisibilityChange: setColumnVisibility, + onRowSelectionChange: setRowSelection, + columnResizeMode: 'onChange', + enableColumnResizing: false, + state: { + sorting, + columnFilters, + columnVisibility, + rowSelection, + }, + }) + + return ( +
+
+ table.getColumn("name")?.setFilterValue(event.target.value)} + className="max-w-sm" + /> + + +
+
+ + + {table.getHeaderGroups().map((headerGroup) => ( + + {headerGroup.headers.map((header) => { + return ( + + {header.isPlaceholder ? null : flexRender(header.column.columnDef.header, header.getContext())} + + ) + })} + + ))} + + + {table.getRowModel().rows?.length ? ( + table.getRowModel().rows.map((row) => ( + + {row.getVisibleCells().map((cell) => ( + + {flexRender(cell.column.columnDef.cell, cell.getContext())} + + ))} + + )) + ) : ( + + + No results. + + + )} + +
+
+
+
+ {table.getFilteredRowModel().rows.length} {data.length > 1 ? 'connections' : 'connection'} total +
+
+ + +
+
+
+ ) +} diff --git a/packages/web/src/app/[domain]/settings/connections/layout.tsx b/packages/web/src/app/[domain]/settings/connections/layout.tsx new file mode 100644 index 000000000..0143b4e82 --- /dev/null +++ b/packages/web/src/app/[domain]/settings/connections/layout.tsx @@ -0,0 +1,39 @@ +import { getMe } from "@/actions"; +import { getOrgFromDomain } from "@/data/org"; +import { ServiceErrorException } from "@/lib/serviceError"; +import { notFound } from "next/navigation"; +import { isServiceError } from "@/lib/utils"; +import { OrgRole } from "@sourcebot/db"; + + +interface ConnectionsLayoutProps { + children: React.ReactNode; + params: Promise<{ + domain: string + }>; +} + +export default async function ConnectionsLayout({ children, params }: ConnectionsLayoutProps) { + const { domain } = await params; + + const org = await getOrgFromDomain(domain); + if (!org) { + throw new Error("Organization not found"); + } + + const me = await getMe(); + if (isServiceError(me)) { + throw new ServiceErrorException(me); + } + + const userRoleInOrg = me.memberships.find((membership) => membership.id === org.id)?.role; + if (!userRoleInOrg) { + throw new Error("User role not found"); + } + + if (userRoleInOrg !== OrgRole.OWNER) { + return notFound(); + } + + return children; +} \ No newline at end of file diff --git a/packages/web/src/app/[domain]/settings/connections/page.tsx b/packages/web/src/app/[domain]/settings/connections/page.tsx new file mode 100644 index 000000000..ccac5284c --- /dev/null +++ b/packages/web/src/app/[domain]/settings/connections/page.tsx @@ -0,0 +1,77 @@ +import { sew } from "@/actions"; +import { ServiceErrorException } from "@/lib/serviceError"; +import { CodeHostType, isServiceError } from "@/lib/utils"; +import { withAuthV2 } from "@/withAuthV2"; +import Link from "next/link"; +import { ConnectionsTable } from "./components/connectionsTable"; +import { ConnectionSyncJobStatus } from "@prisma/client"; + +const DOCS_URL = "https://docs.sourcebot.dev/docs/connections/overview"; + +export default async function ConnectionsPage() { + const _connections = await getConnectionsWithLatestJob(); + if (isServiceError(_connections)) { + throw new ServiceErrorException(_connections); + } + + // Sort connections so that first time syncs are at the top. + const connections = _connections + .map((connection) => ({ + ...connection, + isFirstTimeSync: connection.syncedAt === null && connection.syncJobs.filter((job) => job.status === ConnectionSyncJobStatus.PENDING || job.status === ConnectionSyncJobStatus.IN_PROGRESS).length > 0, + latestJobStatus: connection.syncJobs.length > 0 ? connection.syncJobs[0].status : null, + })) + .sort((a, b) => { + if (a.isFirstTimeSync && !b.isFirstTimeSync) { + return -1; + } + if (!a.isFirstTimeSync && b.isFirstTimeSync) { + return 1; + } + return a.name.localeCompare(b.name); + }); + + return ( +
+
+

Code Host Connections

+

Manage your connections to external code hosts. Learn more

+
+ ({ + id: connection.id, + name: connection.name, + codeHostType: connection.connectionType as CodeHostType, + syncedAt: connection.syncedAt, + latestJobStatus: connection.latestJobStatus, + isFirstTimeSync: connection.isFirstTimeSync, + }))} /> +
+ ) +} + +const getConnectionsWithLatestJob = async () => sew(() => + withAuthV2(async ({ prisma, org }) => { + const connections = await prisma.connection.findMany({ + where: { + orgId: org.id, + }, + include: { + _count: { + select: { + syncJobs: true, + } + }, + syncJobs: { + orderBy: { + createdAt: 'desc' + }, + take: 1 + }, + }, + orderBy: { + name: 'asc' + }, + }); + + return connections; + })); \ No newline at end of file diff --git a/packages/web/src/app/[domain]/settings/layout.tsx b/packages/web/src/app/[domain]/settings/layout.tsx index 7259746d4..8367a4ba5 100644 --- a/packages/web/src/app/[domain]/settings/layout.tsx +++ b/packages/web/src/app/[domain]/settings/layout.tsx @@ -1,13 +1,12 @@ import React from "react" import { Metadata } from "next" -import { SidebarNav } from "./components/sidebar-nav" +import { SidebarNav, SidebarNavItem } from "./components/sidebar-nav" import { NavigationMenu } from "../components/navigationMenu" -import { Header } from "./components/header"; import { IS_BILLING_ENABLED } from "@/ee/features/billing/stripe"; import { redirect } from "next/navigation"; import { auth } from "@/auth"; import { isServiceError } from "@/lib/utils"; -import { getMe, getOrgAccountRequests } from "@/actions"; +import { getConnectionStats, getMe, getOrgAccountRequests } from "@/actions"; import { ServiceErrorException } from "@/lib/serviceError"; import { getOrgFromDomain } from "@/data/org"; import { OrgRole } from "@prisma/client"; @@ -64,7 +63,12 @@ export default async function SettingsLayout( numJoinRequests = requests.length; } - const sidebarNavItems = [ + const connectionStats = await getConnectionStats(); + if (isServiceError(connectionStats)) { + throw new ServiceErrorException(connectionStats); + } + + const sidebarNavItems: SidebarNavItem[] = [ { title: "General", href: `/${domain}/settings`, @@ -94,6 +98,14 @@ export default async function SettingsLayout( ), href: `/${domain}/settings/members`, }] : []), + ...(userRoleInOrg === OrgRole.OWNER ? [ + { + title: "Connections", + href: `/${domain}/settings/connections`, + hrefRegex: `/${domain}/settings/connections(/[^/]+)?$`, + isNotificationDotVisible: connectionStats.numberOfConnectionsWithFirstTimeSyncJobsInProgress > 0, + } + ] : []), { title: "Secrets", href: `/${domain}/settings/secrets`, @@ -115,21 +127,24 @@ export default async function SettingsLayout( ] return ( -
+
-
-
-
-

Settings

-
-
- -
{children}
+
+
+
+
+

Settings

+
+
+ +
{children}
+
-
+
) -} \ No newline at end of file +} + diff --git a/packages/web/src/app/[domain]/settings/secrets/components/importSecretCard.tsx b/packages/web/src/app/[domain]/settings/secrets/components/importSecretCard.tsx index 0972e8611..f92e27125 100644 --- a/packages/web/src/app/[domain]/settings/secrets/components/importSecretCard.tsx +++ b/packages/web/src/app/[domain]/settings/secrets/components/importSecretCard.tsx @@ -27,7 +27,7 @@ export const ImportSecretCard = ({ className }: ImportSecretCardProps) => { { setSelectedCodeHost("github"); setIsImportSecretDialogOpen(true); @@ -35,7 +35,7 @@ export const ImportSecretCard = ({ className }: ImportSecretCardProps) => { /> { setSelectedCodeHost("gitlab"); setIsImportSecretDialogOpen(true); @@ -43,7 +43,7 @@ export const ImportSecretCard = ({ className }: ImportSecretCardProps) => { /> { setSelectedCodeHost("gitea"); setIsImportSecretDialogOpen(true); diff --git a/packages/web/src/app/api/(server)/stripe/route.ts b/packages/web/src/app/api/(server)/stripe/route.ts index 1150e0d19..84b671141 100644 --- a/packages/web/src/app/api/(server)/stripe/route.ts +++ b/packages/web/src/app/api/(server)/stripe/route.ts @@ -2,7 +2,7 @@ import { headers } from 'next/headers'; import { NextRequest } from 'next/server'; import Stripe from 'stripe'; import { prisma } from '@/prisma'; -import { ConnectionSyncStatus, StripeSubscriptionStatus } from '@sourcebot/db'; +import { StripeSubscriptionStatus } from '@sourcebot/db'; import { stripeClient } from '@/ee/features/billing/stripe'; import { env } from '@/env.mjs'; import { createLogger } from "@sourcebot/logger"; @@ -85,16 +85,6 @@ export async function POST(req: NextRequest) { }); logger.info(`Org ${org.id} subscription status updated to ACTIVE`); - // mark all of this org's connections for sync, since their repos may have been previously garbage collected - await prisma.connection.updateMany({ - where: { - orgId: org.id - }, - data: { - syncStatus: ConnectionSyncStatus.SYNC_NEEDED - } - }); - return new Response(JSON.stringify({ received: true }), { status: 200 }); diff --git a/packages/web/src/features/chat/components/searchScopeIcon.tsx b/packages/web/src/features/chat/components/searchScopeIcon.tsx index 933471f4e..67170dca8 100644 --- a/packages/web/src/features/chat/components/searchScopeIcon.tsx +++ b/packages/web/src/features/chat/components/searchScopeIcon.tsx @@ -1,5 +1,5 @@ -import { cn, getCodeHostIcon } from "@/lib/utils"; -import { FolderIcon, LibraryBigIcon } from "lucide-react"; +import { cn, CodeHostType, getCodeHostIcon } from "@/lib/utils"; +import { LibraryBigIcon } from "lucide-react"; import Image from "next/image"; import { SearchScope } from "../types"; @@ -13,20 +13,16 @@ export const SearchScopeIcon = ({ searchScope, className = "h-4 w-4" }: SearchSc return ; } else { // Render code host icon for repos - const codeHostIcon = searchScope.codeHostType ? getCodeHostIcon(searchScope.codeHostType) : null; - if (codeHostIcon) { - const size = className.includes('h-3') ? 12 : 16; - return ( - {`${searchScope.codeHostType} - ); - } else { - return ; - } + const codeHostIcon = getCodeHostIcon(searchScope.codeHostType as CodeHostType); + const size = className.includes('h-3') ? 12 : 16; + return ( + {`${searchScope.codeHostType} + ); } }; \ No newline at end of file diff --git a/packages/web/src/initialize.ts b/packages/web/src/initialize.ts index 1063e58c7..63eb6a473 100644 --- a/packages/web/src/initialize.ts +++ b/packages/web/src/initialize.ts @@ -1,129 +1,17 @@ -import { ConnectionSyncStatus, OrgRole, Prisma } from '@sourcebot/db'; -import { env } from './env.mjs'; -import { prisma } from "@/prisma"; -import { SINGLE_TENANT_ORG_ID, SINGLE_TENANT_ORG_DOMAIN, SOURCEBOT_GUEST_USER_ID, SINGLE_TENANT_ORG_NAME } from './lib/constants'; -import chokidar from 'chokidar'; -import { ConnectionConfig } from '@sourcebot/schemas/v3/connection.type'; -import { hasEntitlement, loadConfig, isRemotePath, syncSearchContexts } from '@sourcebot/shared'; -import { isServiceError, getOrgMetadata } from './lib/utils'; -import { ServiceErrorException } from './lib/serviceError'; +import { createGuestUser } from '@/lib/authUtils'; import { SOURCEBOT_SUPPORT_EMAIL } from "@/lib/constants"; +import { prisma } from "@/prisma"; +import { OrgRole } from '@sourcebot/db'; import { createLogger } from "@sourcebot/logger"; -import { createGuestUser } from '@/lib/authUtils'; +import { hasEntitlement, loadConfig } from '@sourcebot/shared'; import { getOrgFromDomain } from './data/org'; +import { env } from './env.mjs'; +import { SINGLE_TENANT_ORG_DOMAIN, SINGLE_TENANT_ORG_ID, SOURCEBOT_GUEST_USER_ID } from './lib/constants'; +import { ServiceErrorException } from './lib/serviceError'; +import { getOrgMetadata, isServiceError } from './lib/utils'; const logger = createLogger('web-initialize'); -const syncConnections = async (connections?: { [key: string]: ConnectionConfig }) => { - if (connections) { - for (const [key, newConnectionConfig] of Object.entries(connections)) { - const currentConnection = await prisma.connection.findUnique({ - where: { - name_orgId: { - name: key, - orgId: SINGLE_TENANT_ORG_ID, - } - }, - include: { - repos: { - include: { - repo: true, - } - } - } - }); - - const currentConnectionConfig = currentConnection ? currentConnection.config as unknown as ConnectionConfig : undefined; - const syncNeededOnUpdate = - (currentConnectionConfig && JSON.stringify(currentConnectionConfig) !== JSON.stringify(newConnectionConfig)) || - (currentConnection?.syncStatus === ConnectionSyncStatus.FAILED); - - const connectionDb = await prisma.connection.upsert({ - where: { - name_orgId: { - name: key, - orgId: SINGLE_TENANT_ORG_ID, - } - }, - update: { - config: newConnectionConfig as unknown as Prisma.InputJsonValue, - syncStatus: syncNeededOnUpdate ? ConnectionSyncStatus.SYNC_NEEDED : undefined, - isDeclarative: true, - }, - create: { - name: key, - connectionType: newConnectionConfig.type, - config: newConnectionConfig as unknown as Prisma.InputJsonValue, - isDeclarative: true, - org: { - connect: { - id: SINGLE_TENANT_ORG_ID, - } - } - } - }); - - logger.info(`Upserted connection with name '${key}'. Connection ID: ${connectionDb.id}`); - } - } - - // Delete any connections that are no longer in the config. - const deletedConnections = await prisma.connection.findMany({ - where: { - isDeclarative: true, - name: { - notIn: Object.keys(connections ?? {}), - }, - orgId: SINGLE_TENANT_ORG_ID, - } - }); - - for (const connection of deletedConnections) { - logger.info(`Deleting connection with name '${connection.name}'. Connection ID: ${connection.id}`); - await prisma.connection.delete({ - where: { - id: connection.id, - } - }) - } -} - -const syncDeclarativeConfig = async (configPath: string) => { - const config = await loadConfig(configPath); - - const forceEnableAnonymousAccess = config.settings?.enablePublicAccess ?? env.FORCE_ENABLE_ANONYMOUS_ACCESS === 'true'; - if (forceEnableAnonymousAccess) { - const hasAnonymousAccessEntitlement = hasEntitlement("anonymous-access"); - if (!hasAnonymousAccessEntitlement) { - logger.warn(`FORCE_ENABLE_ANONYMOUS_ACCESS env var is set to true but anonymous access entitlement is not available. Setting will be ignored.`); - } else { - const org = await getOrgFromDomain(SINGLE_TENANT_ORG_DOMAIN); - if (org) { - const currentMetadata = getOrgMetadata(org); - const mergedMetadata = { - ...(currentMetadata ?? {}), - anonymousAccessEnabled: true, - }; - - await prisma.org.update({ - where: { id: org.id }, - data: { - metadata: mergedMetadata, - }, - }); - logger.info(`Anonymous access enabled via FORCE_ENABLE_ANONYMOUS_ACCESS environment variable`); - } - } - } - - await syncConnections(config.connections); - await syncSearchContexts({ - contexts: config.contexts, - orgId: SINGLE_TENANT_ORG_ID, - db: prisma, - }); -} - const pruneOldGuestUser = async () => { // The old guest user doesn't have the GUEST role const guestUser = await prisma.userToOrg.findUnique({ @@ -150,35 +38,6 @@ const pruneOldGuestUser = async () => { } const initSingleTenancy = async () => { - // Back fill the inviteId if the org has already been created to prevent needing to wipe the db - await prisma.$transaction(async (tx) => { - const org = await tx.org.findUnique({ - where: { - id: SINGLE_TENANT_ORG_ID, - }, - }); - - if (!org) { - await tx.org.create({ - data: { - id: SINGLE_TENANT_ORG_ID, - name: SINGLE_TENANT_ORG_NAME, - domain: SINGLE_TENANT_ORG_DOMAIN, - inviteLinkId: crypto.randomUUID(), - } - }); - } else if (!org.inviteLinkId) { - await tx.org.update({ - where: { - id: SINGLE_TENANT_ORG_ID, - }, - data: { - inviteLinkId: crypto.randomUUID(), - } - }); - } - }); - // This is needed because v4 introduces the GUEST org role as well as making authentication required. // To keep things simple, we'll just delete the old guest user if it exists in the DB await pruneOldGuestUser(); @@ -205,30 +64,32 @@ const initSingleTenancy = async () => { } } - // Load any connections defined declaratively in the config file. - const configPath = env.CONFIG_PATH; - if (configPath) { - await syncDeclarativeConfig(configPath); - - // watch for changes assuming it is a local file - if (!isRemotePath(configPath)) { - const watcher = chokidar.watch(configPath, { - ignoreInitial: true, // Don't fire events for existing files - awaitWriteFinish: { - stabilityThreshold: 100, // File size stable for 100ms - pollInterval: 100 // Check every 100ms - }, - atomic: true // Handle atomic writes (temp file + rename) - }); - - watcher.on('change', async () => { - logger.info(`Config file ${configPath} changed. Re-syncing...`); - try { - await syncDeclarativeConfig(configPath); - } catch (error) { - logger.error(`Failed to sync config: ${error}`); + // Sync anonymous access config from the config file + if (env.CONFIG_PATH) { + const config = await loadConfig(env.CONFIG_PATH); + const forceEnableAnonymousAccess = config.settings?.enablePublicAccess ?? env.FORCE_ENABLE_ANONYMOUS_ACCESS === 'true'; + + if (forceEnableAnonymousAccess) { + if (!hasAnonymousAccessEntitlement) { + logger.warn(`FORCE_ENABLE_ANONYMOUS_ACCESS env var is set to true but anonymous access entitlement is not available. Setting will be ignored.`); + } else { + const org = await getOrgFromDomain(SINGLE_TENANT_ORG_DOMAIN); + if (org) { + const currentMetadata = getOrgMetadata(org); + const mergedMetadata = { + ...(currentMetadata ?? {}), + anonymousAccessEnabled: true, + }; + + await prisma.org.update({ + where: { id: org.id }, + data: { + metadata: mergedMetadata, + }, + }); + logger.info(`Anonymous access enabled via FORCE_ENABLE_ANONYMOUS_ACCESS environment variable`); } - }); + } } } } diff --git a/packages/web/src/lib/syncStatusMetadataSchema.ts b/packages/web/src/lib/syncStatusMetadataSchema.ts deleted file mode 100644 index f9855db9d..000000000 --- a/packages/web/src/lib/syncStatusMetadataSchema.ts +++ /dev/null @@ -1,17 +0,0 @@ -import { z } from "zod"; - -export const NotFoundSchema = z.object({ - users: z.array(z.string()), - orgs: z.array(z.string()), - repos: z.array(z.string()), -}); - -export const SyncStatusMetadataSchema = z.object({ - notFound: NotFoundSchema.optional(), - error: z.string().optional(), - secretKey: z.string().optional(), - status: z.number().optional(), -}); - -export type NotFoundData = z.infer; -export type SyncStatusMetadata = z.infer; \ No newline at end of file diff --git a/packages/web/src/lib/utils.ts b/packages/web/src/lib/utils.ts index f25206a52..bc4850f0c 100644 --- a/packages/web/src/lib/utils.ts +++ b/packages/web/src/lib/utils.ts @@ -74,7 +74,7 @@ export type CodeHostType = "azuredevops" | "generic-git-host"; -export type AuthProviderType = +export type AuthProviderType = "github" | "gitlab" | "google" | @@ -105,7 +105,7 @@ export const getAuthProviderInfo = (providerId: string): AuthProviderInfo => { }; case "gitlab": return { - id: "gitlab", + id: "gitlab", name: "GitLab", displayName: "GitLab", icon: { @@ -115,7 +115,7 @@ export const getAuthProviderInfo = (providerId: string): AuthProviderInfo => { case "google": return { id: "google", - name: "Google", + name: "Google", displayName: "Google", icon: { src: googleLogo, @@ -125,7 +125,7 @@ export const getAuthProviderInfo = (providerId: string): AuthProviderInfo => { return { id: "okta", name: "Okta", - displayName: "Okta", + displayName: "Okta", icon: { src: oktaLogo, className: "dark:invert", @@ -145,7 +145,7 @@ export const getAuthProviderInfo = (providerId: string): AuthProviderInfo => { id: "microsoft-entra-id", name: "Microsoft Entra ID", displayName: "Microsoft Entra ID", - icon: { + icon: { src: microsoftLogo, }, }; @@ -283,7 +283,7 @@ export const getCodeHostInfoForRepo = (repo: { } } -export const getCodeHostIcon = (codeHostType: string): { src: string, className?: string } | null => { +export const getCodeHostIcon = (codeHostType: CodeHostType): { src: string, className?: string } => { switch (codeHostType) { case "github": return { @@ -315,8 +315,6 @@ export const getCodeHostIcon = (codeHostType: string): { src: string, className? return { src: gitLogo, } - default: - return null; } } @@ -364,7 +362,7 @@ export const getCodeHostBrowseAtBranchUrl = ({ if (!webUrl) { return undefined; } - + switch (codeHostType) { case 'github': return `${webUrl}/tree/${branchName}`; @@ -416,7 +414,7 @@ export const getFormattedDate = (date: Date) => { const now = new Date(); const diffMinutes = (now.getTime() - date.getTime()) / (1000 * 60); const isFuture = diffMinutes < 0; - + // Use absolute values for calculations const minutes = Math.abs(diffMinutes); const hours = minutes / 60; @@ -426,7 +424,7 @@ export const getFormattedDate = (date: Date) => { const formatTime = (value: number, unit: 'minute' | 'hour' | 'day' | 'month', isFuture: boolean) => { const roundedValue = Math.floor(value); const pluralUnit = roundedValue === 1 ? unit : `${unit}s`; - + if (isFuture) { return `In ${roundedValue} ${pluralUnit}`; } else { @@ -508,7 +506,7 @@ export const measure = async (cb: () => Promise, measureName: string, outp * @param promise The promise to unwrap. * @returns The data from the promise. */ -export const unwrapServiceError = async (promise: Promise): Promise => { +export const unwrapServiceError = async (promise: Promise): Promise => { const data = await promise; if (isServiceError(data)) { throw new Error(data.message); @@ -531,10 +529,10 @@ export const requiredQueryParamGuard = (request: NextRequest, param: string): Se export const getRepoImageSrc = (imageUrl: string | undefined, repoId: number): string | undefined => { if (!imageUrl) return undefined; - + try { const url = new URL(imageUrl); - + // List of known public instances that don't require authentication const publicHostnames = [ 'github.com', @@ -542,9 +540,9 @@ export const getRepoImageSrc = (imageUrl: string | undefined, repoId: number): s 'gitea.com', 'bitbucket.org', ]; - + const isPublicInstance = publicHostnames.includes(url.hostname); - + if (isPublicInstance) { return imageUrl; } else { @@ -566,8 +564,8 @@ export const IS_MAC = typeof navigator !== 'undefined' && /Mac OS X/.test(naviga export const isHttpError = (error: unknown, status: number): boolean => { - return error !== null + return error !== null && typeof error === 'object' - && 'status' in error + && 'status' in error && error.status === status; } \ No newline at end of file diff --git a/schemas/v3/app.json b/schemas/v3/app.json index 007033269..2fadfef4a 100644 --- a/schemas/v3/app.json +++ b/schemas/v3/app.json @@ -1,9 +1,44 @@ { "$schema": "http://json-schema.org/draft-07/schema#", "title": "AppConfig", + "definitions": { + "GitHubAppConfig": { + "type": "object", + "properties": { + "type": { + "const": "githubApp", + "description": "GitHub App Configuration" + }, + "deploymentHostname": { + "type": "string", + "format": "hostname", + "default": "github.com", + "description": "The hostname of the GitHub App deployment.", + "examples": [ + "github.com", + "github.example.com" + ] + }, + "id": { + "type": "string", + "description": "The ID of the GitHub App." + }, + "privateKey": { + "$ref": "./shared.json#/definitions/Token", + "description": "The private key of the GitHub App." + } + }, + "required": [ + "type", + "id", + "privateKey" + ], + "additionalProperties": false + } + }, "oneOf": [ { - "$ref": "./githubApp.json" + "$ref": "#/definitions/GitHubAppConfig" } ] } \ No newline at end of file diff --git a/schemas/v3/githubApp.json b/schemas/v3/githubApp.json deleted file mode 100644 index c83553ced..000000000 --- a/schemas/v3/githubApp.json +++ /dev/null @@ -1,42 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "title": "GithubAppConfig", - "properties": { - "type": { - "const": "githubApp", - "description": "GitHub App Configuration" - }, - "deploymentHostname": { - "type": "string", - "format": "hostname", - "default": "github.com", - "description": "The hostname of the GitHub App deployment.", - "examples": [ - "github.com", - "github.example.com" - ] - }, - "id": { - "type": "string", - "description": "The ID of the GitHub App." - }, - "privateKey": { - "$ref": "./shared.json#/definitions/Token", - "description": "The private key of the GitHub App." - } - }, - "required": [ - "type", - "id" - ], - "oneOf": [ - { - "required": ["privateKey"] - }, - { - "required": ["privateKeyPath"] - } - ], - "additionalProperties": false -} \ No newline at end of file diff --git a/yarn.lock b/yarn.lock index 34e6c1b22..44ae05584 100644 --- a/yarn.lock +++ b/yarn.lock @@ -7796,6 +7796,7 @@ __metadata: argparse: "npm:^2.0.1" azure-devops-node-api: "npm:^15.1.1" bullmq: "npm:^5.34.10" + chokidar: "npm:^4.0.3" cross-env: "npm:^7.0.3" cross-fetch: "npm:^4.0.0" dotenv: "npm:^16.4.5" @@ -8055,7 +8056,6 @@ __metadata: ai: "npm:^5.0.45" ajv: "npm:^8.17.1" bcryptjs: "npm:^3.0.2" - chokidar: "npm:^4.0.3" class-variance-authority: "npm:^0.7.0" client-only: "npm:^0.0.1" clsx: "npm:^2.1.1"