From b800b74925a17bb5f639fd8fb2417e5026b89cfc Mon Sep 17 00:00:00 2001 From: dingzhenznen <1324652616@qq.com> Date: Mon, 4 Aug 2025 10:31:47 +0800 Subject: [PATCH 1/4] fix:add s3 proxy when env config --- bun.lock | 4 +++- package.json | 4 +++- src/s3/config.ts | 14 +++++++++++++- src/s3/controller.ts | 3 ++- 4 files changed, 21 insertions(+), 4 deletions(-) diff --git a/bun.lock b/bun.lock index c50936bf..e22e75a6 100644 --- a/bun.lock +++ b/bun.lock @@ -23,6 +23,8 @@ "comlink": "^4.4.2", "date-fns": "^4.1.0", "express": "^5.1.0", + "http-proxy-agent": "^7.0.2", + "https-proxy-agent": "^7.0.6", "json5": "^2.2.3", "minio": "^8.0.5", "nanoid": "^5.1.5", @@ -429,7 +431,7 @@ }, "sdk": { "name": "@fastgpt-sdk/plugin", - "version": "0.1.3", + "version": "0.1.7", "dependencies": { "@fortaine/fetch-event-source": "^3.0.6", "@ts-rest/core": "^3.52.1", diff --git a/package.json b/package.json index ea0e958f..d78c5668 100644 --- a/package.json +++ b/package.json @@ -36,6 +36,8 @@ "comlink": "^4.4.2", "date-fns": "^4.1.0", "express": "^5.1.0", + "http-proxy-agent": "^7.0.2", + "https-proxy-agent": "^7.0.6", "json5": "^2.2.3", "minio": "^8.0.5", "nanoid": "^5.1.5", @@ -73,4 +75,4 @@ "npm run prettier" ] } -} +} \ No newline at end of file diff --git a/src/s3/config.ts b/src/s3/config.ts index 7e02b82f..b2905165 100644 --- a/src/s3/config.ts +++ b/src/s3/config.ts @@ -1,5 +1,7 @@ import { z } from 'zod'; import { S3Service } from './controller'; +import { HttpsProxyAgent } from 'https-proxy-agent'; +import { HttpProxyAgent } from 'http-proxy-agent'; export type FileConfig = { maxFileSize: number; // 文件大小限制（字节） @@ -10,8 +12,17 @@ export type FileConfig = { accessKey: string; // MinIO access key secretKey: string; // MinIO secret key bucket: string; // 存储桶名称 + transportAgent: any; // 代理 }; +let transportAgent; +if (process.env.HTTP_PROXY) { + transportAgent = + process.env.MINIO_USE_SSL === 'true' + ? new HttpsProxyAgent(process.env.HTTP_PROXY) + : new HttpProxyAgent(process.env.HTTP_PROXY); +} + // 默认配置（动态从环境变量读取） export const defaultFileConfig: FileConfig = { maxFileSize: process.env.MAX_FILE_SIZE ? parseInt(process.env.MAX_FILE_SIZE) : 20 * 1024 * 1024, // 默认 20MB @@ -21,7 +32,8 @@ export const defaultFileConfig: FileConfig = { useSSL: process.env.MINIO_USE_SSL === 'true', accessKey: process.env.MINIO_ACCESS_KEY || 'minioadmin', secretKey: process.env.MINIO_SECRET_KEY || 'minioadmin', - bucket: process.env.MINIO_BUCKET || 'files' + bucket: process.env.MINIO_BUCKET || 'files', + transportAgent: transportAgent }; export const FileMetadataSchema = z.object({ diff --git a/src/s3/controller.ts b/src/s3/controller.ts index b275b259..defadb94 100644 --- a/src/s3/controller.ts +++ b/src/s3/controller.ts @@ -52,7 +52,8 @@ export class S3Service { port: this.config.port, useSSL: this.config.useSSL, accessKey: this.config.accessKey, - secretKey: this.config.secretKey + secretKey: this.config.secretKey, + transportAgent: this.config.transportAgent }); } From b25119537ab3651a8f052343a6ae2dee0a2837d3 Mon Sep 17 00:00:00 2001 From: dingzhenznen <1324652616@qq.com> Date: Thu, 7 Aug 2025 14:40:56 +0800 Subject: [PATCH 2/4] fix: add process.env and setup fetch proxy --- modules/tool/utils/setupProxy.ts | 6 ++++++ package.json | 2 +- scripts/build.ts | 4 ++-- scripts/plugin.ts | 21 +++++++++++++++++++++ 4 files changed, 30 insertions(+), 3 deletions(-) create mode 100644 modules/tool/utils/setupProxy.ts diff --git a/modules/tool/utils/setupProxy.ts b/modules/tool/utils/setupProxy.ts new file mode 100644 index 00000000..e73bc45e --- /dev/null +++ b/modules/tool/utils/setupProxy.ts @@ -0,0 +1,6 @@ +// setup fetch Proxy +import { ProxyAgent, setGlobalDispatcher } from 'undici'; +if (process.env.HTTP_PROXY) { + const httpDispatcher = new ProxyAgent(process.env.HTTP_PROXY as string); + setGlobalDispatcher(httpDispatcher); +} diff --git a/package.json b/package.json index d78c5668..7d7ead35 100644 --- a/package.json +++ b/package.json @@ -5,7 +5,7 @@ "type": "module", "scripts": { "prepare": "husky install", - "build": "tsc && bun ./scripts/build.ts", + "build": "tsc && bun ./scripts/build.ts --env inline", "build:no-tsc": "bun ./scripts/build.ts", "build:main": "bun build --env=disable --outfile=dist/index.js --target=node --minify ./src/index.ts", "build:worker": "bun build --env=disable --outfile=dist/worker.js --target=node --minify ./src/worker/worker.ts", diff --git a/scripts/build.ts b/scripts/build.ts index 5f92ff7d..65fedaae 100644 --- a/scripts/build.ts +++ b/scripts/build.ts @@ -3,7 +3,7 @@ import { $ } from 'bun'; import fs from 'fs'; import path from 'path'; import { copyToolIcons } from '../modules/tool/utils/icon'; -import { autoToolIdPlugin } from './plugin'; +import { autoToolIdPlugin, setupFetchPlugin } from './plugin'; import { exit } from 'process'; const toolsDir = path.join(__dirname, '..', 'modules', 'tool', 'packages'); @@ -17,7 +17,7 @@ export const buildATool = async (tool: string, dist: string = distToolDir) => { outdir: dist, naming: tool + '.js', target: 'node', - plugins: [autoToolIdPlugin], + plugins: [autoToolIdPlugin, setupFetchPlugin], minify: true }); }; diff --git a/scripts/plugin.ts b/scripts/plugin.ts index 1eb168ba..1025ec3c 100644 --- a/scripts/plugin.ts +++ b/scripts/plugin.ts @@ -159,3 +159,24 @@ export const autoToolIdPlugin: BunPlugin = { }, target: 'node' }; + +export const setupFetchPlugin: BunPlugin = { + name: 'import-fetch', + setup(build) { + build.onLoad( + { + filter: /packages\/.+\/src\/index\.ts/ + }, + async (args) => { + const content = await Bun.file(args.path).text(); + if (content.includes(' fetch(')) { + return { + contents: `import '@tool/utils/setupProxy'\n${content}`, + loader: 'ts' + }; + } + } + ); + }, + target: 'node' +}; From 0e48a5d7d6a040b38502c80dc1f717f2b3517c8d Mon Sep 17 00:00:00 2001 From: FinleyGe Date: Fri, 15 Aug 2025 12:05:02 +0800 Subject: [PATCH 3/4] chore: adjust the code --- bun.lock | 5 ++++- modules/tool/utils/setupProxy.ts | 6 ------ package.json | 5 +++-- scripts/build.ts | 4 ++-- scripts/plugin.ts | 21 --------------------- src/index.ts | 2 ++ src/s3/config.ts | 14 +------------- src/s3/controller.ts | 8 +++++++- src/utils/setupProxy.ts | 11 +++++++++++ src/worker/index.ts | 5 ++++- src/worker/worker.ts | 2 ++ 11 files changed, 36 insertions(+), 47 deletions(-) delete mode 100644 modules/tool/utils/setupProxy.ts create mode 100644 src/utils/setupProxy.ts diff --git a/bun.lock b/bun.lock index e22e75a6..682874ea 100644 --- a/bun.lock +++ b/bun.lock @@ -28,6 +28,7 @@ "json5": "^2.2.3", "minio": "^8.0.5", "nanoid": "^5.1.5", + "undici": "^7.13.0", "uuid": "^11.1.0", "zod": "^3.24.3", }, @@ -1893,7 +1894,7 @@ "ufo": ["ufo@1.6.1", "", {}, "sha512-9a4/uxlTWJ4+a5i0ooc1rU7C7YOw3wT+UGqdeNNHWnOF9qcMBgLRS+4IYUqbczewFx4mLEig6gawh7X6mFlEkA=="], - "undici": ["undici@7.12.0", "", {}, "sha512-GrKEsc3ughskmGA9jevVlIOPMiiAHJ4OFUtaAH+NhfTUSiZ1wMPIQqQvAJUrJspFXJt3EBWgpAeoHEDVT1IBug=="], + "undici": ["undici@7.13.0", "", {}, "sha512-l+zSMssRqrzDcb3fjMkjjLGmuiiK2pMIcV++mJaAc9vhjSGpvM7h43QgP+OAMb1GImHmbPyG2tBXeuyG5iY4gA=="], "undici-types": ["undici-types@7.8.0", "", {}, "sha512-9UJ2xGDvQ43tYyVMpuHlsgApydB8ZKfVYTsLDhXkFL/6gfkp+U8xTGdh8pMJv1SpZna0zxG1DwsKZsreLbXBxw=="], @@ -2069,6 +2070,8 @@ "bl/readable-stream": ["readable-stream@4.7.0", "", { "dependencies": { "abort-controller": "^3.0.0", "buffer": "^6.0.3", "events": "^3.3.0", "process": "^0.11.10", "string_decoder": "^1.3.0" } }, "sha512-oIGGmcpTLwPga8Bn6/Z75SVaH1z5dUut2ibSyAMVhmUggWpmDn2dapB0n7f8nwaSiRtepAsfJyfXIO5DCVAODg=="], + "cheerio/undici": ["undici@7.12.0", "", {}, "sha512-GrKEsc3ughskmGA9jevVlIOPMiiAHJ4OFUtaAH+NhfTUSiZ1wMPIQqQvAJUrJspFXJt3EBWgpAeoHEDVT1IBug=="], + "cli-truncate/string-width": ["string-width@5.1.2", "", { "dependencies": { "eastasianwidth": "^0.2.0", "emoji-regex": "^9.2.2", "strip-ansi": "^7.0.1" } }, "sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA=="], "cliui/strip-ansi": ["strip-ansi@6.0.1", "", { "dependencies": { "ansi-regex": "^5.0.1" } }, "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A=="], diff --git a/modules/tool/utils/setupProxy.ts b/modules/tool/utils/setupProxy.ts deleted file mode 100644 index e73bc45e..00000000 --- a/modules/tool/utils/setupProxy.ts +++ /dev/null @@ -1,6 +0,0 @@ -// setup fetch Proxy -import { ProxyAgent, setGlobalDispatcher } from 'undici'; -if (process.env.HTTP_PROXY) { - const httpDispatcher = new ProxyAgent(process.env.HTTP_PROXY as string); - setGlobalDispatcher(httpDispatcher); -} diff --git a/package.json b/package.json index 7d7ead35..0d283809 100644 --- a/package.json +++ b/package.json @@ -5,7 +5,7 @@ "type": "module", "scripts": { "prepare": "husky install", - "build": "tsc && bun ./scripts/build.ts --env inline", + "build": "tsc && bun ./scripts/build.ts", "build:no-tsc": "bun ./scripts/build.ts", "build:main": "bun build --env=disable --outfile=dist/index.js --target=node --minify ./src/index.ts", "build:worker": "bun build --env=disable --outfile=dist/worker.js --target=node --minify ./src/worker/worker.ts", @@ -41,6 +41,7 @@ "json5": "^2.2.3", "minio": "^8.0.5", "nanoid": "^5.1.5", + "undici": "^7.13.0", "uuid": "^11.1.0", "zod": "^3.24.3" }, @@ -75,4 +76,4 @@ "npm run prettier" ] } -} \ No newline at end of file +} diff --git a/scripts/build.ts b/scripts/build.ts index 65fedaae..5f92ff7d 100644 --- a/scripts/build.ts +++ b/scripts/build.ts @@ -3,7 +3,7 @@ import { $ } from 'bun'; import fs from 'fs'; import path from 'path'; import { copyToolIcons } from '../modules/tool/utils/icon'; -import { autoToolIdPlugin, setupFetchPlugin } from './plugin'; +import { autoToolIdPlugin } from './plugin'; import { exit } from 'process'; const toolsDir = path.join(__dirname, '..', 'modules', 'tool', 'packages'); @@ -17,7 +17,7 @@ export const buildATool = async (tool: string, dist: string = distToolDir) => { outdir: dist, naming: tool + '.js', target: 'node', - plugins: [autoToolIdPlugin, setupFetchPlugin], + plugins: [autoToolIdPlugin], minify: true }); }; diff --git a/scripts/plugin.ts b/scripts/plugin.ts index 1025ec3c..1eb168ba 100644 --- a/scripts/plugin.ts +++ b/scripts/plugin.ts @@ -159,24 +159,3 @@ export const autoToolIdPlugin: BunPlugin = { }, target: 'node' }; - -export const setupFetchPlugin: BunPlugin = { - name: 'import-fetch', - setup(build) { - build.onLoad( - { - filter: /packages\/.+\/src\/index\.ts/ - }, - async (args) => { - const content = await Bun.file(args.path).text(); - if (content.includes(' fetch(')) { - return { - contents: `import '@tool/utils/setupProxy'\n${content}`, - loader: 'ts' - }; - } - } - ); - }, - target: 'node' -}; diff --git a/src/index.ts b/src/index.ts index 4b366e4b..778552cf 100644 --- a/src/index.ts +++ b/src/index.ts @@ -7,6 +7,7 @@ import { isProd } from './constants'; import { initS3Server } from './s3/config'; import { connectSignoz } from './utils/signoz'; import { initModels } from '@model/init'; +import { setupProxy } from './utils/setupProxy'; const app = express().use( express.json(), @@ -19,6 +20,7 @@ connectSignoz(); // System initOpenAPI(app); initRouter(app); +setupProxy(); // DB try { diff --git a/src/s3/config.ts b/src/s3/config.ts index b2905165..7e02b82f 100644 --- a/src/s3/config.ts +++ b/src/s3/config.ts @@ -1,7 +1,5 @@ import { z } from 'zod'; import { S3Service } from './controller'; -import { HttpsProxyAgent } from 'https-proxy-agent'; -import { HttpProxyAgent } from 'http-proxy-agent'; export type FileConfig = { maxFileSize: number; // 文件大小限制（字节） @@ -12,17 +10,8 @@ export type FileConfig = { accessKey: string; // MinIO access key secretKey: string; // MinIO secret key bucket: string; // 存储桶名称 - transportAgent: any; // 代理 }; -let transportAgent; -if (process.env.HTTP_PROXY) { - transportAgent = - process.env.MINIO_USE_SSL === 'true' - ? new HttpsProxyAgent(process.env.HTTP_PROXY) - : new HttpProxyAgent(process.env.HTTP_PROXY); -} - // 默认配置（动态从环境变量读取） export const defaultFileConfig: FileConfig = { maxFileSize: process.env.MAX_FILE_SIZE ? parseInt(process.env.MAX_FILE_SIZE) : 20 * 1024 * 1024, // 默认 20MB @@ -32,8 +21,7 @@ export const defaultFileConfig: FileConfig = { useSSL: process.env.MINIO_USE_SSL === 'true', accessKey: process.env.MINIO_ACCESS_KEY || 'minioadmin', secretKey: process.env.MINIO_SECRET_KEY || 'minioadmin', - bucket: process.env.MINIO_BUCKET || 'files', - transportAgent: transportAgent + bucket: process.env.MINIO_BUCKET || 'files' }; export const FileMetadataSchema = z.object({ diff --git a/src/s3/controller.ts b/src/s3/controller.ts index defadb94..b062304a 100644 --- a/src/s3/controller.ts +++ b/src/s3/controller.ts @@ -7,6 +7,8 @@ import { z } from 'zod'; import { addLog } from '@/utils/log'; import { getErrText } from '@tool/utils/err'; import { catchError } from '@/utils/catch'; +import { HttpProxyAgent } from 'http-proxy-agent'; +import { HttpsProxyAgent } from 'https-proxy-agent'; export const FileInputSchema = z .object({ @@ -53,7 +55,11 @@ export class S3Service { useSSL: this.config.useSSL, accessKey: this.config.accessKey, secretKey: this.config.secretKey, - transportAgent: this.config.transportAgent + transportAgent: process.env.HTTP_PROXY + ? new HttpProxyAgent(process.env.HTTP_PROXY) + : process.env.HTTPS_PROXY + ? new HttpsProxyAgent(process.env.HTTPS_PROXY) + : undefined }); } diff --git a/src/utils/setupProxy.ts b/src/utils/setupProxy.ts new file mode 100644 index 00000000..962cb656 --- /dev/null +++ b/src/utils/setupProxy.ts @@ -0,0 +1,11 @@ +import { ProxyAgent, setGlobalDispatcher } from 'undici'; +const httpProxy = process.env.HTTP_PROXY; +const httpsProxy = process.env.HTTPS_PROXY; + +export function setupProxy() { + const proxy = httpProxy || httpsProxy; + if (proxy) { + const proxyAgent = new ProxyAgent(proxy); + setGlobalDispatcher(proxyAgent); + } +} diff --git a/src/worker/index.ts b/src/worker/index.ts index 91259cb4..693e0474 100644 --- a/src/worker/index.ts +++ b/src/worker/index.ts @@ -93,7 +93,10 @@ export class WorkerPool, Response = unknown> { // Create a new worker and push it queue. const workerId = `${Date.now()}${Math.random()}`; const worker = new Worker('./worker.js', { - env: {}, + env: { + ...(process.env.HTTP_PROXY ? { HTTP_PROXY: process.env.HTTP_PROXY } : {}), + ...(process.env.HTTPS_PROXY ? { HTTPS_PROXY: process.env.HTTPS_PROXY } : {}) + }, resourceLimits: { maxOldGenerationSizeMb: parseInt(process.env.MAX_MEMORYMB || '1024') } diff --git a/src/worker/worker.ts b/src/worker/worker.ts index bbefece0..c03964d3 100644 --- a/src/worker/worker.ts +++ b/src/worker/worker.ts @@ -4,6 +4,8 @@ import { LoadToolsByFilename } from '@tool/init'; import { isProd } from '@/constants'; import { getErrText } from '@tool/utils/err'; import type { Main2WorkerMessageType } from './type'; +import { setupProxy } from '@/utils/setupProxy'; +setupProxy(); // rewrite console.log to send to parent console.log = (...args: any[]) => { From e44bbb99dd0fe01c787b2871e1497ba4559fac2a Mon Sep 17 00:00:00 2001 From: FinleyGe Date: Fri, 15 Aug 2025 15:57:15 +0800 Subject: [PATCH 4/4] fix: use undiciFetch instead of fetch --- src/utils/setupProxy.ts | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/src/utils/setupProxy.ts b/src/utils/setupProxy.ts index 962cb656..46f32af4 100644 --- a/src/utils/setupProxy.ts +++ b/src/utils/setupProxy.ts @@ -1,4 +1,6 @@ -import { ProxyAgent, setGlobalDispatcher } from 'undici'; +import { ProxyAgent, setGlobalDispatcher, fetch as undiciFetch } from 'undici'; +import { addLog } from './log'; +import { isProd } from '@/constants'; const httpProxy = process.env.HTTP_PROXY; const httpsProxy = process.env.HTTPS_PROXY; @@ -7,5 +9,15 @@ export function setupProxy() { if (proxy) { const proxyAgent = new ProxyAgent(proxy); setGlobalDispatcher(proxyAgent); + + // Replace global fetch with undici's fetch to ensure proxy is used + if (isProd) { + // Node + global.fetch = ((input: any, init: any) => { + return undiciFetch(input, init); + }) as any; + } + + addLog.info(`Using proxy: ${proxy}`); } }