Enhance file upload functionality and system tool integration

Author: ctrlz526

packages/global/core/app/plugin/type.d.ts

@@ -52,6 +52,9 @@ export type SystemPluginTemplateItemType = WorkflowTemplateType & {
   // Admin config
   inputList?: FlowNodeInputItemType['inputList'];
   hasSystemSecret?: boolean;
+
+  // Plugin source type
+  toolSource?: 'uploaded' | 'built-in';
 };
 
 export type SystemPluginTemplateListItemType = Omit<

packages/global/core/workflow/type/node.d.ts

@@ -123,6 +123,7 @@ export type NodeTemplateListItemType = {
   instructions?: string; // 使用说明
   courseUrl?: string; // 教程链接
   sourceMember?: SourceMember;
+  toolSource?: 'uploaded' | 'built-in'; // Plugin source type
 };
 
 export type NodeTemplateListType = {

packages/service/common/file/plugin/config.ts

@@ -0,0 +1,63 @@
+import { mimeTypes } from './utils';
+
+// 插件类型枚举
+export enum PluginTypeEnum {
+  tool = 'tool'
+}
+
+// 插件路径枚举
+export enum PluginPathEnum {
+  tools = 'plugin/tools'
+}
+
+// 插件类型到路径的映射
+export const PLUGIN_TYPE_TO_PATH_MAP: Record<PluginTypeEnum, PluginPathEnum> = {
+  [PluginTypeEnum.tool]: PluginPathEnum.tools
+};
+
+export type UploadFileConfig = {
+  maxFileSize: number; // 文件大小限制（字节）
+  allowedExtensions?: string[]; // 允许的文件扩展名
+  bucket: string; // 存储桶名称
+};
+
+// 默认配置
+export const defaultUploadConfig: UploadFileConfig = {
+  maxFileSize: process.env.UPLOAD_MAX_FILE_SIZE
+    ? parseInt(process.env.UPLOAD_MAX_FILE_SIZE)
+    : 100 * 1024 * 1024, // 默认 100MB
+  bucket: process.env.MINIO_UPLOAD_BUCKET || 'fastgpt-uploads',
+  allowedExtensions: Object.keys(mimeTypes) // 从 mimeTypes 映射表中获取支持的扩展名
+};
+
+export type FileMetadata = {
+  fileId: string;
+  originalFilename: string;
+  contentType: string;
+  size: number;
+  uploadTime: Date;
+  accessUrl: string;
+};
+
+// 预签名URL请求输入类型
+export type PresignedUrlInput = {
+  filename: string;
+  pluginType?: PluginTypeEnum; // 插件类型，默认为 tool
+  contentType?: string;
+  metadata?: Record<string, string>;
+  maxSize?: number;
+};
+
+// 预签名URL响应类型
+export type PresignedUrlResponse = {
+  fileId: string;
+  objectName: string;
+  uploadUrl: string;
+  formData: Record<string, string>;
+};
+
+// 保留原有类型以兼容现有代码
+export type FileUploadInput = {
+  buffer: Buffer;
+  filename: string;
+};

packages/service/common/file/plugin/controller.ts

@@ -0,0 +1,140 @@
+import { randomBytes } from 'crypto';
+import {
+  defaultUploadConfig,
+  type UploadFileConfig,
+  type PresignedUrlInput,
+  type PresignedUrlResponse,
+  PluginTypeEnum,
+  PLUGIN_TYPE_TO_PATH_MAP
+} from './config';
+import { addLog } from '../../system/log';
+import { ensureBucket } from '../../minio/init';
+import * as path from 'path';
+import { connectionMinio } from '../../minio/index';
+import { MongoFastGPTPlugin } from './schema';
+import { inferContentType } from './utils';
+
+let globalConfig: UploadFileConfig = defaultUploadConfig;
+
+export const initFileUploadService = async (config?: Partial<UploadFileConfig>) => {
+  globalConfig = { ...defaultUploadConfig, ...config };
+
+  try {
+    addLog.info(`Initializing upload bucket: ${globalConfig.bucket}`);
+    await ensureBucket(globalConfig.bucket, true);
+    addLog.info(`Upload bucket initialized successfully: ${globalConfig.bucket}`);
+    return true;
+  } catch (error) {
+    addLog.error(`Failed to initialize upload bucket: ${globalConfig.bucket}`, error);
+    throw error;
+  }
+};
+
+const generateFileId = (): string => {
+  return randomBytes(16).toString('hex');
+};
+
+export const generateDownloadUrl = (objectName: string, config: UploadFileConfig): string => {
+  const pathParts = objectName.split('/');
+  const encodedParts = pathParts.map((part) => encodeURIComponent(part));
+  const encodedObjectName = encodedParts.join('/');
+  return `${config.bucket}/${encodedObjectName}`;
+};
+
+const validatePresignedInput = (input: PresignedUrlInput, config: UploadFileConfig): void => {
+  if (!input.filename) {
+    throw new Error('Filename is required');
+  }
+
+  const maxSize = input.maxSize || config.maxFileSize;
+  if (maxSize > config.maxFileSize) {
+    throw new Error(`Max size exceeds limit of ${config.maxFileSize} bytes`);
+  }
+
+  const ext = path.extname(input.filename).toLowerCase();
+  if (config.allowedExtensions && !config.allowedExtensions.includes(ext)) {
+    throw new Error(
+      `File extension ${ext} is not allowed. Supported extensions: ${config.allowedExtensions.join(', ')}`
+    );
+  }
+};
+
+//Generate a pre-signed URL for direct file upload
+export const generatePresignedUrl = async (
+  input: PresignedUrlInput
+): Promise<PresignedUrlResponse> => {
+  const currentConfig = { ...globalConfig };
+
+  validatePresignedInput(input, currentConfig);
+
+  const fileId = generateFileId();
+  const pluginType = input.pluginType || PluginTypeEnum.tool;
+  const pluginPath = PLUGIN_TYPE_TO_PATH_MAP[pluginType];
+  const objectName = `${pluginPath}/${fileId}/${input.filename}`;
+  const contentType = input.contentType || inferContentType(input.filename);
+  const maxSize = input.maxSize || currentConfig.maxFileSize;
+
+  try {
+    const policy = connectionMinio.newPostPolicy();
+    policy.setBucket(currentConfig.bucket);
+    policy.setKey(objectName);
+
+    policy.setContentType(contentType);
+
+    policy.setContentLengthRange(1, maxSize);
+
+    policy.setExpires(new Date(Date.now() + 10 * 60 * 1000));
+
+    const metadata = {
+      'original-filename': encodeURIComponent(input.filename),
+      'upload-time': new Date().toISOString(),
+      'file-id': fileId,
+      ...input.metadata
+    };
+    policy.setUserMetaData(metadata);
+
+    const { postURL, formData } = await connectionMinio.presignedPostPolicy(policy);
+
+    const response: PresignedUrlResponse = {
+      fileId,
+      objectName,
+      uploadUrl: postURL,
+      formData
+    };
+
+    console.log('Generated presigned URL successfully', response);
+
+    return response;
+  } catch (error) {
+    addLog.error('Failed to generate presigned URL', error);
+    const errorMessage = error instanceof Error ? error.message : 'Unknown error';
+    return Promise.reject(`Failed to generate presigned URL: ${errorMessage}`);
+  }
+};
+
+//Confirm the presigned URL upload is complete and save the file information to MongoDB
+export const confirmPresignedUpload = async (objectName: string, size: string): Promise<string> => {
+  try {
+    const currentConfig = { ...globalConfig };
+    const stat = await connectionMinio.statObject(currentConfig.bucket, objectName);
+
+    if (stat.size !== Number(size)) {
+      return Promise.reject(`File size mismatch. Expected: ${size}, Actual: ${stat.size}`);
+    }
+
+    const accessUrl = generateDownloadUrl(objectName, currentConfig);
+
+    const pluginData = {
+      url: accessUrl,
+      type: 'tool' as const
+    };
+
+    const result = await MongoFastGPTPlugin.create(pluginData);
+
+    return result.url;
+  } catch (error) {
+    addLog.error('Failed to confirm presigned upload', error);
+    const errorMessage = error instanceof Error ? error.message : 'Unknown error';
+    return Promise.reject(`Failed to confirm presigned upload: ${errorMessage}`);
+  }
+};

packages/service/common/file/plugin/schema.ts

@@ -0,0 +1,31 @@
+import { Schema, getMongoModel } from '../../mongo';
+
+export type FastGPTPluginSchema = {
+  toolId?: string;
+  type: 'tool';
+  url: string;
+};
+
+const collectionName = 'fastgpt_plugins';
+
+const FastGPTPluginSchema = new Schema({
+  toolId: {
+    type: String,
+    required: false
+  },
+  type: {
+    type: String,
+    required: true,
+    enum: ['tool'],
+    default: 'tool'
+  },
+  url: {
+    type: String,
+    required: true
+  }
+});
+
+export const MongoFastGPTPlugin = getMongoModel<FastGPTPluginSchema>(
+  collectionName,
+  FastGPTPluginSchema
+);

packages/service/common/file/plugin/utils.ts

@@ -0,0 +1,10 @@
+import * as path from 'path';
+
+export const mimeTypes: Record<string, string> = {
+  '.js': 'application/javascript'
+};
+
+export const inferContentType = (filename: string): string => {
+  const ext = path.extname(filename).toLowerCase();
+  return mimeTypes[ext] || 'application/octet-stream';
+};

packages/service/common/minio/index.ts

@@ -0,0 +1,27 @@
+import { Client } from 'minio';
+
+export * from 'minio';
+export { Client };
+
+export const MINIO_ENDPOINT = process.env.MINIO_ENDPOINT || 'localhost';
+export const MINIO_PORT = process.env.MINIO_PORT ? parseInt(process.env.MINIO_PORT) : 9000;
+export const MINIO_USE_SSL = process.env.MINIO_USE_SSL === 'true';
+export const MINIO_ACCESS_KEY = process.env.MINIO_ACCESS_KEY || 'minioadmin';
+export const MINIO_SECRET_KEY = process.env.MINIO_SECRET_KEY || 'minioadmin';
+
+export const connectionMinio = (() => {
+  if (!global.minioClient) {
+    global.minioClient = new Client({
+      endPoint: MINIO_ENDPOINT,
+      port: MINIO_PORT,
+      useSSL: MINIO_USE_SSL,
+      accessKey: MINIO_ACCESS_KEY,
+      secretKey: MINIO_SECRET_KEY
+    });
+  }
+  return global.minioClient;
+})();
+
+export const getMinioClient = () => connectionMinio;
+
+export default connectionMinio;

packages/service/common/minio/init.ts

@@ -0,0 +1,64 @@
+import { connectionMinio } from './index';
+import { addLog } from '../system/log';
+import { retryFn } from '@fastgpt/global/common/system/utils';
+
+export const initMinio = async () => {
+  try {
+    addLog.info('Connecting to MinIO...');
+
+    // Test connection by listing buckets
+    await connectionMinio.listBuckets();
+
+    addLog.info('MinIO connected successfully');
+    return true;
+  } catch (error) {
+    addLog.error('Failed to connect to MinIO:', error);
+    return false;
+  }
+};
+
+export const ensureBucket = async (bucketName: string, isPublic: boolean = false) => {
+  return retryFn(async () => {
+    try {
+      const bucketExists = await connectionMinio.bucketExists(bucketName);
+
+      if (!bucketExists) {
+        addLog.info(`Creating bucket: ${bucketName}`);
+        await connectionMinio.makeBucket(bucketName);
+      }
+
+      if (isPublic) {
+        // Set public read policy
+        const policy = {
+          Version: '2012-10-17',
+          Statement: [
+            {
+              Effect: 'Allow',
+              Principal: '*',
+              Action: ['s3:GetObject'],
+              Resource: [`arn:aws:s3:::${bucketName}/*`]
+            }
+          ]
+        };
+
+        await connectionMinio.setBucketPolicy(bucketName, JSON.stringify(policy));
+        addLog.info(`Set public read policy for bucket: ${bucketName}`);
+      }
+
+      return true;
+    } catch (error) {
+      addLog.error(`Failed to ensure bucket ${bucketName}:`, error);
+      throw error;
+    }
+  }, 3);
+};
+
+export const listBuckets = async () => {
+  try {
+    const buckets = await connectionMinio.listBuckets();
+    return buckets;
+  } catch (error) {
+    addLog.error('Failed to list buckets:', error);
+    throw error;
+  }
+};

packages/service/common/minio/type.d.ts

@@ -0,0 +1,5 @@
+import type { Client } from 'minio';
+
+declare global {
+  var minioClient: Client;
+}