mine from PST & OST files (#2564)

* install pst-extractor

* mine from pst

* remove "user email" from pst minings

* use supabase storage

* create a global prettier:fix

* handle mining completion

* prettier

* add indeterminate progress bar

* publishFetchingProgress and stop on mining finish

* Update 20251218184211_create_pst_bucket.sql

5GB maximum file

* add file upload progress bar and start next step

* mining type; pst

* getTotalMessages & await processFolder

* Pre-scan PST to compute total messages before processing

* Update PSTEmailsFetcher.ts

* createSignedUploadUrl

* i18n & typing fixes

* delete file after mining

* create scheduled job to remove PST files

* linting fixes

* linting fixes

* linting fixes

Author: J43fura

backend/package-lock.json

@@ -60,6 +60,7 @@
     "pg": "^8.16.3",
     "pg-format": "^1.0.4",
     "planer": "^1.2.0",
+    "pst-extractor": "^1.11.0",
     "qs": "^6.14.0",
     "quoted-printable": "^1.0.1",
     "rate-limiter-flexible": "^9.0.0",

backend/package.json

@@ -22,12 +22,14 @@ import initializeStreamRouter from './routes/stream.routes';
 import AuthResolver from './services/auth/AuthResolver';
 import TasksManagerFile from './services/tasks-manager/TaskManagerFile';
 import TasksManager from './services/tasks-manager/TasksManager';
+import TasksManagerPST from './services/tasks-manager/TasksManagerPST';
 import Billing from './utils/billing-plugin';
 
 export default function initializeApp(
   authResolver: AuthResolver,
   tasksManager: TasksManager,
   tasksManagerFile: TasksManagerFile,
+  tasksManagerPST: TasksManagerPST,
   miningSources: MiningSources,
   contacts: Contacts,
   userResolver: Users,
@@ -62,13 +64,19 @@ export default function initializeApp(
   app.use('/api/imap', initializeImapRoutes(authResolver, miningSources));
   app.use(
     '/api/imap',
-    initializeStreamRouter(tasksManager, tasksManagerFile, authResolver)
+    initializeStreamRouter(
+      tasksManager,
+      tasksManagerFile,
+      tasksManagerPST,
+      authResolver
+    )
   );
   app.use(
     '/api/imap',
     initializeMiningRoutes(
       tasksManager,
       tasksManagerFile,
+      tasksManagerPST,
       miningSources,
       authResolver
     )

backend/src/app.ts

@@ -9,10 +9,13 @@ import {
 import { ContactFormat } from '../services/extractors/engines/FileImport';
 import TaskManagerFile from '../services/tasks-manager/TaskManagerFile';
 import TasksManager from '../services/tasks-manager/TasksManager';
+import TasksManagerPST from '../services/tasks-manager/TasksManagerPST';
+import { Task } from '../services/tasks-manager/types';
 import { ImapAuthError } from '../utils/errors';
 import validateType from '../utils/helpers/validation';
 import logger from '../utils/logger';
 import redis from '../utils/redis';
+import supabaseClient from '../utils/supabase';
 import {
   generateErrorObjectFromImapError,
   getValidImapLogin,
@@ -24,8 +27,6 @@ import {
   getTokenWithScopeValidation,
   validateFileContactsData
 } from './mining.helpers';
-import supabaseClient from '../utils/supabase';
-import { Task } from '../services/tasks-manager/types';
 
 /**
  * Exchanges an OAuth authorization code for tokens and extracts user email
@@ -56,6 +57,7 @@ async function exchangeForToken(
 export default function initializeMiningController(
   tasksManager: TasksManager,
   tasksManagerFile: TaskManagerFile,
+  tasksManagerPST: TasksManagerPST,
   miningSources: MiningSources
 ) {
   return {
@@ -340,6 +342,55 @@ export default function initializeMiningController(
       }
     },
 
+    async startMiningPST(req: Request, res: Response, next: NextFunction) {
+      const user = res.locals.user as User;
+
+      const {
+        name,
+        extractSignatures
+      }: {
+        name: string;
+        extractSignatures: boolean;
+        // file
+      } = req.body;
+
+      const errors = [
+        validateType('name', name, 'string'),
+        validateType('extractSignatures', extractSignatures, 'boolean')
+      ].filter(Boolean);
+
+      if (errors.length) {
+        return res
+          .status(400)
+          .json({ message: `Invalid input: ${errors.join(', ')}` });
+      }
+      try {
+        const miningTask = await tasksManagerPST.createTask(
+          user.id,
+          name,
+          extractSignatures
+        );
+        return res.status(201).send({ error: null, data: miningTask });
+      } catch (err) {
+        if (
+          err instanceof Error &&
+          err.message.toLowerCase().startsWith('invalid credentials')
+        ) {
+          return res.status(401).json({ message: err.message });
+        }
+        if (
+          err instanceof Error &&
+          'textCode' in err &&
+          err.textCode === 'CANNOT'
+        ) {
+          return res.sendStatus(409);
+        }
+
+        res.status(500);
+        return next(err);
+      }
+    },
+
     async stopMiningTask(req: Request, res: Response, next: NextFunction) {
       const { type: miningType } = req.params;
       const { user } = res.locals;
@@ -349,7 +400,14 @@ export default function initializeMiningController(
         return next(new Error('user does not exists.'));
       }
 
-      const manager = miningType === 'file' ? tasksManagerFile : tasksManager;
+      let manager;
+      if (miningType === 'file') {
+        manager = tasksManagerFile;
+      } else if (miningType === 'pst') {
+        manager = tasksManagerPST;
+      } else {
+        manager = tasksManager;
+      }
 
       const { id: taskId } = req.params;
       const {
@@ -452,6 +510,11 @@ export default function initializeMiningController(
           (!extractTask || !cleanTask)
         ) {
           throw new Error(`File mining with id: ${miningId} not found`);
+        } else if (
+          task.miningSource.type === 'pst' &&
+          (!extractTask || !cleanTask)
+        ) {
+          throw new Error(`PST mining with id: ${miningId} not found`);
         }
 
         if (user.id !== task.userId) {

backend/src/controllers/mining.controller.ts

@@ -1,19 +1,29 @@
 import { Request, Response } from 'express';
+import TasksManagerFile from '../services/tasks-manager/TaskManagerFile';
 import TasksManager from '../services/tasks-manager/TasksManager';
+import TasksManagerPST from '../services/tasks-manager/TasksManagerPST';
 import logger from '../utils/logger';
-import TasksManagerFile from '../services/tasks-manager/TaskManagerFile';
 
 export default function initializeStreamController(
   tasksManager: TasksManager,
-  tasksManagerFile: TasksManagerFile
+  tasksManagerFile: TasksManagerFile,
+  tasksManagerPST: TasksManagerPST
 ) {
   return {
     /**
      * Stream the progress of email extraction and scanning via Server-Sent Events (SSE).
      */
     streamProgress: (req: Request, res: Response) => {
       const { id: taskId, type: miningType } = req.params;
-      const manager = miningType === 'file' ? tasksManagerFile : tasksManager;
+
+      let manager;
+      if (miningType === 'file') {
+        manager = tasksManagerFile;
+      } else if (miningType === 'pst') {
+        manager = tasksManagerPST;
+      } else {
+        manager = tasksManager;
+      }
 
       try {
         const task = manager.getActiveTask(taskId);

backend/src/controllers/stream.controller.ts

@@ -3,12 +3,14 @@ import initializeMiningController from '../controllers/mining.controller';
 import { MiningSources } from '../db/interfaces/MiningSources';
 import initializeAuthMiddleware from '../middleware/auth';
 import AuthResolver from '../services/auth/AuthResolver';
-import TasksManager from '../services/tasks-manager/TasksManager';
 import TasksManagerFile from '../services/tasks-manager/TaskManagerFile';
+import TasksManager from '../services/tasks-manager/TasksManager';
+import TasksManagerPST from '../services/tasks-manager/TasksManagerPST';
 
 export default function initializeMiningRoutes(
   tasksManager: TasksManager,
   tasksManagerFile: TasksManagerFile,
+  tasksManagerPST: TasksManagerPST,
   miningSource: MiningSources,
   authResolver: AuthResolver
 ) {
@@ -17,13 +19,19 @@ export default function initializeMiningRoutes(
   const {
     startMining,
     startMiningFile,
+    startMiningPST,
     stopMiningTask,
     getMiningTask,
     createProviderMiningSource,
     createProviderMiningSourceCallback,
     createImapMiningSource,
     getMiningSources
-  } = initializeMiningController(tasksManager, tasksManagerFile, miningSource);
+  } = initializeMiningController(
+    tasksManager,
+    tasksManagerFile,
+    tasksManagerPST,
+    miningSource
+  );
 
   const authMiddleware = initializeAuthMiddleware(authResolver);
 
@@ -44,6 +52,7 @@ export default function initializeMiningRoutes(
   router.get('/mine/:userId/', authMiddleware, getMiningTask);
   router.post('/mine/email/:userId', authMiddleware, startMining);
   router.post('/mine/file/:userId', authMiddleware, startMiningFile);
+  router.post('/mine/pst/:userId', authMiddleware, startMiningPST);
   router.post('/mine/:type/:userId/:id', authMiddleware, stopMiningTask);
 
   return router;

backend/src/routes/mining.routes.ts

@@ -2,12 +2,14 @@ import { Router } from 'express';
 import initializeStreamController from '../controllers/stream.controller';
 import initializeAuthMiddleware from '../middleware/auth';
 import AuthResolver from '../services/auth/AuthResolver';
-import TasksManager from '../services/tasks-manager/TasksManager';
 import TasksManagerFile from '../services/tasks-manager/TaskManagerFile';
+import TasksManager from '../services/tasks-manager/TasksManager';
+import TasksManagerPST from '../services/tasks-manager/TasksManagerPST';
 
 export default function initializeStreamRouter(
   tasksManager: TasksManager,
   tasksManagerFile: TasksManagerFile,
+  tasksManagerPST: TasksManagerPST,
   authResolver: AuthResolver
 ) {
   const router = Router();
@@ -16,7 +18,8 @@ export default function initializeStreamRouter(
 
   const { streamProgress } = initializeStreamController(
     tasksManager,
-    tasksManagerFile
+    tasksManagerFile,
+    tasksManagerPST
   );
   router.get('/mine/:type/:id/progress/', authMiddleware, streamProgress);
 

backend/src/routes/stream.routes.ts

@@ -5,17 +5,19 @@ import ENV from './config';
 import pool from './db/pg';
 import PgContacts from './db/pg/PgContacts';
 import PgMiningSources from './db/pg/PgMiningSources';
+import SupabaseTasks from './db/supabase/tasks';
 import SupabaseUsers from './db/supabase/users';
 import SupabaseAuthResolver from './services/auth/SupabaseAuthResolver';
+import EmailFetcherClient from './services/email-fetching';
+import PSTFetcherClient from './services/email-fetching/pst';
 import SSEBroadcasterFactory from './services/factory/SSEBroadcasterFactory';
+import TasksManagerFile from './services/tasks-manager/TaskManagerFile';
 import TasksManager from './services/tasks-manager/TasksManager';
+import TasksManagerPST from './services/tasks-manager/TasksManagerPST';
 import { flickrBase58IdGenerator } from './services/tasks-manager/utils';
 import logger from './utils/logger';
 import redis from './utils/redis';
 import supabaseClient from './utils/supabase';
-import SupabaseTasks from './db/supabase/tasks';
-import TasksManagerFile from './services/tasks-manager/TaskManagerFile';
-import EmailFetcherClient from './services/email-fetching';
 
 // eslint-disable-next-line no-console
 console.log(
@@ -63,10 +65,24 @@ console.log(
     new SSEBroadcasterFactory(),
     flickrBase58IdGenerator()
   );
+  const tasksManagerPST = new TasksManagerPST(
+    tasksResolver,
+    redis.getSubscriberClient(),
+    redis.getClient(),
+    new PSTFetcherClient(
+      logger,
+      ENV.EMAIL_FETCHING_SERVICE_API_TOKEN, //! RDNDNT
+      ENV.EMAIL_FETCHING_SERVICE_URL
+    ),
+    new SSEBroadcasterFactory(),
+    flickrBase58IdGenerator()
+  );
+
   const app = initializeApp(
     authResolver,
     tasksManager,
     tasksManagerFile,
+    tasksManagerPST,
     miningSources,
     contactsResolver,
     userResolver,