background passive mining (#2616)

* fix passive mining dialog ui

* create sources page

* create passive-mining edge function

* Update 20260130124036_passive_mining_in_mining_sources.sql

* Update 20260130124036_passive_mining_in_mining_sources.sql

* Update 20260130124036_passive_mining_in_mining_sources.sql

* Update 20260130124036_passive_mining_in_mining_sources.sql

* rm getMiningSources from backend

* Update mining.controller.ts

* Update index.ts

* Update index.ts

* Update EmailFetcherFactory.ts

* t

* fill user data if sent backend request

* add passive_mining to task details

* add passive_mining to task details

* fix mining & imap error propagation

* Update index.ts

* Update leadminer.ts

* Update auth.ts

* Update sources.vue

* Create boxes.ts

* Update index.ts

* add /sources to redirectoptions

* fixes

* Update TasksManager.ts

* rername `taskManagerFile` to `tasksManagerFile`

* fixes

* i18n

* Update stream.routes.ts

* Update app.ts

* i18n

* Update index.ts

* fixes

* Update mining.controller.ts

* Update mining.controller.ts

* use getUser instead of a service user mock

* Update auth.ts

* Update auth.ts

Author: J43fura

backend/src/app.ts

@@ -20,8 +20,8 @@ import initializeImapRoutes from './routes/imap.routes';
 import initializeMiningRoutes from './routes/mining.routes';
 import initializeStreamRouter from './routes/stream.routes';
 import AuthResolver from './services/auth/AuthResolver';
-import TasksManagerFile from './services/tasks-manager/TaskManagerFile';
 import TasksManager from './services/tasks-manager/TasksManager';
+import TasksManagerFile from './services/tasks-manager/TasksManagerFile';
 import TasksManagerPST from './services/tasks-manager/TasksManagerPST';
 import Billing from './utils/billing-plugin';
 

backend/src/controllers/mining.controller.ts

@@ -9,8 +9,8 @@ import {
 } from '../db/interfaces/MiningSources';
 import RedisQueuedEmailsCache from '../services/cache/redis/RedisQueuedEmailsCache';
 import { ContactFormat } from '../services/extractors/engines/FileImport';
-import TaskManagerFile from '../services/tasks-manager/TaskManagerFile';
 import TasksManager from '../services/tasks-manager/TasksManager';
+import TasksManagerFile from '../services/tasks-manager/TasksManagerFile';
 import TasksManagerPST from '../services/tasks-manager/TasksManagerPST';
 import { Task } from '../services/tasks-manager/types';
 import { ImapAuthError } from '../utils/errors';
@@ -160,7 +160,7 @@ async function publishPreviouslyUnverifiedEmailsToCleaning(
 
 export default function initializeMiningController(
   tasksManager: TasksManager,
-  tasksManagerFile: TaskManagerFile,
+  tasksManagerFile: TasksManagerFile,
   tasksManagerPST: TasksManagerPST,
   miningSources: MiningSources,
   contactsDB: Contacts
@@ -293,31 +293,8 @@ export default function initializeMiningController(
       }
     },
 
-    async getMiningSources(_req: Request, res: Response, next: NextFunction) {
-      const user = res.locals.user as User;
-
-      try {
-        const sourcesData = await miningSources.getByUser(user.id);
-
-        const sources = sourcesData.map((s) => ({
-          email: s.email,
-          type: s.type,
-          passive_mining: s.passive_mining
-        }));
-
-        return res.status(200).send({
-          message: 'Mining sources retrieved successfully',
-          sources
-        });
-      } catch (error) {
-        res.status(500);
-        return next(error);
-      }
-    },
-
     async startMining(req: Request, res: Response, next: NextFunction) {
       const user = res.locals.user as User;
-
       const {
         extractSignatures,
         miningSource: { email },
@@ -330,6 +307,8 @@ export default function initializeMiningController(
         extractSignatures: boolean;
       } = req.body;
 
+      user.email = email; // used when user is not provided (edge function req)
+
       const errors = [
         validateType('email', email, 'string'),
         validateType('boxes', folders, 'string[]'),
@@ -384,16 +363,30 @@ export default function initializeMiningController(
       } catch (err) {
         if (
           err instanceof Error &&
-          err.message.toLowerCase().startsWith('invalid credentials')
+          'textCode' in err &&
+          err.textCode === 'CANNOT'
         ) {
-          return res.status(401).json({ message: err.message });
+          return res.sendStatus(409);
         }
+
         if (
           err instanceof Error &&
-          'textCode' in err &&
-          err.textCode === 'CANNOT'
+          err.message.includes('Request failed with status code 401')
         ) {
-          return res.sendStatus(409);
+          res
+            .status(401)
+            .send('Failed to start fetching: Invalid credentials 401');
+        }
+
+        if (
+          err instanceof Error &&
+          err.message.includes('Request failed with status code 503')
+        ) {
+          res
+            .status(503)
+            .send(
+              'Failed to start fetching: Connection not available, please try again later 503'
+            );
         }
 
         const newError = generateErrorObjectFromImapError(err);

backend/src/controllers/stream.controller.ts

@@ -1,6 +1,6 @@
 import { Request, Response } from 'express';
-import TasksManagerFile from '../services/tasks-manager/TaskManagerFile';
 import TasksManager from '../services/tasks-manager/TasksManager';
+import TasksManagerFile from '../services/tasks-manager/TasksManagerFile';
 import TasksManagerPST from '../services/tasks-manager/TasksManagerPST';
 import logger from '../utils/logger';
 

backend/src/middleware/auth.ts

@@ -1,9 +1,37 @@
 import { NextFunction, Request, Response } from 'express';
+import ENV from '../config';
 import AuthResolver from '../services/auth/AuthResolver';
+import supabase from '../utils/supabase';
 
 export default function initializeAuthMiddleware(authResolver: AuthResolver) {
   const verifyJWT = async (req: Request, res: Response, next: NextFunction) => {
     try {
+      const authHeader = req.headers.authorization;
+      const token = authHeader?.split(' ')[1];
+      // Check for service role authentication
+      if (
+        ENV.SUPABASE_SECRET_PROJECT_TOKEN &&
+        token === ENV.SUPABASE_SECRET_PROJECT_TOKEN
+      ) {
+        // Extract userId from route or query params
+        const userId = req.params.userId ?? req.query.userId;
+
+        if (userId) {
+          // Create a service user object with the extracted userId
+          const { user } = (await supabase.auth.admin.getUserById(userId)).data;
+
+          if (!user) {
+            return res.status(404).json({
+              message: 'User not found'
+            });
+          }
+
+          res.locals.user = user;
+          return next();
+        }
+      }
+
+      // Standard JWT validation
       const accessToken = authResolver.getAccessToken(req);
 
       if (!accessToken) {

backend/src/routes/mining.routes.ts

@@ -1,12 +1,12 @@
 import { Router } from 'express';
 import initializeMiningController from '../controllers/mining.controller';
+import { Contacts } from '../db/interfaces/Contacts';
 import { MiningSources } from '../db/interfaces/MiningSources';
 import initializeAuthMiddleware from '../middleware/auth';
 import AuthResolver from '../services/auth/AuthResolver';
-import TasksManagerFile from '../services/tasks-manager/TaskManagerFile';
 import TasksManager from '../services/tasks-manager/TasksManager';
+import TasksManagerFile from '../services/tasks-manager/TasksManagerFile';
 import TasksManagerPST from '../services/tasks-manager/TasksManagerPST';
-import { Contacts } from '../db/interfaces/Contacts';
 
 export default function initializeMiningRoutes(
   tasksManager: TasksManager,
@@ -26,8 +26,7 @@ export default function initializeMiningRoutes(
     getMiningTask,
     createProviderMiningSource,
     createProviderMiningSourceCallback,
-    createImapMiningSource,
-    getMiningSources
+    createImapMiningSource
   } = initializeMiningController(
     tasksManager,
     tasksManagerFile,
@@ -38,8 +37,6 @@ export default function initializeMiningRoutes(
 
   const authMiddleware = initializeAuthMiddleware(authResolver);
 
-  router.get('/mine/sources', authMiddleware, getMiningSources);
-
   router.post('/mine/sources/imap', authMiddleware, createImapMiningSource);
 
   router.post(

backend/src/routes/stream.routes.ts

@@ -2,8 +2,8 @@ import { Router } from 'express';
 import initializeStreamController from '../controllers/stream.controller';
 import initializeAuthMiddleware from '../middleware/auth';
 import AuthResolver from '../services/auth/AuthResolver';
-import TasksManagerFile from '../services/tasks-manager/TaskManagerFile';
 import TasksManager from '../services/tasks-manager/TasksManager';
+import TasksManagerFile from '../services/tasks-manager/TasksManagerFile';
 import TasksManagerPST from '../services/tasks-manager/TasksManagerPST';
 
 export default function initializeStreamRouter(

backend/src/server.ts

@@ -11,8 +11,8 @@ import SupabaseAuthResolver from './services/auth/SupabaseAuthResolver';
 import EmailFetcherClient from './services/email-fetching';
 import PSTFetcherClient from './services/email-fetching/pst';
 import SSEBroadcasterFactory from './services/factory/SSEBroadcasterFactory';
-import TasksManagerFile from './services/tasks-manager/TaskManagerFile';
 import TasksManager from './services/tasks-manager/TasksManager';
+import TasksManagerFile from './services/tasks-manager/TasksManagerFile';
 import TasksManagerPST from './services/tasks-manager/TasksManagerPST';
 import { flickrBase58IdGenerator } from './services/tasks-manager/utils';
 import logger from './utils/logger';

backend/src/services/tasks-manager/TasksManager.ts

@@ -1,3 +1,4 @@
+import { AxiosError } from 'axios';
 import { Request, Response } from 'express';
 import { Redis } from 'ioredis';
 import {
@@ -10,11 +11,10 @@ import {
   TaskProgressType
 } from './types';
 // eslint-disable-next-line max-classes-per-file
-import { TaskCategory, TaskStatus, TaskType } from '../../db/types';
-
 import ENV from '../../config';
 import { mailMiningComplete, refineContacts } from '../../db/mail';
 import SupabaseTasks from '../../db/supabase/tasks';
+import { TaskCategory, TaskStatus, TaskType } from '../../db/types';
 import logger from '../../utils/logger';
 import EmailFetcherClient from '../email-fetching';
 import SSEBroadcasterFactory from '../factory/SSEBroadcasterFactory';
@@ -145,12 +145,10 @@ export default class TasksManager {
    * @throws {Error} If a task with the same mining ID already exists.
    * @throws {Error} If there is an error when creating the task.
    */
-  async createTask({
-    email,
-    boxes,
-    fetchEmailBody,
-    userId
-  }: ImapEmailsFetcherOptions) {
+  async createTask(
+    { email, boxes, fetchEmailBody, userId }: ImapEmailsFetcherOptions,
+    passive_mining = false
+  ): Promise<RedactedTask> {
     let miningTaskId: string | null = null;
     try {
       const { miningId, stream } = await this.generateTaskInformation();
@@ -183,7 +181,8 @@ export default class TasksManager {
               },
               progress: {
                 signatures: 0
-              }
+              },
+              passive_mining
             }
           },
           fetch: {
@@ -200,7 +199,8 @@ export default class TasksManager {
                 totalMessages: 0,
                 folders: boxes,
                 fetched: 0
-              }
+              },
+              passive_mining
             }
           },
           extract: {
@@ -217,7 +217,8 @@ export default class TasksManager {
               },
               progress: {
                 extracted: 0
-              }
+              },
+              passive_mining
             }
           },
           clean: {
@@ -234,7 +235,8 @@ export default class TasksManager {
               progress: {
                 verifiedContacts: 0,
                 createdContacts: 0
-              }
+              },
+              passive_mining
             }
           }
         },
@@ -294,6 +296,9 @@ export default class TasksManager {
         logger.error(`Failed to start fetching task with id: ${miningId}`, {
           error
         });
+        if (error instanceof AxiosError) {
+          throw new Error(`Failed to start fetching: ${error.message}`);
+        }
         throw new Error('Failed to start fetching');
       }
 

…ervices/tasks-manager/TaskManagerFile.ts …rvices/tasks-manager/TasksManagerFile.tsbackend/src/services/tasks-manager/TaskManagerFile.ts renamed to backend/src/services/tasks-manager/TasksManagerFile.ts backend/src/services/tasks-manager/TaskManagerFile.ts renamed to backend/src/services/tasks-manager/TasksManagerFile.ts

@@ -66,6 +66,7 @@ export interface TaskFetch extends Task {
       fetched: number;
       folders: string[];
     };
+    passive_mining?: boolean;
   };
 }
 
@@ -78,6 +79,7 @@ export interface TaskExtract extends Task {
     progress: {
       extracted: number;
     };
+    passive_mining?: boolean;
   };
 }
 
@@ -91,6 +93,7 @@ export interface TaskClean extends Task {
       verifiedContacts: number;
       createdContacts: number;
     };
+    passive_mining?: boolean;
   };
 }
 
@@ -108,6 +111,7 @@ export interface TaskEnrich extends Task {
       data: Array<Partial<Contact>>;
       raw_data: Array<unknown>;
     }[];
+    passive_mining?: boolean;
   };
 }
 
@@ -130,6 +134,7 @@ export interface MiningTask {
   progress: TaskProgress;
   progressHandlerSSE: RealtimeSSE;
   startedAt: number;
+  passive_mining?: boolean;
 }
 
 /**