Imap emails fetcher fixes (#2381)

Author: malek10xdev

backend/src/middleware/cors.ts

@@ -8,7 +8,8 @@ const corsOptions: CorsOptions = {
     'Authorization',
     'X-Requested-With',
     'Content-type',
-    'x-sb-jwt'
+    'x-sb-jwt',
+    'last-event-id'
   ],
   optionsSuccessStatus: 200, // some legacy browsers (IE11, various SmartTVs) choke on 204,
   credentials: true

backend/src/services/imap/ImapEmailsFetcher.ts

@@ -74,9 +74,9 @@ function buildSequenceRanges(total: number, chunkSize: number): string[] {
 }
 
 export default class ImapEmailsFetcher {
-  private readonly CONNECTION_TIMEOUT_MS = 10000;
+  private readonly CONNECTION_TIMEOUT_MS = 20000;
 
-  private readonly EMAIL_TEXT_MAX_LENGTH = 4000;
+  private readonly EMAIL_TEXT_MAX_LENGTH = 3000;
 
   private readonly userIdentifier: string;
 
@@ -90,6 +90,8 @@ export default class ImapEmailsFetcher {
 
   private isCanceled = false;
 
+  private hasAuthFailureLogged = false;
+
   private activeConnections = new Set<Connection>();
 
   private readonly bodies = ['HEADER'];
@@ -124,7 +126,7 @@ export default class ImapEmailsFetcher {
     this.processSetKey = `caching:${miningId}`;
 
     if (this.fetchEmailBody) {
-      this.bodies.push('');
+      this.bodies.push('TEXT');
     }
   }
 
@@ -182,6 +184,30 @@ export default class ImapEmailsFetcher {
     }
   }
 
+  /**
+   * Checks if an error represents a fatal authentication failure.
+   * @param error - The error to check
+   * @returns True if this is an authentication failure
+   */
+  private static isAuthFailure(error: unknown): boolean {
+    if (typeof error !== 'object' || error === null) {
+      return false;
+    }
+
+    const err = error as Record<string, unknown>;
+
+    return (
+      err.authenticationFailed === true ||
+      err.serverResponseCode === 'AUTHENTICATIONFAILED' ||
+      (err.responseStatus === 'NO' &&
+        typeof err.responseText === 'string' &&
+        err.responseText.includes('Invalid credentials')) ||
+      (typeof err.message === 'string' &&
+        (err.message.includes('AUTHENTICATIONFAILED') ||
+          err.message.includes('Invalid credentials')))
+    );
+  }
+
   /**
    * Fetches the total number of messages across the specified folders on an IMAP server.
    */
@@ -250,7 +276,7 @@ export default class ImapEmailsFetcher {
       for (const folder of this.folders) {
         if (this.isCanceled) {
           logger.info(
-            `[${this.miningId}] Cancellation detected after processing folder ${folder}. No further folders will be processed.`
+            `[${this.miningId}] Cancellation requested; stopping before folder ${folder}`
           );
           break;
         }
@@ -265,10 +291,22 @@ export default class ImapEmailsFetcher {
           // eslint-disable-next-line no-await-in-loop
           await this.process;
         } catch (error) {
-          logger.error(
-            `[${this.miningId}] Error when fetching emails from folder ${folder}:`,
-            error
-          );
+          if (ImapEmailsFetcher.isAuthFailure(error)) {
+            if (!this.hasAuthFailureLogged) {
+              logger.error(
+                `[${this.miningId}] Authentication failed; aborting mining task`,
+                error
+              );
+              this.hasAuthFailureLogged = true;
+            }
+            this.isCanceled = true;
+            break;
+          } else {
+            logger.error(
+              `[${this.miningId}] Error when fetching emails from folder ${folder}:`,
+              error
+            );
+          }
         }
       }
 
@@ -313,8 +351,15 @@ export default class ImapEmailsFetcher {
         source: false,
         envelope: true,
         headers: true,
-        bodyParts: ['HEADER', 'TEXT']
+        bodyParts: this.bodies
       })) {
+        if (this.isCanceled) {
+          logger.info(
+            `[${this.miningId}] Cancellation detected; stopping range ${range}`
+          );
+          break;
+        }
+
         processedCount += 1;
 
         const { seq, headers, envelope } = msg;
@@ -385,7 +430,7 @@ export default class ImapEmailsFetcher {
           })
         );
 
-        if (text && from && date) {
+        if (text.length && from && date) {
           pipeline.xadd(
             this.signatureStream,
             '*',
@@ -418,15 +463,15 @@ export default class ImapEmailsFetcher {
 
         header = null;
 
-        if (pipeline.length >= parallelBatchSize) {
+        if (publishedEmails >= parallelBatchSize) {
           await pipeline.exec();
           await publishFetchingProgress(this.miningId, publishedEmails);
           pipeline = redisClient.multi();
           publishedEmails = 0;
         }
       }
 
-      if (pipeline.length) {
+      if (publishedEmails > 0) {
         await pipeline.exec();
         await publishFetchingProgress(this.miningId, publishedEmails);
       }
@@ -558,6 +603,13 @@ export default class ImapEmailsFetcher {
     folderPath: string,
     totalInFolder: number
   ): Promise<void> {
+    if (this.isCanceled) {
+      logger.info(
+        `[${this.miningId}] Cancellation detected; skipping large folder processing for ${folderPath}`
+      );
+      return;
+    }
+
     logger.info(
       `[${this.miningId}] Parallel fetching ${totalInFolder} messages from ${folderPath}`
     );
@@ -600,6 +652,13 @@ export default class ImapEmailsFetcher {
 
     try {
       for (let i = 0; i < ranges.length; i += connections.length) {
+        if (this.isCanceled) {
+          logger.info(
+            `[${this.miningId}] Cancellation detected; stopping batch processing for ${folderPath}`
+          );
+          break;
+        }
+
         const batch = ranges.slice(i, i + connections.length);
         const batchTasks = batch.map((range, index) => {
           const connection = connections[index];

frontend/src/components/ProgressCard.vue

@@ -8,7 +8,7 @@
 
     <div id="progress-time" class="mb-3">
       <slot name="progress-time">
-        <div v-if="progressPercentage < 100">
+        <div v-if="progressPercentage > 0 && progressPercentage < 100">
           {{ t('remaining_time', { t: estimatedRemainingTimeConverted }) }}
         </div>
         <div v-else-if="progressPercentage === 100">
@@ -114,12 +114,12 @@ onMounted(() => {
 {
   "en": {
     "finished_in": "Finished in {t}",
-    "estimated_time": "Estimated time: {t}",
+    "estimated_time": "The contact mining may take around 20 minutes depending on your inbox size",
     "remaining_time": "{t} remaining"
   },
   "fr": {
     "finished_in": "Terminé en {t}",
-    "estimated_time": "Temps estimé : {t}",
+    "estimated_time": "L'extraction de contacts peut prendre environ 20 minutes selon la taille de votre boîte mail",
     "remaining_time": "{t} restantes"
   }
 }