fix: clear table messages after refine & minor bugs (#2420)

* add: clear table messages after refine

* fix: bug publishing to signature extraction

* fix: use same font

* fix: use default folders filter && use \\All  as a default if exists

* fix: formatting

Author: malek10xdev

backend/src/services/imap/ImapEmailsFetcher.ts

@@ -67,8 +67,6 @@ export default class ImapEmailsFetcher {
 
   private totalFetched: number;
 
-  private connections: Connection[];
-
   private hasAuthFailureLogged: boolean;
 
   private readonly bodies: string[];
@@ -110,7 +108,6 @@ export default class ImapEmailsFetcher {
     // Set the key for the process set. used for caching.
     this.processSetKey = `caching:${miningId}`;
 
-    this.connections = [];
     this.totalFetched = 0;
 
     this.isCanceled = false;
@@ -123,7 +120,7 @@ export default class ImapEmailsFetcher {
       intervalCap: 1, // only 1 job starts per interval
       interval: 100 // 100ms gap between job starts
     });
-    this.bodies = this.fetchEmailBody ? ['HEADER', 'TEXT'] : ['HEADER'];
+    this.bodies = this.fetchEmailBody ? ['TEXT'] : []; // HEADER is handled by ImapFlow;
   }
 
   /**
@@ -337,38 +334,17 @@ export default class ImapEmailsFetcher {
 
       let header: Record<string, string[]>;
       const { seq, headers, envelope } = msg;
+      const from = envelope?.from?.pop();
+      const date = envelope?.date?.toISOString?.();
 
-      if (envelope?.from?.pop()?.address === this.userEmail) continue;
+      if (from?.address === this.userEmail) continue;
 
       try {
         header = parseHeader((headers as Buffer).toString('utf8'));
       } catch {
         continue;
       }
 
-      let text = '';
-      if (msg.bodyParts?.has('text')) {
-        const textPart = msg.bodyParts.get('text');
-        if (headers && textPart?.length) {
-          try {
-            const { text: parsedText } = await simpleParser(
-              Buffer.concat([headers, textPart]),
-              {
-                skipHtmlToText: true,
-                skipTextToHtml: true,
-                skipImageLinks: true,
-                skipTextLinks: true
-              }
-            );
-            text = parsedText?.slice(0, this.EMAIL_TEXT_MAX_LENGTH) || '';
-          } catch {
-            text = '';
-          }
-        }
-      }
-
-      const from = envelope?.from?.pop();
-      const date = envelope?.date?.toISOString?.();
       const messageId = getMessageId(header);
       header['message-id'] = [messageId];
 
@@ -396,6 +372,25 @@ export default class ImapEmailsFetcher {
         })
       );
 
+      let text = msg.bodyParts?.get('text') ?? '';
+
+      if (headers && text?.length) {
+        try {
+          const { text: parsedText } = await simpleParser(
+            Buffer.concat([headers, text as Uint8Array<ArrayBufferLike>]),
+            {
+              skipHtmlToText: true,
+              skipTextToHtml: true,
+              skipImageLinks: true,
+              skipTextLinks: true
+            }
+          );
+          text = parsedText?.slice(0, this.EMAIL_TEXT_MAX_LENGTH) || '';
+        } catch {
+          text = '';
+        }
+      }
+
       if (text.length && from && date) {
         await redisClient.xadd(
           this.signatureStream,

frontend/src/components/Mining/MiningSettingsDialog.vue

@@ -6,10 +6,21 @@
     :maximizable="$screenStore?.size?.md"
     :pt:root:class="{ 'p-dialog-maximized': !$screenStore?.size?.md }"
     :style="{ width: '60vw', height: '70vh' }"
-    pt:content:class="grow p-3 border-y border-slate-200"
-    pt:footer:class="p-3"
+    pt:content:class="grow p-4 border-y border-slate-200"
+    pt:footer:class="p-4"
     :header="t('fine_tune_mining')"
   >
+    <!-- aligned switch (quick) -->
+    <div class="flex items-center gap-2 pb-4">
+      <ToggleSwitch
+        v-model="$leadminerStore.extractSignatures"
+        input-id="extractSignatures"
+      />
+      <label for="extractSignatures" class="cursor-pointer">
+        {{ t('extract_signatures_option') }} {{ t('extract_signatures_sub') }}
+      </label>
+    </div>
+
     <div class="flex items-center gap-2">
       <div class="text-h6">{{ t('select_folders_to_mine') }}</div>
       <Button
@@ -28,22 +39,11 @@
         <i class="pi pi-envelope ml-1.5" />
       </Badge>
     </div>
-
-    <!-- aligned switch (quick) -->
-    <div class="flex items-center gap-2 mt-4 pl-8">
-      <ToggleSwitch
-        v-model="$leadminerStore.extractSignatures"
-        input-id="extractSignatures"
-      />
-      <label for="extractSignatures" class="text-sm cursor-pointer">
-        {{ t('extract_signatures_option') }}
-      </label>
-      <small class="">{{ t('extract_signatures_sub') }}</small>
-    </div>
     <TreeCard
       v-if="shouldShowTreeCard"
       :class="{ disabled: $leadminerStore.activeMiningTask }"
     />
+
     <template #footer>
       <Button :label="$t('common.save')" @click="close" />
     </template>
@@ -103,14 +103,14 @@ defineExpose({
     "select_folders_to_mine": "Select folders to mine",
     "email_messages_selected": "Email messages selected",
     "extract_signatures_option": "Extract contact details from signatures",
-    "extract_signatures_sub": "(this may take more time)"
+    "extract_signatures_sub": "( this may take more time )"
   },
   "fr": {
     "fine_tune_mining": "Affinez l'extraction",
     "select_folders_to_mine": "Sélectionnez les dossiers à extraire",
     "email_messages_selected": "E-mails sélectionnés",
     "extract_signatures_option": "Extraire les coordonnées depuis les signatures",
-    "extract_signatures_sub": "(cela peut prendre plus de temps)"
+    "extract_signatures_sub": "( cela peut prendre plus de temps )"
   }
 }
 </i18n>

frontend/src/utils/boxes.ts

@@ -15,42 +15,21 @@ export const EMAIL_EXCLUDED_FOLDERS = [
   'drafts',
   'junk',
   'trash',
-  '\\all',
   '\\drafts',
   '\\junk',
   '\\trash',
 ];
 
-/**
- * Gets all selected folders
- * @param boxes - The array of folder names to filter
- * @returns The filtered array of boxes
- */
-function getAllFolders(boxes: BoxNode[]) {
-  const folders: TreeSelectionKeys = [];
-  objectScan(['**.key'], {
-    joined: true,
-    // eslint-disable-next-line @typescript-eslint/no-explicit-any
-    filterFn: ({ parent }: any) => {
-      const { key } = parent;
-      folders[key] = {
-        checked: true,
-        partialChecked: false,
-        isNoSelect: Boolean(parent.attribs?.includes('\\Noselect')),
-      };
-    },
-  })(boxes);
-  return folders;
-}
-
 /**
  * Filters out default selected folders from the input boxes based on email service
  * @param boxes - The array of folder names to filter
  * @returns The filtered array of boxes
  */
-// eslint-disable-next-line @typescript-eslint/no-unused-vars
+
 function getFilteredFolders(boxes: BoxNode[]) {
   const filteredFolders: TreeSelectionKeys = [];
+  let foundAllMailKey: string | null = null;
+
   objectScan(['**.key'], {
     joined: true,
     // eslint-disable-next-line @typescript-eslint/no-explicit-any
@@ -59,24 +38,43 @@ function getFilteredFolders(boxes: BoxNode[]) {
       const folder = key.split('/');
       const folderName = folder.pop();
       const folderParent = folder.pop();
+      const isAllMail = attribs?.includes('\\All');
+
+      if (foundAllMailKey && !isAllMail) return;
 
       const isExcluded = [...(attribs ?? []), folderName, folderParent]
         .filter(Boolean)
         .map((name) => name.toLowerCase())
         .some((name) => EMAIL_EXCLUDED_FOLDERS.includes(name));
 
-      if (!isExcluded) {
-        // Format to be like PrimeVue's TreeSelectionKeys
-        const checked = attribs && !attribs.includes('\\HasChildren');
-        const partialChecked = !checked;
+      if (isExcluded) return;
 
+      // Format to be like PrimeVue's TreeSelectionKeys
+      const checked = attribs && !attribs.includes('\\HasChildren');
+      const partialChecked = !checked;
+      const isNoSelect = Boolean(attribs?.includes('\\Noselect'));
+
+      if (isAllMail && !isNoSelect) {
+        // skipcq: JS-0320
+        // eslint-disable-next-line @typescript-eslint/no-dynamic-delete
+        Object.keys(filteredFolders).forEach((k) => delete filteredFolders[k]);
+        // Add All Mail as the only selected folder
+        filteredFolders[key] = {
+          checked: true,
+          partialChecked: false,
+          isNoSelect,
+        };
+        foundAllMailKey = key;
+      } else {
         filteredFolders[key] = {
           checked,
           partialChecked,
+          isNoSelect,
         };
       }
     },
   })(boxes);
+
   return filteredFolders;
 }
 
@@ -86,6 +84,6 @@ function getFilteredFolders(boxes: BoxNode[]) {
  * @returns The filtered array of boxes
  */
 export function getDefaultSelectedFolders(boxes: BoxNode[]) {
-  const defaultFolders = getAllFolders(boxes);
+  const defaultFolders = getFilteredFolders(boxes);
   return defaultFolders;
 }

supabase/migrations/20250122185859_optimize_refine_persons.sql

@@ -132,5 +132,8 @@ BEGIN
     DROP TABLE IF EXISTS real_names;
     DROP TABLE IF EXISTS email_aggregates;
     DROP TABLE IF EXISTS combined_data;
+
+    -- Clear table messages
+    DELETE FROM private.messages m WHERE m.user_id = user_id
 END;
 $$;