Fix signature bugs (#2329)

* use string concat instead of array

* enable loki graceful shutdown

* consider enriched when there is fields other than email, user_id

* update useful signature rules

* fix: set to null instead of empty array

* update prompt and llm models list

* set the best accurate llm model

Author: malek10xdev

backend/src/emailSignatureWorker.ts

@@ -23,15 +23,15 @@ const subscriberRedisClient = redis.getSubscriberClient();
 
 const emailSignatureCache = new RedisEmailSignatureCache(redisClient);
 
-const llmModel = LLMModels.mistralai7bInstruct;
+const llmModel = LLMModels.cohere;
 
 const { processStreamData } = initializeEmailSignatureProcessor(
   supabaseClient,
   new Signature(
     new TokenBucketRateLimiter(llmModel.includes('free') ? 15 : 500, 60 * 1000),
     logger,
     {
-      model: LLMModels.mistralai7bInstruct,
+      model: llmModel,
       apiKey: ENV.SIGNATURE_OPENROUTER_API_KEY,
       useLLM: ENV.SIGNATURE_USE_LLM
     }

backend/src/services/imap/ImapEmailsFetcher.ts

@@ -250,8 +250,8 @@ export default class ImapEmailsFetcher {
       let messageCounter = 0;
 
       fetchResult.on('message', (msg, seqNumber) => {
-        let headerChunks: Buffer[] = [];
-        let bodyChunks: Buffer[] = [];
+        let headerChunks = '';
+        let bodyChunks = '';
 
         if (this.isCanceled === true) {
           const message = `Canceled process on folder ${folderPath} with ID ${this.miningId}`;
@@ -264,25 +264,22 @@ export default class ImapEmailsFetcher {
         msg.on('body', (stream, streamInfo) => {
           stream.on('data', (chunk) => {
             if (streamInfo.which.includes('HEADER')) {
-              headerChunks.push(chunk);
+              headerChunks += chunk;
             } else if (this.fetchEmailBody) {
-              bodyChunks.push(chunk);
+              bodyChunks += chunk;
             }
           });
         });
 
         msg.once('end', async () => {
-          const headerBuf = Buffer.concat(headerChunks);
-          const bodyBuf = Buffer.concat(bodyChunks);
+          const parsedHeader = parseHeader(headerChunks);
 
-          const parsedHeader = parseHeader(headerBuf.toString('utf8'));
-
-          const mail = await simpleParser(Buffer.concat([headerBuf, bodyBuf]));
+          const mail = await simpleParser(headerChunks + bodyChunks);
           const text = (mail.text || '').slice(0, 4000);
 
           // Clear large chunks early
-          headerChunks = [];
-          bodyChunks = [];
+          headerChunks = '';
+          bodyChunks = '';
 
           const messageId = getMessageId(parsedHeader);
 

backend/src/services/signature/llm/index.ts

@@ -14,11 +14,12 @@ import {
 } from './output-checkers';
 
 export enum LLMModels {
-  DeepSeek8bFree = 'deepseek/deepseek-r1-0528-qwen3-8b:free',
-  qwen7bInstructFree = 'qwen/qwen-2.5-7b-instruct:free',
-  googleGemma9bIt = 'google/gemma-2-9b-it',
-  deepseekR1DistillQwen32B = 'deepseek/deepseek-r1-distill-qwen-1.5b',
-  mistralai7bInstruct = 'mistralai/mistral-7b-instruct-v0.2'
+  qwenFree = 'qwen/qwen-2.5-7b-instruct:free',
+  deepseekFree = 'deepseek/deepseek-r1-0528-qwen3-8b:free',
+  cohere = 'cohere/command-r',
+  cohere7b = 'cohere/command-r7b-12-2024',
+  meta = 'meta-llama/llama-3.1-8b-instruct',
+  google = 'google/gemma-2-9b-it'
 }
 
 export type LLMModelType = `${LLMModels}`;
@@ -107,7 +108,8 @@ export const SignaturePrompt = {
     type: 'json_object'
   },
 
-  buildUserPrompt: (signature: string) => `${signature}`
+  buildUserPrompt: (signature: string) =>
+    `RETURN NULL IF NOT A REAL PERSON SIGNATURE:\n${signature}`
 };
 
 export class SignatureLLM implements ExtractSignature {

backend/src/utils/logger.ts

@@ -26,6 +26,7 @@ function initLogger() {
           json: true,
           replaceTimestamp: true,
           format: format.combine(commonFormat, format.json()),
+          gracefulShutdown: true,
           // eslint-disable-next-line no-console
           onConnectionError: (err) => console.error(err)
         })

backend/src/workers/email-signature/handler.ts

@@ -198,17 +198,25 @@ export class EmailSignatureProcessor {
 
     const contact = await this.signature.extract(signature);
     if (!contact) return null;
-    return {
+
+    const enrichedContact: Partial<Contact> = {
       email,
       user_id: userId,
-      name: contact?.name,
-      image: contact?.image,
-      location: contact?.address,
-      telephone: contact?.telephone,
-      job_title: contact?.jobTitle,
-      works_for: contact?.worksFor,
-      same_as: contact?.sameAs
+      name: contact.name,
+      image: contact.image,
+      location: contact.address,
+      telephone: contact.telephone,
+      job_title: contact.jobTitle,
+      works_for: contact.worksFor,
+      same_as: contact.sameAs
     };
+
+    // Check if anything beyond email and user_id is present
+    const hasExtraInfo = Object.entries(enrichedContact).some(
+      ([key, value]) => !['email', 'user_id'].includes(key) && value
+    );
+
+    return hasExtraInfo ? enrichedContact : null;
   }
 
   private async upsertContact(contact: Partial<Contact>): Promise<void> {

backend/src/workers/email-signature/utils.ts

@@ -29,13 +29,23 @@ export async function pushNotificationDB(
 
 export function isUsefulSignatureContent(signature: string): boolean {
   const text = signature.trim();
-
+  const words = text.split(/\s+/);
   const hasURL = /(https?:\/\/|www\.)\S+/i.test(text);
   const hasDigits = /\d{3,}/.test(text);
-  const hasMultipleWords = text.split(/\s+/).length >= 5;
   const hasSymbols = /[@+:]/.test(text); // Common in phone/email/title formats
 
-  const useful = hasURL || hasDigits || hasSymbols || hasMultipleWords;
+  const wordsMinMax =
+    words.length >= 5 && words.length <= 40 && text.length <= 300;
+
+  // 2. At least one positive signal
+  const positive = hasURL || hasDigits || hasSymbols;
+
+  const blocks = [
+    /^(Envoyé\s+à\s+partir\s+de|Sent\s+from)\s+(Outlook|Gmail|iPhone|Android)/i
+  ];
+
+  const isUseful =
+    wordsMinMax && positive && !blocks.some((rx) => rx.test(text));
 
-  return useful;
+  return isUseful;
 }

supabase/migrations/20250520002931_add_persons_phone_numbers.sql

@@ -209,9 +209,7 @@ BEGIN
     END IF;
 
     -- Merge incoming phone numbers into telephone array
-    new_telephone := (
-        string_to_array(contact_record->>'telephone', ',')
-    );
+    new_telephone := string_to_array(NULLIF(contact_record->>'telephone', ''), ',');
 
     IF new_telephone IS NOT NULL THEN
         SELECT p.telephone
@@ -257,14 +255,14 @@ BEGIN
         name = COALESCE(pp.name, new_name::TEXT),
         url = COALESCE(pp.url, (contact_record->>'url')::TEXT),
         image = COALESCE(pp.image, (contact_record->>'image')::TEXT),
-        location = COALESCE(pp.location, string_to_array(contact_record->>'location', ',')::TEXT[]),
+        location = COALESCE(pp.location, string_to_array(NULLIF(contact_record->>'location', ''), ',')::TEXT[]),
         alternate_name = COALESCE(pp.alternate_name, (new_alternate_name)::TEXT[]),
-        same_as = COALESCE(pp.same_as, string_to_array(contact_record->>'same_as', ',')::TEXT[]),
+        same_as = COALESCE(pp.same_as, string_to_array(NULLIF(contact_record->>'same_as', ''), ',')::TEXT[]),
         given_name = COALESCE(pp.given_name, (contact_record->>'given_name')::TEXT),
         family_name = COALESCE(pp.family_name, (contact_record->>'family_name')::TEXT),
         job_title = COALESCE(pp.job_title, (contact_record->>'job_title')::TEXT),
         works_for = COALESCE(pp.works_for, organization_id),
-        identifiers = COALESCE(pp.identifiers, string_to_array(contact_record->>'identifiers', ',')::TEXT[]),
+        identifiers = COALESCE(pp.identifiers, string_to_array(NULLIF(contact_record->>'identifiers', ''), ',')::TEXT[]),
         status = COALESCE(pp.status, (contact_record->>'status')::TEXT),
         telephone = COALESCE(pp.telephone, (merged_telephone)::TEXT[])
       WHERE 
@@ -276,14 +274,14 @@ BEGIN
         name = COALESCE(new_name::TEXT, pp.name),
         url = COALESCE((contact_record->>'url')::TEXT, pp.url),
         image = COALESCE((contact_record->>'image')::TEXT, pp.image),
-        location = COALESCE(string_to_array(contact_record->>'location', ',')::TEXT[], pp.location),
+        location = COALESCE(string_to_array(NULLIF(contact_record->>'location', ''), ',')::TEXT[], pp.location),
         alternate_name = COALESCE((new_alternate_name)::TEXT[], pp.alternate_name),
-        same_as = COALESCE(string_to_array(contact_record->>'same_as', ',')::TEXT[], pp.same_as),
+        same_as = COALESCE(string_to_array(NULLIF(contact_record->>'same_as', ''), ',')::TEXT[], pp.same_as),
         given_name = COALESCE((contact_record->>'given_name')::TEXT, pp.given_name),
         family_name = COALESCE((contact_record->>'family_name')::TEXT, pp.family_name),
         job_title = COALESCE((contact_record->>'job_title')::TEXT, pp.job_title),
         works_for = COALESCE(organization_id, pp.works_for),
-        identifiers = COALESCE(string_to_array(contact_record->>'identifiers', ',')::TEXT[], pp.identifiers),
+        identifiers = COALESCE(string_to_array(NULLIF(contact_record->>'identifiers', ''), ',')::TEXT[], pp.identifiers),
         status = COALESCE((contact_record->>'status')::TEXT, pp.status),
         telephone = COALESCE((merged_telephone)::TEXT[], pp.telephone)