clean & convert location column type to text (#2509)

J43fura · web-flow · commit a6124ed4fa1f · 2025-11-07T14:12:42.000+01:00
* fix style

* Create 20251105175657_location_text.sql

* Update contact.ts

* Update MiningTable.vue

* Update ContactInformationSidebar.vue

* Update ContactInformationSidebar.vue

* update backend

* Update 20251105175657_location_text.sql

* Update 20251105175657_location_text.sql

* backend fixes

* backend: fix location

* Update enrichments.test.ts

* Update enricher.test.ts

* Update enricher.test.ts

* Update enricher.test.ts

* Update enricher.test.ts

* Update enricher.test.ts

* prettier

* fix 'clearSyncInterval' was used before it was defined

* fix style
diff --git a/backend/src/db/supabase/enrichments.ts b/backend/src/db/supabase/enrichments.ts
@@ -1,10 +1,10 @@
-import { Logger } from 'winston';
 import { SupabaseClient } from '@supabase/supabase-js';
+import { Logger } from 'winston';
 import { TaskCategory, TaskStatus, TaskType } from '../types';
 
+import { TaskEnrich } from '../../services/tasks-manager/types';
 import Engagements from './engagements';
 import SupabaseTasks from './tasks';
-import { TaskEnrich } from '../../services/tasks-manager/types';
 
 interface Contact {
   id: string;
@@ -14,7 +14,7 @@ interface Contact {
   givenName?: string;
   familyName?: string;
   alternateName?: string[];
-  location?: string[];
+  location?: string;
   organization?: string;
   jobTitle?: string;
   sameAs?: string[];
@@ -134,7 +134,7 @@ export default class Enrichments {
       family_name: contact.familyName,
       works_for: contact.organization,
       same_as: contact.sameAs?.join(','),
-      location: contact.location?.join(','),
+      location: contact.location,
       alternate_name: contact.alternateName?.join(','),
       user_id: task.userId
     }));
diff --git a/backend/src/db/types.ts b/backend/src/db/types.ts
@@ -55,7 +55,7 @@ export interface Person {
   url?: string;
   name?: string;
   image?: string;
-  location?: string[];
+  location?: string;
   jobTitle?: string;
   sameAs?: string[];
   givenName?: string;
@@ -103,7 +103,7 @@ export interface Contact {
   given_name?: string;
   family_name?: string;
   alternate_name?: string[];
-  location?: string[];
+  location?: string;
   works_for?: string;
   job_title?: string;
   same_as?: string[];
diff --git a/backend/src/services/enrichment/Engine.ts b/backend/src/services/enrichment/Engine.ts
@@ -8,7 +8,7 @@ export interface Person {
   family_name?: string;
   works_for?: string;
   alternate_name?: string[];
-  location?: string[];
+  location?: string;
   same_as?: string[];
   identifiers: string[];
 }
@@ -17,7 +17,7 @@ export interface EngineResult {
   email: string;
   name?: string;
   image?: string;
-  location?: string[];
+  location?: string;
   jobTitle?: string;
   organization?: string;
   givenName?: string;
diff --git a/backend/src/services/enrichment/enrich-layer/index.ts b/backend/src/services/enrichment/enrich-layer/index.ts
@@ -81,15 +81,15 @@ export default class EnrichLayer implements Engine {
             Boolean(id)
           )
         ),
-        location: undefinedIfEmpty([
+        location: undefinedIfFalsy(
           [
             response?.profile?.city,
             response?.profile?.state,
             response?.profile?.country_full_name
           ]
             .filter((loc): loc is string => Boolean(loc))
             .join(', ')
-        ]),
+        ),
         sameAs: undefinedIfEmpty([
           response?.linkedin_profile_url,
           response?.facebook_profile_url,
diff --git a/backend/src/services/enrichment/thedig/client.ts b/backend/src/services/enrichment/thedig/client.ts
@@ -24,8 +24,8 @@ export interface EnrichPersonRequest {
   identifier?: string[];
   nationality?: string[];
   description?: string[];
-  homeLocation?: string[];
-  workLocation?: string[];
+  homeLocation?: string;
+  workLocation?: string;
   alternateName?: string[];
   knowsLanguage?: string[];
 }
@@ -39,8 +39,8 @@ export interface EnrichPersonResponse {
   image?: string[];
   jobTitle?: string[];
   worksFor?: string[];
-  homeLocation?: string[];
-  workLocation?: string[];
+  homeLocation?: string;
+  workLocation?: string;
   sameAs?: string[];
   identifier?: string[];
   description?: string[];
diff --git a/backend/src/services/enrichment/thedig/index.ts b/backend/src/services/enrichment/thedig/index.ts
@@ -1,13 +1,13 @@
 import { Logger } from 'winston';
+import {
+  undefinedIfEmpty,
+  undefinedIfFalsy
+} from '../../../utils/helpers/validation';
 import { Engine, EngineResult, Person } from '../Engine';
 import VoilanorbertApi, {
   EnrichPersonRequest,
   EnrichPersonResponse
 } from './client';
-import {
-  undefinedIfEmpty,
-  undefinedIfFalsy
-} from '../../../utils/helpers/validation';
 
 export default class Thedig implements Engine {
   readonly name = 'thedig';
@@ -111,11 +111,11 @@ export default class Thedig implements Engine {
         sameAs: undefinedIfEmpty(person.sameAs ?? []),
         identifiers: undefinedIfEmpty(person.identifier ?? []),
         alternateName: undefinedIfEmpty(person.alternateName ?? []),
-        location: undefinedIfEmpty(
-          [
-            [person.homeLocation].flat().filter(Boolean).join(','),
-            [person.workLocation].flat().filter(Boolean).join(',')
-          ].flat()
+        location: undefinedIfFalsy(
+          [person.homeLocation, person.workLocation]
+            .flat()
+            .filter(Boolean)
+            .join(', ')
         )
       }))
       .filter(
diff --git a/backend/src/services/enrichment/voilanorbert/index.ts b/backend/src/services/enrichment/voilanorbert/index.ts
@@ -74,7 +74,7 @@ export default class Voilanorbert implements Engine {
         name: undefinedIfFalsy(result.fullName),
         organization: undefinedIfFalsy(result.organization),
         jobTitle: undefinedIfFalsy(result.title),
-        location: undefinedIfEmpty([result.location]),
+        location: undefinedIfFalsy(result.location),
         sameAs: undefinedIfEmpty([
           result.facebook,
           result.linkedin,
diff --git a/backend/src/services/extractors/engines/FileImport.ts b/backend/src/services/extractors/engines/FileImport.ts
@@ -83,7 +83,7 @@ export class CsvXlsxContactEngine {
       familyName: undefinedIfFalsy(familyName ?? ''),
       jobTitle: undefinedIfFalsy(jobTitle ?? ''),
       image: undefinedIfFalsy(image ?? ''),
-      location: undefinedIfEmpty(location?.split(',') ?? []),
+      location: undefinedIfFalsy(location ?? ''),
       sameAs: undefinedIfEmpty(sameAs?.split(',') ?? []),
       worksFor: undefinedIfFalsy(worksFor ?? ''),
       alternateName: undefinedIfEmpty(alternateName?.split(',') ?? [])
diff --git a/backend/src/services/signature/llm/index.ts b/backend/src/services/signature/llm/index.ts
@@ -1,12 +1,13 @@
-import { Logger } from 'winston';
 import assert from 'assert';
-import { IRateLimiter } from '../../rate-limiter/RateLimiter';
-import { ExtractSignature, PersonLD } from '../types';
+import { Logger } from 'winston';
 import {
-  undefinedIfFalsy,
-  undefinedIfEmpty
+  undefinedIfEmpty,
+  undefinedIfFalsy
 } from '../../../utils/helpers/validation';
+import { IRateLimiter } from '../../rate-limiter/RateLimiter';
+import { ExtractSignature, PersonLD } from '../types';
 import {
+  parseLocationString,
   parseString,
   parseStringArray,
   removeFalsePositives,
@@ -254,7 +255,7 @@ export class SignatureLLM implements ExtractSignature {
         // name: undefinedIfFalsy(parseString(person.name)),
         jobTitle: undefinedIfFalsy(parseString(person.jobTitle)),
         worksFor: undefinedIfFalsy(parseString(person.worksFor)),
-        address: undefinedIfEmpty(parseStringArray(person.address) ?? []),
+        address: undefinedIfFalsy(parseLocationString(person.address)),
         telephone: undefinedIfEmpty(
           validatePhones(signature, parseStringArray(person.telephone) ?? [])
         ),
diff --git a/backend/src/services/signature/llm/output-checkers.ts b/backend/src/services/signature/llm/output-checkers.ts
@@ -73,6 +73,20 @@ export function parseStringArray(value: unknown): string[] | undefined {
   return cleaned.length ? [...new Set(cleaned)] : undefined;
 }
 
+export function parseLocationString(value: unknown): string | undefined {
+  if (!value) return undefined;
+
+  const combined = Array.isArray(value) ? value.join(', ') : String(value);
+
+  return (
+    combined
+      .replace(/\\n|\n/g, ', ') // replace literal or real newlines with ", "
+      .replace(/\s*,\s*/g, ', ') // normalize commas and spaces
+      .replace(/\s{2,}/g, ' ') // collapse multiple spaces
+      .trim() || undefined
+  );
+}
+
 export function removeFalsePositives(
   person: PersonLD,
   signature: string,
diff --git a/backend/src/services/signature/types.ts b/backend/src/services/signature/types.ts
@@ -3,7 +3,7 @@ export interface PersonLD {
   image?: string;
   jobTitle?: string;
   worksFor?: string;
-  address?: string[];
+  address?: string;
   telephone?: string[];
   sameAs?: string[];
 }
diff --git a/backend/src/utils/helpers/csv.ts b/backend/src/utils/helpers/csv.ts
@@ -64,7 +64,7 @@ export function exportContactsToCSV(contacts: Contact[], csvSeparator = ',') {
     given_name: contact.given_name,
     family_name: contact.family_name,
     alternate_name: contact.alternate_name?.join(','),
-    location: contact.location?.join(','),
+    location: contact.location,
     works_for: contact.works_for,
     job_title: contact.job_title,
     same_as: contact.same_as?.join(','),
diff --git a/backend/src/workers/email-signature/handler.ts b/backend/src/workers/email-signature/handler.ts
@@ -275,7 +275,7 @@ export class EmailSignatureProcessor {
       family_name: contact.family_name ?? null,
       works_for: contact.works_for ?? null,
       same_as: (contact.same_as ?? []).join(','),
-      location: (contact.location ?? []).join(','),
+      location: contact.location ?? null,
       alternate_name: contact.alternate_name ?? null,
       telephone: Array.isArray(contact.telephone)
         ? contact.telephone.join(',')
diff --git a/backend/test/unit/database/enrichments.test.ts b/backend/test/unit/database/enrichments.test.ts
@@ -109,7 +109,7 @@ describe('Enrichments Class', () => {
             'https://linkedin.com/in/johndoe',
             'https://github.com/johndoe'
           ],
-          location: ['San Francisco, CA', 'USA'],
+          location: 'San Francisco, CA, USA',
           alternateName: ['Johnny', 'J.D.']
         }
       ];
@@ -133,7 +133,7 @@ describe('Enrichments Class', () => {
             image: 'https://example.com/image.jpg',
             same_as:
               'https://linkedin.com/in/johndoe,https://github.com/johndoe',
-            location: 'San Francisco, CA,USA'
+            location: 'San Francisco, CA, USA'
           }
         ],
         p_update_empty_fields_only: true
diff --git a/backend/test/unit/enrichment-service/enrichLayer/enricher.test.ts b/backend/test/unit/enrichment-service/enrichLayer/enricher.test.ts
@@ -181,7 +181,7 @@ describe('EnrichLayer', () => {
             organization: 'Tech Inc.',
             image: 'https://example.com/pic.jpg',
             identifiers: ['john.doe'],
-            location: ['New York, NY, United States'],
+            location: 'New York, NY, United States',
             sameAs: [
               'https://linkedin.com/in/johndoe',
               'https://facebook.com/johndoe',
@@ -235,7 +235,7 @@ describe('EnrichLayer', () => {
             familyName: 'User',
             identifiers: ['partial2.user'],
             sameAs: ['https://linkedin.com/in/partial2user'],
-            location: ['city, state'],
+            location: 'city, state',
             image: undefined,
             jobTitle: undefined,
             organization: undefined,
@@ -316,7 +316,7 @@ describe('EnrichLayer', () => {
             organization: 'Tech Inc.',
             image: 'https://example.com/pic.jpg',
             identifiers: ['john.doe'],
-            location: ['New York, NY, United States'],
+            location: 'New York, NY, United States',
             sameAs: [
               'https://linkedin.com/in/johndoe',
               'https://facebook.com/johndoe',
diff --git a/backend/test/unit/enrichment-service/thedig/enricher.test.ts b/backend/test/unit/enrichment-service/thedig/enricher.test.ts
@@ -21,8 +21,8 @@ describe('Thedig', () => {
     jobTitle: ['Software Engineer'],
     worksFor: ['Tech Co.'],
     image: ['https://example.com/pic.jpg'],
-    homeLocation: ['City, State'],
-    workLocation: ['City, State'],
+    homeLocation: 'City, State',
+    workLocation: 'City, State',
     alternateName: ['JDoe'],
     sameAs: ['https://linkedin.com/in/janedoe'],
     identifier: ['jane.doe'],
@@ -41,7 +41,7 @@ describe('Thedig', () => {
         organization: 'Tech Co.',
         image: 'https://example.com/pic.jpg',
         identifiers: ['jane.doe'],
-        location: ['City, State', 'City, State'],
+        location: 'City, State, City, State',
         sameAs: ['https://linkedin.com/in/janedoe']
       }
     ],
@@ -83,8 +83,8 @@ describe('Thedig', () => {
             'https://x.com/validuser/profile/pic.png',
             'https://linkedin.com/in/validuser/pic.png'
           ],
-          homeLocation: ['country, city'],
-          workLocation: ['country, city'],
+          homeLocation: 'country, city',
+          workLocation: 'country, city',
           alternateName: ['valid_user_123'],
           sameAs: [
             'https://linkedin.com/in/validuser',
@@ -99,7 +99,7 @@ describe('Thedig', () => {
           givenName: '',
           familyName: 'User',
           worksFor: ['Leadminer'],
-          workLocation: ['country, city'],
+          workLocation: 'country, city',
           sameAs: [
             'https://linkedin.com/in/partialuser',
             'https://x.com/partialuser/'
@@ -127,7 +127,7 @@ describe('Thedig', () => {
             jobTitle: 'Software Engineer',
             organization: 'Leadminer',
             identifiers: ['valid.user'],
-            location: ['country, city', 'country, city'],
+            location: 'country, city, country, city',
             sameAs: [
               'https://linkedin.com/in/validuser',
               'https://x.com/validuser/'
@@ -143,7 +143,7 @@ describe('Thedig', () => {
             jobTitle: undefined,
             organization: 'Leadminer',
             identifiers: undefined,
-            location: ['country, city'],
+            location: 'country, city',
             sameAs: [
               'https://linkedin.com/in/partialuser',
               'https://x.com/partialuser/'
diff --git a/backend/test/unit/enrichment-service/voilanorbert/enricher.test.ts b/backend/test/unit/enrichment-service/voilanorbert/enricher.test.ts
@@ -144,7 +144,7 @@ describe('Voilanorbert', () => {
           {
             email: 'valid@example.com',
             name: 'Valid User',
-            location: ['country, city'],
+            location: 'country, city',
             organization: 'Leadminer',
             jobTitle: 'Senior Developer',
             sameAs: [
@@ -157,7 +157,7 @@ describe('Voilanorbert', () => {
           {
             email: 'partial@example.com',
             name: 'Partial User',
-            location: ['country, city'],
+            location: 'country, city',
             organization: 'Leadminer',
             jobTitle: 'Intern',
             sameAs: ['facebook.com/partialUser', 'linkedin.com/in/partialUser'],
diff --git a/frontend/src/components/Mining/ContactInformationSidebar.vue b/frontend/src/components/Mining/ContactInformationSidebar.vue
diff --git a/frontend/src/components/Mining/Table/MiningTable.vue b/frontend/src/components/Mining/Table/MiningTable.vue
diff --git a/frontend/src/stores/contacts.ts b/frontend/src/stores/contacts.ts
diff --git a/frontend/src/types/contact.ts b/frontend/src/types/contact.ts
diff --git a/supabase/migrations/20251105175657_location_text.sql b/supabase/migrations/20251105175657_location_text.sql

Original file line number	Diff line number	Diff line change
`@@ -81,15 +81,15 @@ export default class EnrichLayer implements Engine {`
`81`	`81`	`Boolean(id)`
`82`	`82`	`)`
`83`	`83`	`),`
`84`		`- location: undefinedIfEmpty([`
	`84`	`+ location: undefinedIfFalsy(`
`85`	`85`	`[`
`86`	`86`	`response?.profile?.city,`
`87`	`87`	`response?.profile?.state,`
`88`	`88`	`response?.profile?.country_full_name`
`89`	`89`	`]`
`90`	`90`	`.filter((loc): loc is string => Boolean(loc))`
`91`	`91`	`.join(', ')`
`92`		`- ]),`
	`92`	`+ ),`
`93`	`93`	`sameAs: undefinedIfEmpty([`
`94`	`94`	`response?.linkedin_profile_url,`
`95`	`95`	`response?.facebook_profile_url,`