|
| 1 | +-- Knowledge cleanup migration: strip UUIDs from titles, merge duplicates, |
| 2 | +-- fix null/invalid project names. |
| 3 | + |
| 4 | +-- 1. Strip UUID suffixes from knowledge titles. |
| 5 | +-- Pattern: optional whitespace + hex UUID (8-4-4+ with optional trailing segments). |
| 6 | +UPDATE global_knowledge |
| 7 | +SET title = TRIM(REGEXP_REPLACE(title, '\s*[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4,}(-[0-9a-f]*)*\s*', '', 'gi')), |
| 8 | + updated_at = NOW() |
| 9 | +WHERE title ~ '[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4,}'; |
| 10 | + |
| 11 | +-- 2. Merge duplicate knowledge entries (same lowercased title after UUID stripping). |
| 12 | +-- Keep the entry with highest usage_count (ties broken by earliest created_at). |
| 13 | +-- Sum usage_counts, merge source_observations arrays, take highest confidence. |
| 14 | +DO $$ |
| 15 | +DECLARE |
| 16 | + dup RECORD; |
| 17 | + keeper_id TEXT; |
| 18 | + merged_usage BIGINT; |
| 19 | + merged_obs JSONB; |
| 20 | + max_confidence FLOAT8; |
| 21 | +BEGIN |
| 22 | + -- Find groups of duplicates (2+ entries with same normalized title) |
| 23 | + FOR dup IN |
| 24 | + SELECT LOWER(TRIM(title)) AS norm_title, |
| 25 | + COUNT(*) AS cnt |
| 26 | + FROM global_knowledge |
| 27 | + WHERE archived_at IS NULL |
| 28 | + GROUP BY LOWER(TRIM(title)) |
| 29 | + HAVING COUNT(*) > 1 |
| 30 | + LOOP |
| 31 | + -- Determine the keeper: highest usage_count, then earliest created_at |
| 32 | + SELECT id INTO keeper_id |
| 33 | + FROM global_knowledge |
| 34 | + WHERE LOWER(TRIM(title)) = dup.norm_title AND archived_at IS NULL |
| 35 | + ORDER BY usage_count DESC, created_at ASC |
| 36 | + LIMIT 1; |
| 37 | + |
| 38 | + -- Aggregate usage_count, source_observations, and max confidence from all duplicates |
| 39 | + SELECT COALESCE(SUM(usage_count), 0), |
| 40 | + COALESCE( |
| 41 | + (SELECT jsonb_agg(DISTINCT elem) |
| 42 | + FROM global_knowledge g2, |
| 43 | + jsonb_array_elements(g2.source_observations) AS elem |
| 44 | + WHERE LOWER(TRIM(g2.title)) = dup.norm_title |
| 45 | + AND g2.archived_at IS NULL |
| 46 | + AND elem != 'null'::jsonb), |
| 47 | + '[]'::jsonb |
| 48 | + ), |
| 49 | + COALESCE(MAX(confidence), 0.5) |
| 50 | + INTO merged_usage, merged_obs, max_confidence |
| 51 | + FROM global_knowledge |
| 52 | + WHERE LOWER(TRIM(title)) = dup.norm_title AND archived_at IS NULL; |
| 53 | + |
| 54 | + UPDATE global_knowledge |
| 55 | + SET usage_count = merged_usage, |
| 56 | + source_observations = merged_obs, |
| 57 | + confidence = max_confidence, |
| 58 | + updated_at = NOW() |
| 59 | + WHERE id = keeper_id; |
| 60 | + |
| 61 | + DELETE FROM global_knowledge |
| 62 | + WHERE LOWER(TRIM(title)) = dup.norm_title |
| 63 | + AND archived_at IS NULL |
| 64 | + AND id != keeper_id; |
| 65 | + END LOOP; |
| 66 | +END $$; |
| 67 | + |
| 68 | +-- 3. Fix NULL project in observations — set to 'unknown'. |
| 69 | +UPDATE observations |
| 70 | +SET project = 'unknown' |
| 71 | +WHERE project IS NULL; |
| 72 | + |
| 73 | +-- 4. Fix 'ivan plankin' project name (personal name, not a project). |
| 74 | +UPDATE observations |
| 75 | +SET project = 'unknown' |
| 76 | +WHERE LOWER(project) = 'ivan plankin'; |
0 commit comments