From 74d088a01d37f106af78ecdc980c63815d1f29d5 Mon Sep 17 00:00:00 2001 From: Leonardo Rivera Date: Fri, 9 Jan 2026 11:10:18 -0500 Subject: [PATCH 01/33] database changes --- .../migrations/0010_dictionary_migration.sql | 45 ++ .../migrations/meta/0010_snapshot.json | 690 ++++++++++++++++++ .../data-model/migrations/meta/_journal.json | 9 +- .../src/models/audit_submitted_data.ts | 7 +- .../src/models/dictionary_migration.ts | 55 ++ packages/data-model/src/models/index.ts | 1 + .../docs/dictionary-registration.md | 10 +- .../src/repository/auditRepository.ts | 1 + .../dictionaryMigrationRepository.ts | 88 +++ packages/data-provider/src/utils/types.ts | 5 +- 10 files changed, 904 insertions(+), 7 deletions(-) create mode 100644 packages/data-model/migrations/0010_dictionary_migration.sql create mode 100644 packages/data-model/migrations/meta/0010_snapshot.json create mode 100644 packages/data-model/src/models/dictionary_migration.ts create mode 100644 packages/data-provider/src/repository/dictionaryMigrationRepository.ts diff --git a/packages/data-model/migrations/0010_dictionary_migration.sql b/packages/data-model/migrations/0010_dictionary_migration.sql new file mode 100644 index 00000000..2d873c9b --- /dev/null +++ b/packages/data-model/migrations/0010_dictionary_migration.sql @@ -0,0 +1,45 @@ +DO $$ BEGIN + CREATE TYPE "migration_status" AS ENUM('IN-PROGRESS', 'COMPLETED', 'FAILED'); +EXCEPTION + WHEN duplicate_object THEN null; +END $$; +--> statement-breakpoint +ALTER TYPE "audit_action" ADD VALUE 'MIGRATION';--> statement-breakpoint +CREATE TABLE IF NOT EXISTS "dictionary_migration" ( + "id" serial PRIMARY KEY NOT NULL, + "category_id" integer NOT NULL, + "from_dictionary_id" integer NOT NULL, + "to_dictionary_id" integer NOT NULL, + "submission_id" integer NOT NULL, + "status" "migration_status" NOT NULL, + "retries" integer DEFAULT 0 NOT NULL, + "created_at" timestamp, + "created_by" varchar, + "updated_at" timestamp, + "updated_by" varchar +); +--> statement-breakpoint +ALTER TABLE "audit_submitted_data" ADD COLUMN "errors" jsonb;--> statement-breakpoint +DO $$ BEGIN + ALTER TABLE "dictionary_migration" ADD CONSTRAINT "dictionary_migration_category_id_dictionary_categories_id_fk" FOREIGN KEY ("category_id") REFERENCES "dictionary_categories"("id") ON DELETE no action ON UPDATE no action; +EXCEPTION + WHEN duplicate_object THEN null; +END $$; +--> statement-breakpoint +DO $$ BEGIN + ALTER TABLE "dictionary_migration" ADD CONSTRAINT "dictionary_migration_from_dictionary_id_dictionaries_id_fk" FOREIGN KEY ("from_dictionary_id") REFERENCES "dictionaries"("id") ON DELETE no action ON UPDATE no action; +EXCEPTION + WHEN duplicate_object THEN null; +END $$; +--> statement-breakpoint +DO $$ BEGIN + ALTER TABLE "dictionary_migration" ADD CONSTRAINT "dictionary_migration_to_dictionary_id_dictionaries_id_fk" FOREIGN KEY ("to_dictionary_id") REFERENCES "dictionaries"("id") ON DELETE no action ON UPDATE no action; +EXCEPTION + WHEN duplicate_object THEN null; +END $$; +--> statement-breakpoint +DO $$ BEGIN + ALTER TABLE "dictionary_migration" ADD CONSTRAINT "dictionary_migration_submission_id_submissions_id_fk" FOREIGN KEY ("submission_id") REFERENCES "submissions"("id") ON DELETE no action ON UPDATE no action; +EXCEPTION + WHEN duplicate_object THEN null; +END $$; diff --git a/packages/data-model/migrations/meta/0010_snapshot.json b/packages/data-model/migrations/meta/0010_snapshot.json new file mode 100644 index 00000000..bd4c7f49 --- /dev/null +++ b/packages/data-model/migrations/meta/0010_snapshot.json @@ -0,0 +1,690 @@ +{ + "id": "a18a6180-ffa9-44cc-a1f6-00bb5d03cd3b", + "prevId": "9d7d98fa-e067-4891-a2c1-65a19d6430f3", + "version": "5", + "dialect": "pg", + "tables": { + "audit_submitted_data": { + "name": "audit_submitted_data", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "action": { + "name": "action", + "type": "audit_action", + "primaryKey": false, + "notNull": true + }, + "dictionary_category_id": { + "name": "dictionary_category_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "data_diff": { + "name": "data_diff", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "entity_name": { + "name": "entity_name", + "type": "varchar", + "primaryKey": false, + "notNull": true + }, + "errors": { + "name": "errors", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "last_valid_schema_id": { + "name": "last_valid_schema_id", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "new_data_is_valid": { + "name": "new_data_is_valid", + "type": "boolean", + "primaryKey": false, + "notNull": true + }, + "old_data_is_valid": { + "name": "old_data_is_valid", + "type": "boolean", + "primaryKey": false, + "notNull": true + }, + "organization": { + "name": "organization", + "type": "varchar", + "primaryKey": false, + "notNull": true + }, + "original_schema_id": { + "name": "original_schema_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "submission_id": { + "name": "submission_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "system_id": { + "name": "system_id", + "type": "varchar", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_by": { + "name": "created_by", + "type": "varchar", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": { + "audit_submitted_data_dictionary_category_id_dictionary_categories_id_fk": { + "name": "audit_submitted_data_dictionary_category_id_dictionary_categories_id_fk", + "tableFrom": "audit_submitted_data", + "tableTo": "dictionary_categories", + "columnsFrom": [ + "dictionary_category_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "audit_submitted_data_last_valid_schema_id_dictionaries_id_fk": { + "name": "audit_submitted_data_last_valid_schema_id_dictionaries_id_fk", + "tableFrom": "audit_submitted_data", + "tableTo": "dictionaries", + "columnsFrom": [ + "last_valid_schema_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "audit_submitted_data_original_schema_id_dictionaries_id_fk": { + "name": "audit_submitted_data_original_schema_id_dictionaries_id_fk", + "tableFrom": "audit_submitted_data", + "tableTo": "dictionaries", + "columnsFrom": [ + "original_schema_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "audit_submitted_data_submission_id_submissions_id_fk": { + "name": "audit_submitted_data_submission_id_submissions_id_fk", + "tableFrom": "audit_submitted_data", + "tableTo": "submissions", + "columnsFrom": [ + "submission_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "dictionaries": { + "name": "dictionaries", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "dictionary": { + "name": "dictionary", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "varchar", + "primaryKey": false, + "notNull": true + }, + "version": { + "name": "version", + "type": "varchar", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "created_by": { + "name": "created_by", + "type": "varchar", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "dictionary_categories": { + "name": "dictionary_categories", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "active_dictionary_id": { + "name": "active_dictionary_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "default_centric_entity": { + "name": "default_centric_entity", + "type": "varchar", + "primaryKey": false, + "notNull": false + }, + "name": { + "name": "name", + "type": "varchar", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "created_by": { + "name": "created_by", + "type": "varchar", + "primaryKey": false, + "notNull": false + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "updated_by": { + "name": "updated_by", + "type": "varchar", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "dictionary_categories_name_unique": { + "name": "dictionary_categories_name_unique", + "nullsNotDistinct": false, + "columns": [ + "name" + ] + } + } + }, + "dictionary_migration": { + "name": "dictionary_migration", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "category_id": { + "name": "category_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "from_dictionary_id": { + "name": "from_dictionary_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "to_dictionary_id": { + "name": "to_dictionary_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "submission_id": { + "name": "submission_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "migration_status", + "primaryKey": false, + "notNull": true + }, + "retries": { + "name": "retries", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_by": { + "name": "created_by", + "type": "varchar", + "primaryKey": false, + "notNull": false + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "updated_by": { + "name": "updated_by", + "type": "varchar", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": { + "dictionary_migration_category_id_dictionary_categories_id_fk": { + "name": "dictionary_migration_category_id_dictionary_categories_id_fk", + "tableFrom": "dictionary_migration", + "tableTo": "dictionary_categories", + "columnsFrom": [ + "category_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "dictionary_migration_from_dictionary_id_dictionaries_id_fk": { + "name": "dictionary_migration_from_dictionary_id_dictionaries_id_fk", + "tableFrom": "dictionary_migration", + "tableTo": "dictionaries", + "columnsFrom": [ + "from_dictionary_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "dictionary_migration_to_dictionary_id_dictionaries_id_fk": { + "name": "dictionary_migration_to_dictionary_id_dictionaries_id_fk", + "tableFrom": "dictionary_migration", + "tableTo": "dictionaries", + "columnsFrom": [ + "to_dictionary_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "dictionary_migration_submission_id_submissions_id_fk": { + "name": "dictionary_migration_submission_id_submissions_id_fk", + "tableFrom": "dictionary_migration", + "tableTo": "submissions", + "columnsFrom": [ + "submission_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "submissions": { + "name": "submissions", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "data": { + "name": "data", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "dictionary_category_id": { + "name": "dictionary_category_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "dictionary_id": { + "name": "dictionary_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "errors": { + "name": "errors", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "organization": { + "name": "organization", + "type": "varchar", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "submission_status", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "created_by": { + "name": "created_by", + "type": "varchar", + "primaryKey": false, + "notNull": false + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "updated_by": { + "name": "updated_by", + "type": "varchar", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": { + "submissions_dictionary_category_id_dictionary_categories_id_fk": { + "name": "submissions_dictionary_category_id_dictionary_categories_id_fk", + "tableFrom": "submissions", + "tableTo": "dictionary_categories", + "columnsFrom": [ + "dictionary_category_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "submissions_dictionary_id_dictionaries_id_fk": { + "name": "submissions_dictionary_id_dictionaries_id_fk", + "tableFrom": "submissions", + "tableTo": "dictionaries", + "columnsFrom": [ + "dictionary_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "submitted_data": { + "name": "submitted_data", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "data": { + "name": "data", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "dictionary_category_id": { + "name": "dictionary_category_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "entity_name": { + "name": "entity_name", + "type": "varchar", + "primaryKey": false, + "notNull": true + }, + "is_valid": { + "name": "is_valid", + "type": "boolean", + "primaryKey": false, + "notNull": true + }, + "last_valid_schema_id": { + "name": "last_valid_schema_id", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "organization": { + "name": "organization", + "type": "varchar", + "primaryKey": false, + "notNull": true + }, + "original_schema_id": { + "name": "original_schema_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "system_id": { + "name": "system_id", + "type": "varchar", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "created_by": { + "name": "created_by", + "type": "varchar", + "primaryKey": false, + "notNull": false + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "updated_by": { + "name": "updated_by", + "type": "varchar", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "organization_index": { + "name": "organization_index", + "columns": [ + "organization" + ], + "isUnique": false + } + }, + "foreignKeys": { + "submitted_data_dictionary_category_id_dictionary_categories_id_fk": { + "name": "submitted_data_dictionary_category_id_dictionary_categories_id_fk", + "tableFrom": "submitted_data", + "tableTo": "dictionary_categories", + "columnsFrom": [ + "dictionary_category_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "submitted_data_last_valid_schema_id_dictionaries_id_fk": { + "name": "submitted_data_last_valid_schema_id_dictionaries_id_fk", + "tableFrom": "submitted_data", + "tableTo": "dictionaries", + "columnsFrom": [ + "last_valid_schema_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "submitted_data_original_schema_id_dictionaries_id_fk": { + "name": "submitted_data_original_schema_id_dictionaries_id_fk", + "tableFrom": "submitted_data", + "tableTo": "dictionaries", + "columnsFrom": [ + "original_schema_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "submitted_data_system_id_unique": { + "name": "submitted_data_system_id_unique", + "nullsNotDistinct": false, + "columns": [ + "system_id" + ] + } + } + } + }, + "enums": { + "audit_action": { + "name": "audit_action", + "values": { + "UPDATE": "UPDATE", + "DELETE": "DELETE", + "MIGRATION": "MIGRATION" + } + }, + "migration_status": { + "name": "migration_status", + "values": { + "IN-PROGRESS": "IN-PROGRESS", + "COMPLETED": "COMPLETED", + "FAILED": "FAILED" + } + }, + "submission_status": { + "name": "submission_status", + "values": { + "OPEN": "OPEN", + "VALID": "VALID", + "INVALID": "INVALID", + "CLOSED": "CLOSED", + "COMMITTED": "COMMITTED" + } + } + }, + "schemas": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} \ No newline at end of file diff --git a/packages/data-model/migrations/meta/_journal.json b/packages/data-model/migrations/meta/_journal.json index 54143ff5..720b62a1 100644 --- a/packages/data-model/migrations/meta/_journal.json +++ b/packages/data-model/migrations/meta/_journal.json @@ -71,6 +71,13 @@ "when": 1730128901514, "tag": "0009_add_default_centric_entity", "breakpoints": true + }, + { + "idx": 10, + "version": "5", + "when": 1767901182564, + "tag": "0010_dictionary_migration", + "breakpoints": true } ] -} \ No newline at end of file +} diff --git a/packages/data-model/src/models/audit_submitted_data.ts b/packages/data-model/src/models/audit_submitted_data.ts index 3ab5d6a1..97197039 100644 --- a/packages/data-model/src/models/audit_submitted_data.ts +++ b/packages/data-model/src/models/audit_submitted_data.ts @@ -5,15 +5,17 @@ import type { DataRecord } from '@overture-stack/lectern-client'; import { dictionaries } from './dictionaries.js'; import { dictionaryCategories } from './dictionary_categories.js'; -import { submissions } from './submissions.js'; +import { type SubmissionRecordErrorDetails, submissions } from './submissions.js'; -export const audit_action = pgEnum('audit_action', ['UPDATE', 'DELETE']); +export const audit_action = pgEnum('audit_action', ['UPDATE', 'DELETE', 'MIGRATION']); export type DataDiff = { old: DataRecord; new: DataRecord; }; +export type AuditErrors = Record; + export const auditSubmittedData = pgTable('audit_submitted_data', { id: serial('id').primaryKey(), action: audit_action('action').notNull(), @@ -22,6 +24,7 @@ export const auditSubmittedData = pgTable('audit_submitted_data', { .notNull(), dataDiff: jsonb('data_diff').$type(), entityName: varchar('entity_name').notNull(), + errors: jsonb('errors').$type(), lastValidSchemaId: integer('last_valid_schema_id').references(() => dictionaries.id), newDataIsValid: boolean('new_data_is_valid').notNull(), oldDataIsValid: boolean('old_data_is_valid').notNull(), diff --git a/packages/data-model/src/models/dictionary_migration.ts b/packages/data-model/src/models/dictionary_migration.ts new file mode 100644 index 00000000..8e26e13b --- /dev/null +++ b/packages/data-model/src/models/dictionary_migration.ts @@ -0,0 +1,55 @@ +import { relations } from 'drizzle-orm'; +import { integer, pgEnum, pgTable, serial, timestamp, varchar } from 'drizzle-orm/pg-core'; + +import { dictionaries } from './dictionaries.js'; +import { dictionaryCategories } from './dictionary_categories.js'; +import { submissions } from './submissions.js'; + +export const migration_status = pgEnum('migration_status', ['IN-PROGRESS', 'COMPLETED', 'FAILED']); + +export const dictionaryMigration = pgTable('dictionary_migration', { + id: serial('id').primaryKey(), + categoryId: integer('category_id') + .references(() => dictionaryCategories.id) + .notNull(), + fromDictionaryId: integer('from_dictionary_id') + .references(() => dictionaries.id) + .notNull(), + toDictionaryId: integer('to_dictionary_id') + .references(() => dictionaries.id) + .notNull(), + submissionId: integer('submission_id') + .references(() => submissions.id) + .notNull(), + status: migration_status('status').notNull(), + retries: integer('retries').notNull().default(0), + createdAt: timestamp('created_at'), + createdBy: varchar('created_by'), + updatedAt: timestamp('updated_at'), + updatedBy: varchar('updated_by'), +}); + +export const dictionaryMigrationRelations = relations(dictionaryMigration, ({ one }) => ({ + category: one(dictionaryCategories, { + fields: [dictionaryMigration.categoryId], + references: [dictionaryCategories.id], + }), + fromDictionaryId: one(dictionaries, { + fields: [dictionaryMigration.fromDictionaryId], + references: [dictionaries.id], + relationName: 'fromDictionary', + }), + toDictionaryId: one(dictionaries, { + fields: [dictionaryMigration.toDictionaryId], + references: [dictionaries.id], + relationName: 'toDictionary', + }), + submission: one(submissions, { + fields: [dictionaryMigration.submissionId], + references: [submissions.id], + relationName: 'submission', + }), +})); + +export type DictionaryMigration = typeof dictionaryMigration.$inferSelect; // return type when queried +export type NewDictionaryMigration = typeof dictionaryMigration.$inferInsert; // insert type diff --git a/packages/data-model/src/models/index.ts b/packages/data-model/src/models/index.ts index c3236f33..b83e5ae4 100644 --- a/packages/data-model/src/models/index.ts +++ b/packages/data-model/src/models/index.ts @@ -1,5 +1,6 @@ export * from './audit_submitted_data.js'; export * from './dictionaries.js'; export * from './dictionary_categories.js'; +export * from './dictionary_migration.js'; export * from './submissions.js'; export * from './submitted_data.js'; diff --git a/packages/data-provider/docs/dictionary-registration.md b/packages/data-provider/docs/dictionary-registration.md index fe763057..0b4a393b 100644 --- a/packages/data-provider/docs/dictionary-registration.md +++ b/packages/data-provider/docs/dictionary-registration.md @@ -70,7 +70,7 @@ sequenceDiagram rect rgb(230, 242, 255) note over LyricAPI: [Migration] Initiate migration - LyricAPI->>LyricDB: Create a migration in `category_migration` table
(categoryId, fromDictionaryId, toDictionaryId, submissionId, status=INPROGRESS, createdAt, createdBy) + LyricAPI->>LyricDB: Create a migration in `category_migration` table
(categoryId, fromDictionaryId, toDictionaryId, submissionId, status=IN-PROGRESS, createdAt, createdBy) LyricAPI->>LyricDB: Create a new submission (no data, status=COMMITTED) end end @@ -84,11 +84,15 @@ sequenceDiagram LyricAPI->>LyricAPI: Validate record against new schema loop each record failed validation - LyricAPI->>LyricDB: Update record in `submitted_data` table, with isValid = FALSE + LyricAPI->>LyricDB: Update record in `submitted_data` table, with isValid = FALSE, lastValidSchema note over LyricDB: triggers audit table LyricAPI->>LyricDB: Insert record in `audit_submitted_data` table with migration error end - LyricAPI->>LyricDB: Change Submission (`submissions` table) Status to COMMITED + + alt if Migration fails + LyricAPI->>LyricDB: Change Migration (`category_migration` table) Status to FAILED + end + LyricAPI->>LyricDB: Change Migration (`category_migration` table) Status to COMPLETED end end diff --git a/packages/data-provider/src/repository/auditRepository.ts b/packages/data-provider/src/repository/auditRepository.ts index 72811b42..d8e690e9 100644 --- a/packages/data-provider/src/repository/auditRepository.ts +++ b/packages/data-provider/src/repository/auditRepository.ts @@ -16,6 +16,7 @@ const repository = (dependencies: BaseDependencies) => { entityName: true, action: true, dataDiff: true, + errors: true, newDataIsValid: true, oldDataIsValid: true, organization: true, diff --git a/packages/data-provider/src/repository/dictionaryMigrationRepository.ts b/packages/data-provider/src/repository/dictionaryMigrationRepository.ts new file mode 100644 index 00000000..b7e8665a --- /dev/null +++ b/packages/data-provider/src/repository/dictionaryMigrationRepository.ts @@ -0,0 +1,88 @@ +import { and, eq } from 'drizzle-orm'; + +import { + type DictionaryMigration, + dictionaryMigration, + type NewDictionaryMigration, +} from '@overture-stack/lyric-data-model/models'; + +import type { BaseDependencies } from '../config/config.js'; +import { ServiceUnavailable } from '../utils/errors.js'; +import type { migration_status, PaginationOptions } from '../utils/types.js'; + +const repository = (dependencies: BaseDependencies) => { + const LOG_MODULE = 'MIGRATION_REPOSITORY'; + const { db, logger } = dependencies; + + return { + /** + * Save a new Dictionary Migration in Database + **/ + save: async (data: NewDictionaryMigration): Promise<{ id: number }> => { + try { + const savedMigration = await db + .insert(dictionaryMigration) + .values(data) + .returning({ id: dictionaryMigration.id }); + logger.debug(LOG_MODULE, `Dictionary Migration for categoryId '${data.categoryId}' saved successfully`); + return savedMigration[0]; + } catch (error) { + logger.error(LOG_MODULE, `Failed saving dictionary migration for categoryId '${data.categoryId}'`, error); + throw new ServiceUnavailable(); + } + }, + /** Update an existing Dictionary Migration in Database */ + update: async (migrationId: number, data: Partial): Promise => { + try { + await db.update(dictionaryMigration).set(data).where(eq(dictionaryMigration.id, migrationId)); + logger.debug(LOG_MODULE, `Dictionary Migration with id '${migrationId}' updated successfully`); + } catch (error) { + logger.error(LOG_MODULE, `Failed updating dictionary migration with id '${migrationId}'`, error); + throw new ServiceUnavailable(); + } + }, + /** Retrieve a Dictionary Migration by ID */ + getMigrationById: async (migrationId: number): Promise => { + try { + const migration = await db.query.dictionaryMigration.findFirst({ + where: eq(dictionaryMigration.id, migrationId), + }); + if (migration) { + logger.debug(LOG_MODULE, `Fetched migration with id '${migrationId}'`); + } else { + logger.debug(LOG_MODULE, `No migration found with id '${migrationId}'`); + } + return migration; + } catch (error) { + logger.error(LOG_MODULE, `Failed fetching dictionary migration with id '${migrationId}'`, error); + throw new ServiceUnavailable(); + } + }, + /** Retrieve Dictionary Migrations by Category ID with filter options */ + getMigrationsByCategoryId: async ( + categoryId: number, + paginationOptions: PaginationOptions, + filterOptions: { status?: migration_status }, + ): Promise => { + const { page, pageSize } = paginationOptions; + try { + const migrations = await db.query.dictionaryMigration.findMany({ + where: and( + eq(dictionaryMigration.categoryId, categoryId), + filterOptions.status ? eq(dictionaryMigration.status, filterOptions.status) : undefined, + ), + orderBy: (dictionaryMigration, { desc }) => desc(dictionaryMigration.createdAt), + limit: pageSize, + offset: (page - 1) * pageSize, + }); + logger.debug(LOG_MODULE, `Fetched ${migrations.length} migrations for categoryId '${categoryId}'`); + return migrations; + } catch (error) { + logger.error(LOG_MODULE, `Failed fetching dictionary migrations for categoryId '${categoryId}'`, error); + throw new ServiceUnavailable(); + } + }, + }; +}; + +export default repository; diff --git a/packages/data-provider/src/utils/types.ts b/packages/data-provider/src/utils/types.ts index 35484ee3..c46a04b0 100644 --- a/packages/data-provider/src/utils/types.ts +++ b/packages/data-provider/src/utils/types.ts @@ -33,10 +33,13 @@ export const SUBMISSION_STATUS = { } as const; export type SubmissionStatus = ObjectValues; +export const MIGRATION_STATUS = z.enum(['IN-PROGRESS', 'COMPLETED', 'FAILED']); +export type migration_status = z.infer; + /** * Enum matching Audit Action in database */ -export const AUDIT_ACTION = z.enum(['UPDATE', 'DELETE']); +export const AUDIT_ACTION = z.enum(['UPDATE', 'DELETE', 'MIGRATION']); export type AuditAction = z.infer; /** From 28f683a843b5800cf937bf67bb372ae05acaef79 Mon Sep 17 00:00:00 2001 From: Leonardo Rivera Date: Fri, 9 Jan 2026 11:15:37 -0500 Subject: [PATCH 02/33] update dbml --- packages/data-model/docs/schema.dbml | 36 +++++++++++++++------------- 1 file changed, 20 insertions(+), 16 deletions(-) diff --git a/packages/data-model/docs/schema.dbml b/packages/data-model/docs/schema.dbml index 346dc180..fdc99ba0 100644 --- a/packages/data-model/docs/schema.dbml +++ b/packages/data-model/docs/schema.dbml @@ -7,6 +7,7 @@ enum audit_action { enum migration_status { "IN-PROGRESS" COMPLETED + FAILED } enum submission_status { @@ -35,17 +36,6 @@ table audit_submitted_data { created_by varchar } -table category_migration { - id serial [pk, not null, increment] - dictionary_category_id integer [not null] - from_dictionary_id integer [not null] - to_dictionary_id integer [not null] - submission_id integer [not null] - status migration_status [not null] - created_at timestamp - created_by varchar -} - table dictionaries { id serial [pk, not null, increment] dictionary jsonb [not null] @@ -66,6 +56,20 @@ table dictionary_categories { updated_by varchar } +table dictionary_migration { + id serial [pk, not null, increment] + category_id integer [not null] + from_dictionary_id integer [not null] + to_dictionary_id integer [not null] + submission_id integer [not null] + status migration_status [not null] + retries integer [not null, default: 0] + created_at timestamp + created_by varchar + updated_at timestamp + updated_by varchar +} + table submissions { id serial [pk, not null, increment] data jsonb [not null] @@ -108,15 +112,15 @@ ref: audit_submitted_data.original_schema_id - dictionaries.id ref: audit_submitted_data.submission_id - submissions.id -ref: category_migration.dictionary_category_id - dictionary_categories.id +ref: dictionary_categories.active_dictionary_id - dictionaries.id -ref: category_migration.from_dictionary_id - dictionaries.id +ref: dictionary_migration.category_id - dictionary_categories.id -ref: category_migration.to_dictionary_id - dictionaries.id +ref: dictionary_migration.from_dictionary_id - dictionaries.id -ref: category_migration.submission_id - submissions.id +ref: dictionary_migration.to_dictionary_id - dictionaries.id -ref: dictionary_categories.active_dictionary_id - dictionaries.id +ref: dictionary_migration.submission_id - submissions.id ref: submissions.dictionary_id - dictionaries.id From e1ada636e26dd8af94bfaa0e24ab6fe8cbd56617 Mon Sep 17 00:00:00 2001 From: Leonardo Rivera Date: Fri, 9 Jan 2026 11:20:03 -0500 Subject: [PATCH 03/33] migration services --- apps/server/swagger/schemas.yml | 4 ++ .../src/controllers/dictionaryController.ts | 3 +- .../src/services/dictionaryService.ts | 7 +- .../src/services/migrationService.ts | 66 +++++++++++++++++++ .../src/services/submission/processor.ts | 4 +- packages/data-provider/src/utils/types.ts | 1 + 6 files changed, 81 insertions(+), 4 deletions(-) create mode 100644 packages/data-provider/src/services/migrationService.ts diff --git a/apps/server/swagger/schemas.yml b/apps/server/swagger/schemas.yml index 87c80c09..cb93225c 100644 --- a/apps/server/swagger/schemas.yml +++ b/apps/server/swagger/schemas.yml @@ -496,6 +496,10 @@ components: version: type: string description: Version of the Dictionary + migrationId: + type: string + description: ID of the Migration if applicable + allowEmptyValue: true SubmittedDataRecord: type: object diff --git a/packages/data-provider/src/controllers/dictionaryController.ts b/packages/data-provider/src/controllers/dictionaryController.ts index 8f052513..ecaefe69 100644 --- a/packages/data-provider/src/controllers/dictionaryController.ts +++ b/packages/data-provider/src/controllers/dictionaryController.ts @@ -29,7 +29,7 @@ const controller = (dependencies: BaseDependencies) => { `Register Dictionary Request categoryName '${categoryName}' name '${dictionaryName}' version '${dictionaryVersion}'`, ); - const { dictionary, category } = await dictionaryService.register({ + const { dictionary, category, migrationId } = await dictionaryService.register({ categoryName, dictionaryName, dictionaryVersion, @@ -45,6 +45,7 @@ const controller = (dependencies: BaseDependencies) => { dictionary: dictionary.dictionary, name: dictionary.name, version: dictionary.version, + migrationId: migrationId, }; return res.send(result); } catch (error) { diff --git a/packages/data-provider/src/services/dictionaryService.ts b/packages/data-provider/src/services/dictionaryService.ts index d39ae629..065bb2b0 100644 --- a/packages/data-provider/src/services/dictionaryService.ts +++ b/packages/data-provider/src/services/dictionaryService.ts @@ -7,6 +7,7 @@ import { BaseDependencies } from '../config/config.js'; import lecternClient from '../external/lecternClient.js'; import categoryRepository from '../repository/categoryRepository.js'; import dictionaryRepository from '../repository/dictionaryRepository.js'; +import { BadRequest } from '../utils/errors.js'; const dictionaryService = (dependencies: BaseDependencies) => { const LOG_MODULE = 'DICTIONARY_SERVICE'; @@ -103,7 +104,7 @@ const dictionaryService = (dependencies: BaseDependencies) => { dictionaryVersion: string; defaultCentricEntity?: string; username?: string; - }): Promise<{ dictionary: Dictionary; category: Category }> => { + }): Promise<{ dictionary: Dictionary; category: Category; migrationId?: number }> => { logger.debug( LOG_MODULE, `Register new dictionary categoryName '${categoryName}' dictionaryName '${dictionaryName}' dictionaryVersion '${dictionaryVersion}'`, @@ -115,7 +116,7 @@ const dictionaryService = (dependencies: BaseDependencies) => { if (defaultCentricEntity && !dictionary.schemas.some((schema) => schema.name === defaultCentricEntity)) { logger.error(LOG_MODULE, `Entity '${defaultCentricEntity}' does not exist in this dictionary`); - throw new Error(`Entity '${defaultCentricEntity}' does not exist in this dictionary`); + throw new BadRequest(`Entity '${defaultCentricEntity}' does not exist in this dictionary`); } const savedDictionary = await createDictionaryIfDoesNotExist( @@ -141,6 +142,8 @@ const dictionaryService = (dependencies: BaseDependencies) => { updatedBy: username, }); + // TODO: Handle Dictionary Migration process async here + logger.info( LOG_MODULE, `Category '${updatedCategory.name}' updated successfully with Dictionary '${savedDictionary.name}' version '${savedDictionary.version}'`, diff --git a/packages/data-provider/src/services/migrationService.ts b/packages/data-provider/src/services/migrationService.ts new file mode 100644 index 00000000..88ad2328 --- /dev/null +++ b/packages/data-provider/src/services/migrationService.ts @@ -0,0 +1,66 @@ +import type { DictionaryMigration, NewDictionaryMigration } from '@overture-stack/lyric-data-model/models'; + +import type { BaseDependencies } from '../config/config.js'; +import migrationRepository from '../repository/dictionaryMigrationRepository.js'; + +const migrationService = (dependencies: BaseDependencies) => { + const LOG_MODULE = 'MIGRATION_SERVICE'; + const { logger } = dependencies; + const migrationRepo = migrationRepository(dependencies); + + return { + getActiveMigrationByCategoryId: async (categoryId: number): Promise => { + try { + const migrations = await migrationRepo.getMigrationsByCategoryId( + categoryId, + { page: 1, pageSize: 1 }, + { status: 'IN-PROGRESS' }, + ); + if (migrations.length > 0) { + logger.info(LOG_MODULE, `Active migration found for categoryId '${categoryId}'`); + return migrations[0]; + } else { + logger.info(LOG_MODULE, `No active migration for categoryId '${categoryId}'`); + return null; + } + } catch (error) { + logger.error(LOG_MODULE, `Error retrieving active migration for categoryId '${categoryId}'`, error); + throw error; + } + }, + initiateMigration: async ({ + categoryId, + fromDictionaryId, + toDictionaryId, + submissionId, + userName, + }: { + categoryId: number; + fromDictionaryId: number; + toDictionaryId: number; + submissionId: number; + userName: string; + }): Promise<{ id: number }> => { + try { + // TODO: Check if there's already an active migration for this categoryId + // if it exists, increment retries, else if not exists create new migration + const newMigration: NewDictionaryMigration = { + categoryId, + fromDictionaryId, + toDictionaryId, + submissionId, + createdBy: userName, + status: 'IN-PROGRESS', + }; + const savedMigration = await migrationRepo.save(newMigration); + logger.info(LOG_MODULE, `Migration initiated for categoryId '${categoryId}'`); + return savedMigration; + } catch (error) { + logger.error(LOG_MODULE, `Error initiating migration for categoryId '${categoryId}'`, error); + throw error; + } + }, + }; +}; + +export default migrationService; diff --git a/packages/data-provider/src/services/submission/processor.ts b/packages/data-provider/src/services/submission/processor.ts index bd2833d8..0c3ffb70 100644 --- a/packages/data-provider/src/services/submission/processor.ts +++ b/packages/data-provider/src/services/submission/processor.ts @@ -227,7 +227,9 @@ const processor = (dependencies: BaseDependencies) => { // Step 1: Exclude items that are marked for deletion const systemIdsToDelete = new Set(dataToValidate?.deletes?.map((item) => item.systemId) || []); logger.info(LOG_MODULE, `Found '${systemIdsToDelete.size}' Records to delete on Submission '${submission.id}'`); - const submittedData = dataToValidate.submittedData?.filter((item) => !systemIdsToDelete.has(item.systemId)); + const submittedData = systemIdsToDelete.size + ? dataToValidate.submittedData?.filter((item) => !systemIdsToDelete.has(item.systemId)) + : dataToValidate.submittedData; // Step 2: Modify items marked for update const systemIdsToUpdate = new Set(dataToValidate.updates ? Object.keys(dataToValidate.updates) : []); diff --git a/packages/data-provider/src/utils/types.ts b/packages/data-provider/src/utils/types.ts index c46a04b0..e8b7be22 100644 --- a/packages/data-provider/src/utils/types.ts +++ b/packages/data-provider/src/utils/types.ts @@ -130,6 +130,7 @@ export type RegisterDictionaryResult = { dictionary: object; name: string; version: string; + migrationId?: number; }; export type { Schema, SchemasDictionary }; From c1f87dd4365b3e60767dab0b6b19d8a2c3b364a5 Mon Sep 17 00:00:00 2001 From: Leonardo Rivera Date: Tue, 13 Jan 2026 16:09:13 -0500 Subject: [PATCH 04/33] updates migration repository --- .../src/models/dictionary_migration.ts | 4 ++-- .../dictionaryMigrationRepository.ts | 24 +++++++++++++------ packages/data-provider/src/utils/types.ts | 2 +- 3 files changed, 20 insertions(+), 10 deletions(-) diff --git a/packages/data-model/src/models/dictionary_migration.ts b/packages/data-model/src/models/dictionary_migration.ts index 8e26e13b..17f7f0ce 100644 --- a/packages/data-model/src/models/dictionary_migration.ts +++ b/packages/data-model/src/models/dictionary_migration.ts @@ -5,7 +5,7 @@ import { dictionaries } from './dictionaries.js'; import { dictionaryCategories } from './dictionary_categories.js'; import { submissions } from './submissions.js'; -export const migration_status = pgEnum('migration_status', ['IN-PROGRESS', 'COMPLETED', 'FAILED']); +export const migrationStatusEnum = pgEnum('migration_status', ['IN-PROGRESS', 'COMPLETED', 'FAILED']); export const dictionaryMigration = pgTable('dictionary_migration', { id: serial('id').primaryKey(), @@ -21,7 +21,7 @@ export const dictionaryMigration = pgTable('dictionary_migration', { submissionId: integer('submission_id') .references(() => submissions.id) .notNull(), - status: migration_status('status').notNull(), + status: migrationStatusEnum('status').notNull(), retries: integer('retries').notNull().default(0), createdAt: timestamp('created_at'), createdBy: varchar('created_by'), diff --git a/packages/data-provider/src/repository/dictionaryMigrationRepository.ts b/packages/data-provider/src/repository/dictionaryMigrationRepository.ts index b7e8665a..32d933db 100644 --- a/packages/data-provider/src/repository/dictionaryMigrationRepository.ts +++ b/packages/data-provider/src/repository/dictionaryMigrationRepository.ts @@ -8,7 +8,7 @@ import { import type { BaseDependencies } from '../config/config.js'; import { ServiceUnavailable } from '../utils/errors.js'; -import type { migration_status, PaginationOptions } from '../utils/types.js'; +import type { MigrationStatus, PaginationOptions } from '../utils/types.js'; const repository = (dependencies: BaseDependencies) => { const LOG_MODULE = 'MIGRATION_REPOSITORY'; @@ -17,25 +17,31 @@ const repository = (dependencies: BaseDependencies) => { return { /** * Save a new Dictionary Migration in Database + * Returns the inserted record's ID **/ - save: async (data: NewDictionaryMigration): Promise<{ id: number }> => { + save: async (data: NewDictionaryMigration): Promise => { try { const savedMigration = await db .insert(dictionaryMigration) .values(data) .returning({ id: dictionaryMigration.id }); logger.debug(LOG_MODULE, `Dictionary Migration for categoryId '${data.categoryId}' saved successfully`); - return savedMigration[0]; + return savedMigration[0].id; } catch (error) { logger.error(LOG_MODULE, `Failed saving dictionary migration for categoryId '${data.categoryId}'`, error); throw new ServiceUnavailable(); } }, /** Update an existing Dictionary Migration in Database */ - update: async (migrationId: number, data: Partial): Promise => { + update: async (migrationId: number, data: Partial): Promise => { try { - await db.update(dictionaryMigration).set(data).where(eq(dictionaryMigration.id, migrationId)); + const updatedMigration = await db + .update(dictionaryMigration) + .set(data) + .where(eq(dictionaryMigration.id, migrationId)) + .returning({ id: dictionaryMigration.id }); logger.debug(LOG_MODULE, `Dictionary Migration with id '${migrationId}' updated successfully`); + return updatedMigration[0].id; } catch (error) { logger.error(LOG_MODULE, `Failed updating dictionary migration with id '${migrationId}'`, error); throw new ServiceUnavailable(); @@ -62,14 +68,18 @@ const repository = (dependencies: BaseDependencies) => { getMigrationsByCategoryId: async ( categoryId: number, paginationOptions: PaginationOptions, - filterOptions: { status?: migration_status }, + filterOptions: { status?: MigrationStatus; fromDictionaryId?: number; toDictionaryId?: number }, ): Promise => { const { page, pageSize } = paginationOptions; + + const { status, fromDictionaryId, toDictionaryId } = filterOptions; try { const migrations = await db.query.dictionaryMigration.findMany({ where: and( eq(dictionaryMigration.categoryId, categoryId), - filterOptions.status ? eq(dictionaryMigration.status, filterOptions.status) : undefined, + status ? eq(dictionaryMigration.status, status) : undefined, + fromDictionaryId ? eq(dictionaryMigration.fromDictionaryId, fromDictionaryId) : undefined, + toDictionaryId ? eq(dictionaryMigration.toDictionaryId, toDictionaryId) : undefined, ), orderBy: (dictionaryMigration, { desc }) => desc(dictionaryMigration.createdAt), limit: pageSize, diff --git a/packages/data-provider/src/utils/types.ts b/packages/data-provider/src/utils/types.ts index c46a04b0..29effd21 100644 --- a/packages/data-provider/src/utils/types.ts +++ b/packages/data-provider/src/utils/types.ts @@ -34,7 +34,7 @@ export const SUBMISSION_STATUS = { export type SubmissionStatus = ObjectValues; export const MIGRATION_STATUS = z.enum(['IN-PROGRESS', 'COMPLETED', 'FAILED']); -export type migration_status = z.infer; +export type MigrationStatus = z.infer; /** * Enum matching Audit Action in database From 0a26540c734ee9fb004a369201840f04c945e1e8 Mon Sep 17 00:00:00 2001 From: Leonardo Rivera Date: Thu, 15 Jan 2026 06:51:33 -0500 Subject: [PATCH 05/33] update migration service --- .../src/services/dictionaryService.ts | 11 +- .../src/services/migrationService.ts | 222 ++++++++++++++---- 2 files changed, 186 insertions(+), 47 deletions(-) diff --git a/packages/data-provider/src/services/dictionaryService.ts b/packages/data-provider/src/services/dictionaryService.ts index 065bb2b0..a208f707 100644 --- a/packages/data-provider/src/services/dictionaryService.ts +++ b/packages/data-provider/src/services/dictionaryService.ts @@ -8,6 +8,7 @@ import lecternClient from '../external/lecternClient.js'; import categoryRepository from '../repository/categoryRepository.js'; import dictionaryRepository from '../repository/dictionaryRepository.js'; import { BadRequest } from '../utils/errors.js'; +import migrationService from './migrationService.js'; const dictionaryService = (dependencies: BaseDependencies) => { const LOG_MODULE = 'DICTIONARY_SERVICE'; @@ -111,6 +112,7 @@ const dictionaryService = (dependencies: BaseDependencies) => { ); const categoryRepo = categoryRepository(dependencies); + const { initiateMigration } = migrationService(dependencies); const dictionary = await fetchDictionaryByVersion(dictionaryName, dictionaryVersion); @@ -142,14 +144,19 @@ const dictionaryService = (dependencies: BaseDependencies) => { updatedBy: username, }); - // TODO: Handle Dictionary Migration process async here + const migrationId = await initiateMigration({ + categoryId: updatedCategory.id, + fromDictionaryId: foundCategory.activeDictionaryId, + toDictionaryId: savedDictionary.id, + userName: username || '', + }); logger.info( LOG_MODULE, `Category '${updatedCategory.name}' updated successfully with Dictionary '${savedDictionary.name}' version '${savedDictionary.version}'`, ); - return { dictionary: savedDictionary, category: updatedCategory }; + return { dictionary: savedDictionary, category: updatedCategory, migrationId }; } else { // Create a new Category const newCategory: NewCategory = { diff --git a/packages/data-provider/src/services/migrationService.ts b/packages/data-provider/src/services/migrationService.ts index 88ad2328..89878492 100644 --- a/packages/data-provider/src/services/migrationService.ts +++ b/packages/data-provider/src/services/migrationService.ts @@ -1,65 +1,197 @@ -import type { DictionaryMigration, NewDictionaryMigration } from '@overture-stack/lyric-data-model/models'; +import type { DictionaryMigration, NewDictionaryMigration, Submission } from '@overture-stack/lyric-data-model/models'; import type { BaseDependencies } from '../config/config.js'; +import categoryRepository from '../repository/categoryRepository.js'; import migrationRepository from '../repository/dictionaryMigrationRepository.js'; +import submittedDataRepository from '../repository/submittedRepository.js'; +import type { MigrationStatus } from '../utils/types.js'; +import processor from './submission/processor.js'; +import submissionService from './submission/submission.js'; const migrationService = (dependencies: BaseDependencies) => { const LOG_MODULE = 'MIGRATION_SERVICE'; - const { logger } = dependencies; + const { logger, onFinishCommit } = dependencies; const migrationRepo = migrationRepository(dependencies); - return { - getActiveMigrationByCategoryId: async (categoryId: number): Promise => { - try { - const migrations = await migrationRepo.getMigrationsByCategoryId( - categoryId, - { page: 1, pageSize: 1 }, - { status: 'IN-PROGRESS' }, - ); - if (migrations.length > 0) { - logger.info(LOG_MODULE, `Active migration found for categoryId '${categoryId}'`); - return migrations[0]; - } else { - logger.info(LOG_MODULE, `No active migration for categoryId '${categoryId}'`); - return null; - } - } catch (error) { - logger.error(LOG_MODULE, `Error retrieving active migration for categoryId '${categoryId}'`, error); - throw error; + /** + * Update the status of the migration to COMPLETED or FAILED + * @param param0 + * @returns The ID of the finalized migration + */ + const finalizeMigration = async ({ + migrationId, + status, + userName, + }: { + migrationId: number; + status: Extract; + userName: string; + }): Promise => { + try { + const updatedMigration = await migrationRepo.update(migrationId, { + status, + updatedAt: new Date(), + updatedBy: userName, + }); + logger.info(LOG_MODULE, `Migration finalized for migrationId '${migrationId}' with status '${status}'`); + return updatedMigration; + } catch (error) { + logger.error(LOG_MODULE, `Error finalizing migration for migrationId '${migrationId}'`, error); + throw error; + } + }; + + /** + * Find the active migration by category ID + * @param categoryId + * @returns + */ + const getActiveMigrationByCategoryId = async (categoryId: number): Promise => { + try { + const migrations = await migrationRepo.getMigrationsByCategoryId( + categoryId, + { page: 1, pageSize: 1 }, + { status: 'IN-PROGRESS' }, + ); + if (migrations.length > 0) { + logger.info(LOG_MODULE, `Active migration found for categoryId '${categoryId}'`); + return migrations[0]; + } else { + logger.info(LOG_MODULE, `No active migration for categoryId '${categoryId}'`); + return null; } - }, - initiateMigration: async ({ - categoryId, - fromDictionaryId, - toDictionaryId, - submissionId, - userName, - }: { - categoryId: number; - fromDictionaryId: number; - toDictionaryId: number; - submissionId: number; - userName: string; - }): Promise<{ id: number }> => { - try { - // TODO: Check if there's already an active migration for this categoryId - // if it exists, increment retries, else if not exists create new migration + } catch (error) { + logger.error(LOG_MODULE, `Error retrieving active migration for categoryId '${categoryId}'`, error); + throw error; + } + }; + + /** + * Creates a Migration record or update retries if one exists. + * It starts running migration asynchronously + * @param param0 + * @returns The ID of the initiated or updated migration + */ + const initiateMigration = async ({ + categoryId, + fromDictionaryId, + toDictionaryId, + userName, + }: { + categoryId: number; + fromDictionaryId: number; + toDictionaryId: number; + userName: string; + }): Promise => { + const { getOrCreateActiveSubmission } = submissionService(dependencies); + try { + const existingMigrationResult = await migrationRepo.getMigrationsByCategoryId( + categoryId, + { page: 1, pageSize: 1 }, + { fromDictionaryId, toDictionaryId }, + ); + + let migrationId: number; + + // Migration already exists, update retries count + if (existingMigrationResult.length > 0) { + const migration = existingMigrationResult[0]; + const updatedRetriesCount = migration.retries + 1; + + migrationId = await migrationRepo.update(migration.id, { + retries: updatedRetriesCount, + updatedBy: userName, + updatedAt: new Date(), + }); + + logger.info( + LOG_MODULE, + `Existing migration found for categoryId '${categoryId}'. Incremented retries to ${updatedRetriesCount}`, + ); + } else { + // Create new migration record + const newSubmission = await getOrCreateActiveSubmission({ + categoryId, + organization: '', + username: userName, + }); + const newMigration: NewDictionaryMigration = { categoryId, fromDictionaryId, toDictionaryId, - submissionId, + submissionId: newSubmission.id, createdBy: userName, status: 'IN-PROGRESS', }; - const savedMigration = await migrationRepo.save(newMigration); - logger.info(LOG_MODULE, `Migration initiated for categoryId '${categoryId}'`); - return savedMigration; - } catch (error) { - logger.error(LOG_MODULE, `Error initiating migration for categoryId '${categoryId}'`, error); - throw error; + + migrationId = await migrationRepo.save(newMigration); + + logger.info(LOG_MODULE, `Creating migration record for categoryId '${categoryId}'`); + + // Start migration asynchronously + performMigrationValidation({ categoryId, submission: newSubmission, userName }) + .then(() => { + finalizeMigration({ migrationId, status: 'COMPLETED', userName }); + }) + .catch(async () => { + finalizeMigration({ migrationId, status: 'FAILED', userName }); + }); } - }, + + logger.info(LOG_MODULE, `Migration initiated for categoryId '${categoryId}'`); + return migrationId; + } catch (error) { + logger.error(LOG_MODULE, `Error initiating migration for categoryId '${categoryId}'`, error); + throw error; + } + }; + + /** Execute submitted data validation for the migration */ + const performMigrationValidation = async ({ + categoryId, + submission, + userName, + }: { + categoryId: number; + submission: Submission; + userName: string; + }): Promise => { + const { getAllOrganizationsByCategoryId, getSubmittedDataByCategoryIdAndOrganization } = + submittedDataRepository(dependencies); + const { getActiveDictionaryByCategory } = categoryRepository(dependencies); + const { performCommitSubmissionAsync } = processor(dependencies); + + const dictionary = await getActiveDictionaryByCategory(categoryId); + if (!dictionary) { + throw new Error(`Dictionary in category '${categoryId}' not found`); + } + + const organizations = await getAllOrganizationsByCategoryId(categoryId); + for (const organization of organizations) { + const submittedDataToValidate = await getSubmittedDataByCategoryIdAndOrganization(categoryId, organization); + + await performCommitSubmissionAsync({ + dataToValidate: { + inserts: [], + submittedData: submittedDataToValidate, + deletes: [], + updates: {}, + }, + submission, + dictionary, + username: userName, + onFinishCommit, + }); + } + logger.info(LOG_MODULE, `Migration validation completed for submissionId '${submission.id}'`); + }; + + return { + finalizeMigration, + getActiveMigrationByCategoryId, + initiateMigration, + performMigrationValidation, }; }; From 1ec4dc4e7cb8539324fe6cd81f1aa68f3602af7c Mon Sep 17 00:00:00 2001 From: Leonardo Rivera Date: Fri, 30 Jan 2026 16:39:46 -0500 Subject: [PATCH 06/33] data validation --- .../src/models/audit_submitted_data.ts | 8 ++-- .../src/controllers/dictionaryController.ts | 1 - .../src/repository/submittedRepository.ts | 42 +++++++++++------- .../src/services/migrationService.ts | 36 ++++++++++----- .../src/services/submission/processor.ts | 43 +++++++++--------- .../src/utils/submittedDataUtils.ts | 12 +---- packages/data-provider/src/utils/types.ts | 2 +- .../submittedData/hasErrorsByIndex.spec.ts | 44 ------------------- 8 files changed, 80 insertions(+), 108 deletions(-) delete mode 100644 packages/data-provider/test/utils/submittedData/hasErrorsByIndex.spec.ts diff --git a/packages/data-model/src/models/audit_submitted_data.ts b/packages/data-model/src/models/audit_submitted_data.ts index 97197039..a15d969a 100644 --- a/packages/data-model/src/models/audit_submitted_data.ts +++ b/packages/data-model/src/models/audit_submitted_data.ts @@ -1,11 +1,11 @@ import { relations } from 'drizzle-orm'; import { boolean, integer, jsonb, pgEnum, pgTable, serial, timestamp, varchar } from 'drizzle-orm/pg-core'; -import type { DataRecord } from '@overture-stack/lectern-client'; +import type { DataRecord, DictionaryValidationRecordErrorDetails } from '@overture-stack/lectern-client'; import { dictionaries } from './dictionaries.js'; import { dictionaryCategories } from './dictionary_categories.js'; -import { type SubmissionRecordErrorDetails, submissions } from './submissions.js'; +import { submissions } from './submissions.js'; export const audit_action = pgEnum('audit_action', ['UPDATE', 'DELETE', 'MIGRATION']); @@ -14,8 +14,6 @@ export type DataDiff = { new: DataRecord; }; -export type AuditErrors = Record; - export const auditSubmittedData = pgTable('audit_submitted_data', { id: serial('id').primaryKey(), action: audit_action('action').notNull(), @@ -24,7 +22,7 @@ export const auditSubmittedData = pgTable('audit_submitted_data', { .notNull(), dataDiff: jsonb('data_diff').$type(), entityName: varchar('entity_name').notNull(), - errors: jsonb('errors').$type(), + errors: jsonb('errors').$type(), lastValidSchemaId: integer('last_valid_schema_id').references(() => dictionaries.id), newDataIsValid: boolean('new_data_is_valid').notNull(), oldDataIsValid: boolean('old_data_is_valid').notNull(), diff --git a/packages/data-provider/src/controllers/dictionaryController.ts b/packages/data-provider/src/controllers/dictionaryController.ts index ecaefe69..0168267f 100644 --- a/packages/data-provider/src/controllers/dictionaryController.ts +++ b/packages/data-provider/src/controllers/dictionaryController.ts @@ -42,7 +42,6 @@ const controller = (dependencies: BaseDependencies) => { const result: RegisterDictionaryResult = { categoryId: category.id, categoryName: category.name, - dictionary: dictionary.dictionary, name: dictionary.name, version: dictionary.version, migrationId: migrationId, diff --git a/packages/data-provider/src/repository/submittedRepository.ts b/packages/data-provider/src/repository/submittedRepository.ts index e945b786..76666f1d 100644 --- a/packages/data-provider/src/repository/submittedRepository.ts +++ b/packages/data-provider/src/repository/submittedRepository.ts @@ -2,8 +2,8 @@ import type { ExtractTablesWithRelations } from 'drizzle-orm'; import type { PgTransaction } from 'drizzle-orm/pg-core'; import type { PostgresJsQueryResultHKT } from 'drizzle-orm/postgres-js'; import { and, count, eq, or, SQL, sql } from 'drizzle-orm/sql'; -import * as _ from 'lodash-es'; +import type { DictionaryValidationRecordErrorDetails } from '@overture-stack/lectern-client'; import { auditSubmittedData, type DataDiff, @@ -55,18 +55,23 @@ const repository = (dependencies: BaseDependencies) => { oldIsValid, recordUpdated, submissionId, + isMigration, + errors, }: { dataDiff: DataDiff; oldIsValid: boolean; recordUpdated: SubmittedData; submissionId: number; + isMigration: boolean; + errors?: DictionaryValidationRecordErrorDetails[]; }, tx?: PgTransaction>, ) => { const newAudit: NewAuditSubmittedData = { - action: AUDIT_ACTION.Values.UPDATE, + action: isMigration ? AUDIT_ACTION.Values.MIGRATION : AUDIT_ACTION.Values.UPDATE, dictionaryCategoryId: recordUpdated.dictionaryCategoryId, entityName: recordUpdated.entityName, + errors, lastValidSchemaId: recordUpdated.lastValidSchemaId, newDataIsValid: recordUpdated.isValid, dataDiff: dataDiff, @@ -364,36 +369,43 @@ const repository = (dependencies: BaseDependencies) => { * @param newData Set fields to update * @param oldIsValid Previous isValid value * @param submissionId Submission ID + * @param isMigration Whether the update is part of a migration + * @param errors Audit errors, if any * @param tx The transaction to use for the operation, optional * @returns An updated record */ update: async ( { submittedDataId, - dataDiff, - newData, - oldIsValid, - submissionId, + data, + audit: { dataDiff, errors, isMigration, oldIsValid, submissionId }, }: { submittedDataId: number; - dataDiff: DataDiff; - newData: Partial; - oldIsValid: boolean; - submissionId: number; + data: Partial; + audit: { + dataDiff: DataDiff; + errors?: DictionaryValidationRecordErrorDetails[]; + isMigration: boolean; + oldIsValid: boolean; + submissionId: number; + }; }, tx?: PgTransaction>, ): Promise => { try { - const updated = await (tx || db) + const [updated] = await (tx || db) .update(submittedData) - .set({ ...newData, updatedAt: new Date() }) + .set({ ...data, updatedAt: new Date() }) .where(eq(submittedData.id, submittedDataId)) .returning(); - if (features?.audit?.enabled && !_.isEmpty(dataDiff.new) && !_.isEmpty(dataDiff.old)) { - await auditUpdateSubmittedData({ recordUpdated: updated[0], submissionId, dataDiff, oldIsValid }, tx); + if (features?.audit?.enabled && updated) { + await auditUpdateSubmittedData( + { recordUpdated: updated, submissionId, dataDiff, oldIsValid, isMigration, errors }, + tx, + ); } - return updated[0]; + return updated; } catch (error) { logger.error(LOG_MODULE, `Failed updating SubmittedData`, error); throw new ServiceUnavailable(); diff --git a/packages/data-provider/src/services/migrationService.ts b/packages/data-provider/src/services/migrationService.ts index 89878492..081fe1ae 100644 --- a/packages/data-provider/src/services/migrationService.ts +++ b/packages/data-provider/src/services/migrationService.ts @@ -1,6 +1,7 @@ import type { DictionaryMigration, NewDictionaryMigration, Submission } from '@overture-stack/lyric-data-model/models'; import type { BaseDependencies } from '../config/config.js'; +import submissionRepository from '../repository/activeSubmissionRepository.js'; import categoryRepository from '../repository/categoryRepository.js'; import migrationRepository from '../repository/dictionaryMigrationRepository.js'; import submittedDataRepository from '../repository/submittedRepository.js'; @@ -84,6 +85,7 @@ const migrationService = (dependencies: BaseDependencies) => { userName: string; }): Promise => { const { getOrCreateActiveSubmission } = submissionService(dependencies); + const { getSubmissionById } = submissionRepository(dependencies); try { const existingMigrationResult = await migrationRepo.getMigrationsByCategoryId( categoryId, @@ -92,12 +94,19 @@ const migrationService = (dependencies: BaseDependencies) => { ); let migrationId: number; + let submission: Submission; // Migration already exists, update retries count if (existingMigrationResult.length > 0) { const migration = existingMigrationResult[0]; const updatedRetriesCount = migration.retries + 1; + const submissionResponse = await getSubmissionById(migration.id); + if (!submissionResponse) { + throw new Error(`Submission with id '${migration.id}' not found`); + } + submission = submissionResponse; + migrationId = await migrationRepo.update(migration.id, { retries: updatedRetriesCount, updatedBy: userName, @@ -115,6 +124,7 @@ const migrationService = (dependencies: BaseDependencies) => { organization: '', username: userName, }); + submission = newSubmission; const newMigration: NewDictionaryMigration = { categoryId, @@ -128,17 +138,18 @@ const migrationService = (dependencies: BaseDependencies) => { migrationId = await migrationRepo.save(newMigration); logger.info(LOG_MODULE, `Creating migration record for categoryId '${categoryId}'`); - - // Start migration asynchronously - performMigrationValidation({ categoryId, submission: newSubmission, userName }) - .then(() => { - finalizeMigration({ migrationId, status: 'COMPLETED', userName }); - }) - .catch(async () => { - finalizeMigration({ migrationId, status: 'FAILED', userName }); - }); } + // Start migration asynchronously + performMigrationValidation({ categoryId, submission, userName }) + .then(() => { + finalizeMigration({ migrationId, status: 'COMPLETED', userName }); + }) + .catch(async (error) => { + logger.error(LOG_MODULE, `Error during migration validation for categoryId '${categoryId}'`, error); + finalizeMigration({ migrationId, status: 'FAILED', userName }); + }); + logger.info(LOG_MODULE, `Migration initiated for categoryId '${categoryId}'`); return migrationId; } catch (error) { @@ -168,9 +179,13 @@ const migrationService = (dependencies: BaseDependencies) => { } const organizations = await getAllOrganizationsByCategoryId(categoryId); + logger.info(LOG_MODULE, `Starting migration validation for following organizations '${organizations}'`); for (const organization of organizations) { const submittedDataToValidate = await getSubmittedDataByCategoryIdAndOrganization(categoryId, organization); - + logger.info( + LOG_MODULE, + `Performing migration validation for organization '${organization}' with ${submittedDataToValidate.length} submitted records`, + ); await performCommitSubmissionAsync({ dataToValidate: { inserts: [], @@ -181,6 +196,7 @@ const migrationService = (dependencies: BaseDependencies) => { submission, dictionary, username: userName, + isMigration: true, onFinishCommit, }); } diff --git a/packages/data-provider/src/services/submission/processor.ts b/packages/data-provider/src/services/submission/processor.ts index 0c3ffb70..13979da4 100644 --- a/packages/data-provider/src/services/submission/processor.ts +++ b/packages/data-provider/src/services/submission/processor.ts @@ -43,7 +43,6 @@ import { groupByEntityName, groupErrorsByIndex, groupSchemaDataByEntityName, - hasErrorsByIndex, mergeSubmittedDataAndDeduplicateById, updateSubmittedDataArray, } from '../../utils/submittedDataUtils.js'; @@ -261,47 +260,49 @@ const processor = (dependencies: BaseDependencies) => { }; await dependencies.db.transaction(async (tx) => { - Object.entries(schemasDataToValidate.submittedDataByEntityName).forEach(([entityName, dataArray], index) => { - dataArray.forEach((data) => { - const invalidRecordErrors = findInvalidRecordErrorsBySchemaName(resultValidation, entityName); - const hasErrorByIndex = groupErrorsByIndex(invalidRecordErrors); + Object.entries(schemasDataToValidate.submittedDataByEntityName).forEach(([entityName, dataArray]) => { + const invalidRecordErrors = findInvalidRecordErrorsBySchemaName(resultValidation, entityName); + const hasErrorByIndex = groupErrorsByIndex(invalidRecordErrors); + const invalidRecordIndexes = invalidRecordErrors.map((error) => error.recordIndex); + logger.info(LOG_MODULE, `Found '${invalidRecordIndexes.length}' invalid records in entity '${entityName}'`); + dataArray.forEach((data, index) => { const oldIsValid = data.isValid; - const newIsValid = !hasErrorsByIndex(hasErrorByIndex, index); + const errorsForRecord = hasErrorByIndex[index] ?? []; + const newIsValid = errorsForRecord.length === 0; if (data.id) { const inputUpdate: Partial = {}; const submisionUpdateData = dataToValidate.updates && dataToValidate.updates[data.systemId]; if (submisionUpdateData) { - logger.info(LOG_MODULE, `Updating submittedData system ID '${data.systemId}' in entity '${entityName}'`); inputUpdate.data = data.data; } if (oldIsValid !== newIsValid) { inputUpdate.isValid = newIsValid; if (newIsValid) { - logger.info( - LOG_MODULE, - `Updating submittedData system ID '${data.systemId}' as Valid in entity '${entityName}'`, - ); inputUpdate.lastValidSchemaId = dictionary.id; } - logger.info( - LOG_MODULE, - `Updating submittedData system ID '${data.systemId}' as invalid in entity '${entityName}'`, - ); } - if (Object.values(inputUpdate)) { + if (Object.values(inputUpdate).length > 0) { inputUpdate.updatedBy = username; if (newIsValid) { inputUpdate.lastValidSchemaId = dictionary.id; } + logger.debug( + LOG_MODULE, + `Updating submittedData in entity '${entityName}' with system ID '${data.systemId}'`, + ); dataSubmittedRepo.update( { submittedDataId: data.id, - newData: inputUpdate, - dataDiff: { old: submisionUpdateData?.old ?? {}, new: submisionUpdateData?.new ?? {} }, - oldIsValid: oldIsValid, - submissionId: submission.id, + data: inputUpdate, + audit: { + dataDiff: { old: submisionUpdateData?.old ?? {}, new: submisionUpdateData?.new ?? {} }, + errors: errorsForRecord, + isMigration: params.isMigration || false, + submissionId: submission.id, + oldIsValid: oldIsValid, + }, }, tx, ); @@ -318,7 +319,7 @@ const processor = (dependencies: BaseDependencies) => { } } } else { - logger.info( + logger.debug( LOG_MODULE, `Creating new submittedData in entity '${entityName}' with system ID '${data.systemId}'`, ); diff --git a/packages/data-provider/src/utils/submittedDataUtils.ts b/packages/data-provider/src/utils/submittedDataUtils.ts index 9b48c34f..ca121247 100644 --- a/packages/data-provider/src/utils/submittedDataUtils.ts +++ b/packages/data-provider/src/utils/submittedDataUtils.ts @@ -1,4 +1,4 @@ -import { groupBy, has } from 'lodash-es'; +import { groupBy } from 'lodash-es'; import { type DataRecord, @@ -207,16 +207,6 @@ export const groupSchemaDataByEntityName = (data: { ); }; -/** - * Receives any object and finds if it contains an specific key - * @param {object} hasErrorByIndex An object to evaluate - * @param {number} index An object key - * @returns - */ -export const hasErrorsByIndex = (hasErrorByIndex: object, index: number): boolean => { - return has(hasErrorByIndex, index); -}; - /** * Transforms an array of `SubmittedData` into a `Record`, * where each key is the `entityName` from the `SubmittedData`, and the value is an array of diff --git a/packages/data-provider/src/utils/types.ts b/packages/data-provider/src/utils/types.ts index 66590d6f..298f9a05 100644 --- a/packages/data-provider/src/utils/types.ts +++ b/packages/data-provider/src/utils/types.ts @@ -127,7 +127,6 @@ export type CommitSubmissionResult = { export type RegisterDictionaryResult = { categoryId: number; categoryName: string; - dictionary: object; name: string; version: string; migrationId?: number; @@ -180,6 +179,7 @@ export interface CommitSubmissionParams { dictionary: SchemasDictionary & { id: number }; submission: Submission; username: string; + isMigration?: boolean; onFinishCommit?: (resultOnCommit: ResultOnCommit) => void; } diff --git a/packages/data-provider/test/utils/submittedData/hasErrorsByIndex.spec.ts b/packages/data-provider/test/utils/submittedData/hasErrorsByIndex.spec.ts deleted file mode 100644 index 660f58bf..00000000 --- a/packages/data-provider/test/utils/submittedData/hasErrorsByIndex.spec.ts +++ /dev/null @@ -1,44 +0,0 @@ -import { expect } from 'chai'; -import { describe, it } from 'mocha'; - -import { type DictionaryValidationRecordErrorDetails, type SchemaRecordError } from '@overture-stack/lectern-client'; - -import { groupErrorsByIndex, hasErrorsByIndex } from '../../../src/utils/submittedDataUtils.js'; - -describe('Submitted Data Utils - hasErrorsByIndex', () => { - describe('Finds an error by index', () => { - const listOfErrors: SchemaRecordError[] = [ - { - recordIndex: 1, - recordErrors: [ - { - reason: 'UNRECOGNIZED_FIELD', - fieldName: 'systemId', - fieldValue: '', - }, - ], - }, - { - recordIndex: 1, - recordErrors: [ - { - errors: [], - reason: `INVALID_BY_RESTRICTION`, - fieldName: 'sex_at_birth', - fieldValue: 'Homme', - }, - ], - }, - ]; - it('should return true if error is found on index', () => { - const errorsByIndex = groupErrorsByIndex(listOfErrors); - const response = hasErrorsByIndex(errorsByIndex, 1); - expect(response).to.be.true; - }); - it('should return false if no error is found on index', () => { - const errorsByIndex = groupErrorsByIndex(listOfErrors); - const response = hasErrorsByIndex(errorsByIndex, 0); - expect(response).to.be.false; - }); - }); -}); From afc4f870e63cccb8a2555558c571ffdec4418420 Mon Sep 17 00:00:00 2001 From: Leonardo Rivera Date: Mon, 2 Feb 2026 23:26:55 -0500 Subject: [PATCH 07/33] refactor migration process --- .../src/controllers/dictionaryController.ts | 2 +- .../src/repository/submittedRepository.ts | 8 +- .../src/services/migrationService.ts | 3 +- .../src/services/submission/processor.ts | 141 +++++++++--------- packages/data-provider/src/utils/types.ts | 6 + 5 files changed, 84 insertions(+), 76 deletions(-) diff --git a/packages/data-provider/src/controllers/dictionaryController.ts b/packages/data-provider/src/controllers/dictionaryController.ts index 0168267f..16d9d221 100644 --- a/packages/data-provider/src/controllers/dictionaryController.ts +++ b/packages/data-provider/src/controllers/dictionaryController.ts @@ -44,7 +44,7 @@ const controller = (dependencies: BaseDependencies) => { categoryName: category.name, name: dictionary.name, version: dictionary.version, - migrationId: migrationId, + migrationId, }; return res.send(result); } catch (error) { diff --git a/packages/data-provider/src/repository/submittedRepository.ts b/packages/data-provider/src/repository/submittedRepository.ts index 76666f1d..3d28a31b 100644 --- a/packages/data-provider/src/repository/submittedRepository.ts +++ b/packages/data-provider/src/repository/submittedRepository.ts @@ -393,19 +393,19 @@ const repository = (dependencies: BaseDependencies) => { tx?: PgTransaction>, ): Promise => { try { - const [updated] = await (tx || db) + const [recordUpdated] = await (tx || db) .update(submittedData) .set({ ...data, updatedAt: new Date() }) .where(eq(submittedData.id, submittedDataId)) .returning(); - if (features?.audit?.enabled && updated) { + if (features?.audit?.enabled && recordUpdated) { await auditUpdateSubmittedData( - { recordUpdated: updated, submissionId, dataDiff, oldIsValid, isMigration, errors }, + { recordUpdated, submissionId, dataDiff, oldIsValid, isMigration, errors }, tx, ); } - return updated; + return recordUpdated; } catch (error) { logger.error(LOG_MODULE, `Failed updating SubmittedData`, error); throw new ServiceUnavailable(); diff --git a/packages/data-provider/src/services/migrationService.ts b/packages/data-provider/src/services/migrationService.ts index 081fe1ae..34b47c37 100644 --- a/packages/data-provider/src/services/migrationService.ts +++ b/packages/data-provider/src/services/migrationService.ts @@ -131,8 +131,9 @@ const migrationService = (dependencies: BaseDependencies) => { fromDictionaryId, toDictionaryId, submissionId: newSubmission.id, - createdBy: userName, status: 'IN-PROGRESS', + createdBy: userName, + createdAt: new Date(), }; migrationId = await migrationRepo.save(newMigration); diff --git a/packages/data-provider/src/services/submission/processor.ts b/packages/data-provider/src/services/submission/processor.ts index 13979da4..3dfae5cf 100644 --- a/packages/data-provider/src/services/submission/processor.ts +++ b/packages/data-provider/src/services/submission/processor.ts @@ -47,11 +47,11 @@ import { updateSubmittedDataArray, } from '../../utils/submittedDataUtils.js'; import { - CommitSubmissionParams, + type CommitSubmissionParams, type EntityData, + type ResultCommit, type SchemasDictionary, SUBMISSION_STATUS, - type SubmittedDataResponse, } from '../../utils/types.js'; import searchDataRelations from '../submittedData/searchDataRelations.js'; @@ -249,92 +249,91 @@ const processor = (dependencies: BaseDependencies) => { const resultValidation = validateSchemas(dictionary, schemasDataToValidate.schemaDataByEntityName); - const resultCommit: { - inserts: SubmittedDataResponse[]; - updates: SubmittedDataResponse[]; - deletes: SubmittedDataResponse[]; - } = { + const resultCommit: ResultCommit = { inserts: [], updates: [], deletes: [], }; await dependencies.db.transaction(async (tx) => { - Object.entries(schemasDataToValidate.submittedDataByEntityName).forEach(([entityName, dataArray]) => { + Object.entries(schemasDataToValidate.submittedDataByEntityName).forEach(([entityName, records]) => { const invalidRecordErrors = findInvalidRecordErrorsBySchemaName(resultValidation, entityName); - const hasErrorByIndex = groupErrorsByIndex(invalidRecordErrors); - const invalidRecordIndexes = invalidRecordErrors.map((error) => error.recordIndex); - logger.info(LOG_MODULE, `Found '${invalidRecordIndexes.length}' invalid records in entity '${entityName}'`); - dataArray.forEach((data, index) => { - const oldIsValid = data.isValid; - const errorsForRecord = hasErrorByIndex[index] ?? []; - const newIsValid = errorsForRecord.length === 0; - if (data.id) { - const inputUpdate: Partial = {}; - const submisionUpdateData = dataToValidate.updates && dataToValidate.updates[data.systemId]; - if (submisionUpdateData) { - inputUpdate.data = data.data; + const errorsByIndex = groupErrorsByIndex(invalidRecordErrors); + logger.info(LOG_MODULE, `Found '${invalidRecordErrors.length}' invalid records in entity '${entityName}'`); + records.forEach((record, index) => { + const errors = errorsByIndex[index] ?? []; + const newIsValid = errors.length === 0; + + if (record.id) { + const oldIsValid = record.isValid; + const update: Partial = {}; + + const submissionUpdate = dataToValidate.updates?.[record.systemId]; + if (submissionUpdate) { + update.data = record.data; } if (oldIsValid !== newIsValid) { - inputUpdate.isValid = newIsValid; + update.isValid = newIsValid; if (newIsValid) { - inputUpdate.lastValidSchemaId = dictionary.id; + update.lastValidSchemaId = dictionary.id; } } - if (Object.values(inputUpdate).length > 0) { - inputUpdate.updatedBy = username; - if (newIsValid) { - inputUpdate.lastValidSchemaId = dictionary.id; - } - logger.debug( - LOG_MODULE, - `Updating submittedData in entity '${entityName}' with system ID '${data.systemId}'`, - ); - dataSubmittedRepo.update( - { - submittedDataId: data.id, - data: inputUpdate, - audit: { - dataDiff: { old: submisionUpdateData?.old ?? {}, new: submisionUpdateData?.new ?? {} }, - errors: errorsForRecord, - isMigration: params.isMigration || false, - submissionId: submission.id, - oldIsValid: oldIsValid, - }, + if (Object.keys(update).length === 0) { + return; + } + + update.updatedBy = username; + if (newIsValid) { + update.lastValidSchemaId = dictionary.id; + } + logger.debug( + LOG_MODULE, + `Updating submittedData in entity '${entityName}' with system ID '${record.systemId}'`, + ); + dataSubmittedRepo.update( + { + submittedDataId: record.id, + data: update, + audit: { + dataDiff: { old: submissionUpdate?.old ?? {}, new: submissionUpdate?.new ?? {} }, + errors: errors, + isMigration: params.isMigration || false, + oldIsValid, + submissionId: submission.id, }, - tx, - ); - - // Check if either 'data' or 'isValid' keys has been updated - if ('data' in inputUpdate || 'isValid' in inputUpdate) { - resultCommit.updates.push({ - isValid: newIsValid, - entityName, - organization: data.organization, - data: data.data, - systemId: data.systemId, - }); - } + }, + tx, + ); + + // Check if either 'data' or 'isValid' keys has been updated + if ('data' in update || 'isValid' in update) { + resultCommit.updates.push({ + data: record.data, + entityName, + isValid: newIsValid, + organization: record.organization, + systemId: record.systemId, + }); } } else { logger.debug( LOG_MODULE, - `Creating new submittedData in entity '${entityName}' with system ID '${data.systemId}'`, + `Creating new submittedData in entity '${entityName}' with system ID '${record.systemId}'`, ); - data.isValid = newIsValid; + record.isValid = newIsValid; if (newIsValid) { - data.lastValidSchemaId = dictionary.id; + record.lastValidSchemaId = dictionary.id; } - dataSubmittedRepo.save(data, tx); + dataSubmittedRepo.save(record, tx); resultCommit.inserts.push({ - isValid: newIsValid, + data: record.data, entityName, - organization: data.organization, - data: data.data, - systemId: data.systemId, + isValid: newIsValid, + organization: record.organization, + systemId: record.systemId, }); } }); @@ -342,22 +341,24 @@ const processor = (dependencies: BaseDependencies) => { // iterate if there are any record to be deleted dataToValidate?.deletes?.forEach((item) => { + const { data, entityName, isValid, organization, systemId } = item; + dataSubmittedRepo.deleteBySystemId( { + diff: computeDataDiff(data, null), submissionId: submission.id, - systemId: item.systemId, - diff: computeDataDiff(item.data, null), + systemId, username, }, tx, ); resultCommit.deletes.push({ - isValid: item.isValid, - entityName: item.entityName, - organization: item.organization, - data: item.data, - systemId: item.systemId, + data, + entityName, + isValid, + organization, + systemId, }); }); diff --git a/packages/data-provider/src/utils/types.ts b/packages/data-provider/src/utils/types.ts index 298f9a05..45cd2d41 100644 --- a/packages/data-provider/src/utils/types.ts +++ b/packages/data-provider/src/utils/types.ts @@ -169,6 +169,12 @@ export interface ValidateFilesParams { username: string; } +export type ResultCommit = { + inserts: SubmittedDataResponse[]; + updates: SubmittedDataResponse[]; + deletes: SubmittedDataResponse[]; +}; + export interface CommitSubmissionParams { dataToValidate: { inserts: NewSubmittedData[]; From 58d4ad85ffea76d6b9e364c8e482bd5e92bc000b Mon Sep 17 00:00:00 2001 From: Leonardo Rivera Date: Mon, 2 Feb 2026 23:52:08 -0500 Subject: [PATCH 08/33] block commit submission --- .../data-provider/src/services/submission/submission.ts | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/packages/data-provider/src/services/submission/submission.ts b/packages/data-provider/src/services/submission/submission.ts index 4965bdd8..266fe4fd 100644 --- a/packages/data-provider/src/services/submission/submission.ts +++ b/packages/data-provider/src/services/submission/submission.ts @@ -10,6 +10,7 @@ import categoryRepository from '../../repository/categoryRepository.js'; import submittedRepository from '../../repository/submittedRepository.js'; import { getSchemaByName } from '../../utils/dictionaryUtils.js'; import { BadRequest, InternalServerError, StatusConflict } from '../../utils/errors.js'; +import migrationSvc from '../migrationService.js'; import { isSubmissionActive, parseSubmissionResponse, @@ -49,6 +50,7 @@ const service = (dependencies: BaseDependencies) => { const { getSubmittedDataByCategoryIdAndOrganization } = submittedRepository(dependencies); const { getActiveDictionaryByCategory } = categoryRepository(dependencies); const { generateIdentifier } = systemIdGenerator(dependencies); + const { getActiveMigrationByCategoryId } = migrationSvc(dependencies); const submission = await getSubmissionById(submissionId); if (!submission) { @@ -63,6 +65,11 @@ const service = (dependencies: BaseDependencies) => { throw new StatusConflict('Submission does not have status VALID and cannot be committed'); } + const activeMigration = await getActiveMigrationByCategoryId(categoryId); + if (activeMigration) { + throw new StatusConflict('This submission cannot be committed while a migration is running'); + } + const currentDictionary = await getActiveDictionaryByCategory(categoryId); if (_.isEmpty(currentDictionary)) { throw new BadRequest(`Dictionary in category '${categoryId}' not found`); From b081dd9675e1580de127fdb90567e4e9b682fb46 Mon Sep 17 00:00:00 2001 From: Leonardo Rivera Date: Mon, 2 Feb 2026 23:56:37 -0500 Subject: [PATCH 09/33] fix sort imports --- packages/data-provider/src/services/submission/submission.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/data-provider/src/services/submission/submission.ts b/packages/data-provider/src/services/submission/submission.ts index 266fe4fd..2da0e4e3 100644 --- a/packages/data-provider/src/services/submission/submission.ts +++ b/packages/data-provider/src/services/submission/submission.ts @@ -10,7 +10,6 @@ import categoryRepository from '../../repository/categoryRepository.js'; import submittedRepository from '../../repository/submittedRepository.js'; import { getSchemaByName } from '../../utils/dictionaryUtils.js'; import { BadRequest, InternalServerError, StatusConflict } from '../../utils/errors.js'; -import migrationSvc from '../migrationService.js'; import { isSubmissionActive, parseSubmissionResponse, @@ -28,6 +27,7 @@ import { type SubmissionActionType, SubmissionSummaryResponse, } from '../../utils/types.js'; +import migrationSvc from '../migrationService.js'; import processor from './processor.js'; const service = (dependencies: BaseDependencies) => { From 8688240b6213d47f0f8755fb752ee291a3dbeed8 Mon Sep 17 00:00:00 2001 From: Leonardo Rivera Date: Thu, 2 Apr 2026 11:38:38 -0400 Subject: [PATCH 10/33] update docs --- packages/data-model/docs/README.md | 4 +- .../docs/dictionary-registration.md | 70 ++++++++++--------- 2 files changed, 40 insertions(+), 34 deletions(-) diff --git a/packages/data-model/docs/README.md b/packages/data-model/docs/README.md index 719eaa73..cf43adc4 100644 --- a/packages/data-model/docs/README.md +++ b/packages/data-model/docs/README.md @@ -91,7 +91,7 @@ This table records historical changes to submitted data, providing an audit trai Key fields in the audit_submitted_data table include: -- `action`: This field records the type of action performed on the submitted data, represented as an enumerated type (e.g., UPDATE or DELETE) +- `action`: This field records the type of action performed on the submitted data, represented as an enumerated type (e.g., UPDATE, DELETE or MIGRATION) - `dictionary_category_id`: This field links the audit entry to the corresponding dictionary category associated with the submitted data. It helps contextualize the action within the framework of the defined schema. @@ -99,6 +99,8 @@ Key fields in the audit_submitted_data table include: - `entity_name`: This field specifies the name of the entity related to the submitted data, providing additional context for the action logged in the audit entry. +- `errors`: This field stores list of errors that occurred during dictionary migration. + - `last_valid_schema_id`: This field references the schema version that was last validated before the action was taken, ensuring that users can trace back to the applicable schema for the submission. - `new_data_is_valid`: This boolean field indicates whether the newly submitted data is considered valid according to the schema rules. This status is important for ensuring that only compliant data is retained. diff --git a/packages/data-provider/docs/dictionary-registration.md b/packages/data-provider/docs/dictionary-registration.md index 0b4a393b..57821605 100644 --- a/packages/data-provider/docs/dictionary-registration.md +++ b/packages/data-provider/docs/dictionary-registration.md @@ -44,56 +44,60 @@ sequenceDiagram Note over LyricAPI: Find or create Dictionary LyricAPI->>+Lectern: Fetch dictionary (name + version) Lectern-->>-LyricAPI: Dictionary schema response - break Dictionary name and version not found + break If dictionary name and version not found LyricAPI->>User: 400 Bad Request: `Schema with name '${name}' and version '${version}' not found` end - break `defaultCentricEntity` does not exist in Dictionary schema + break If `defaultCentricEntity` does not exist in Dictionary schema LyricAPI->>User: 400 Bad Request: Entity '${defaultCentricEntity}' does not exist in this dictionary end LyricAPI->>+LyricDB: Query dictionary by dictionaryName + version - alt Dictionary does not exists - LyricAPI->>LyricDB: Create new dictionary - end LyricDB-->>-LyricAPI: Return Dictionary + alt If dictionary does not exists + LyricAPI->>+LyricDB: Create new dictionary + LyricDB-->>-LyricAPI: Return Dictionary + end Note over LyricAPI: Find or create Category LyricAPI->>+LyricDB: Query category by categoryName - break Category is using same dictionary version + LyricDB-->>-LyricAPI: Return Category + break If category is using same dictionary version %% Nothing to do here LyricAPI->>User: 200 OK: Return Dictionary and Category end - alt Category does not exists - LyricAPI->>LyricDB: Create new category - else Category exists but uses different dictionary version + alt If category does not exists + LyricAPI->>+LyricDB: Create new category + LyricDB-->>-LyricAPI: Return new category + + LyricAPI->>User: 200 OK: Return Dictionary and Category + else If category exists but uses different dictionary version LyricAPI->>LyricDB: Update category (`dictionary_categories` table) with new Dictionary - rect rgb(230, 242, 255) - note over LyricAPI: [Migration] Initiate migration - LyricAPI->>LyricDB: Create a migration in `category_migration` table
(categoryId, fromDictionaryId, toDictionaryId, submissionId, status=IN-PROGRESS, createdAt, createdBy) - LyricAPI->>LyricDB: Create a new submission (no data, status=COMMITTED) + note over LyricAPI: [Migration] Initiate migration + LyricAPI->>+LyricDB: Create a new submission (no data, status=COMMITTED) + LyricDB-->>-LyricAPI: Return submission + LyricAPI->>+LyricDB: Create a migration in `category_migration` table
(categoryId, fromDictionaryId, toDictionaryId, submissionId, status=IN-PROGRESS, createdAt, createdBy) + LyricDB-->>-LyricAPI: Return migrationId + + LyricAPI->>User: 200 OK: Return Dictionary, Category and migrationId + + create participant Worker + LyricAPI-->Worker: Start a Worker to run the migration validation + note over Worker: [Migration] validation runs in a worker thread + Worker->>+LyricDB: Retrieve all existing submitted Data in category + LyricDB-->>-Worker: All Submitted Data in Category + Worker->>Worker: Validate record against new schema + + loop each record failed validation + Worker->>LyricDB: Update record in `submitted_data` table, with isValid = FALSE, lastValidSchema + note over LyricDB: triggers audit table + Worker->>LyricDB: Insert record in `audit_submitted_data` table with migration error end - end - LyricAPI->>User: 200 OK: Return Dictionary, Category and migrationId (if required) - alt - rect rgb(230, 242, 255) - note over LyricAPI: [Migration] validation occurs in the brackground - LyricAPI->>+LyricDB: Retrieve all existing submitted Data in category - LyricDB-->>-LyricAPI: All Submitted Data in Category - LyricAPI->>LyricAPI: Validate record against new schema - - loop each record failed validation - LyricAPI->>LyricDB: Update record in `submitted_data` table, with isValid = FALSE, lastValidSchema - note over LyricDB: triggers audit table - LyricAPI->>LyricDB: Insert record in `audit_submitted_data` table with migration error - end - - alt if Migration fails - LyricAPI->>LyricDB: Change Migration (`category_migration` table) Status to FAILED - end - - LyricAPI->>LyricDB: Change Migration (`category_migration` table) Status to COMPLETED + alt if Migration succeed + Worker->>LyricDB: Change Migration (`category_migration` table) Status to COMPLETED + else if migration fails + Worker->>LyricDB: Change Migration (`category_migration` table) Status to FAILED end end ``` From 2487bbebde17cbb4ebdd6d090cb9b4b758373e57 Mon Sep 17 00:00:00 2001 From: Leonardo Rivera Date: Tue, 7 Apr 2026 16:37:55 -0400 Subject: [PATCH 11/33] migration table index --- packages/data-model/docs/schema.dbml | 7 ++++ .../migrations/0013_dictionary_migration.sql | 4 +++ .../migrations/meta/0010_snapshot.json | 2 +- .../migrations/meta/0013_snapshot.json | 33 +++++++++++++++++-- .../data-model/migrations/meta/_journal.json | 2 +- .../src/models/dictionary_migration.ts | 11 +++++-- 6 files changed, 53 insertions(+), 6 deletions(-) diff --git a/packages/data-model/docs/schema.dbml b/packages/data-model/docs/schema.dbml index 096c9927..866a56e0 100644 --- a/packages/data-model/docs/schema.dbml +++ b/packages/data-model/docs/schema.dbml @@ -76,6 +76,13 @@ table dictionary_migration { created_by varchar updated_at timestamp updated_by varchar + + indexes { + category_id [name: 'dictionary_migration_category_id_index'] + from_dictionary_id [name: 'dictionary_migration_from_dictionary_id_index'] + to_dictionary_id [name: 'dictionary_migration_to_dictionary_id_index'] + submission_id [name: 'dictionary_migration_submission_id_index'] + } } table submissions { diff --git a/packages/data-model/migrations/0013_dictionary_migration.sql b/packages/data-model/migrations/0013_dictionary_migration.sql index 2d873c9b..28f8671d 100644 --- a/packages/data-model/migrations/0013_dictionary_migration.sql +++ b/packages/data-model/migrations/0013_dictionary_migration.sql @@ -20,6 +20,10 @@ CREATE TABLE IF NOT EXISTS "dictionary_migration" ( ); --> statement-breakpoint ALTER TABLE "audit_submitted_data" ADD COLUMN "errors" jsonb;--> statement-breakpoint +CREATE INDEX IF NOT EXISTS "dictionary_migration_category_id_index" ON "dictionary_migration" ("category_id");--> statement-breakpoint +CREATE INDEX IF NOT EXISTS "dictionary_migration_from_dictionary_id_index" ON "dictionary_migration" ("from_dictionary_id");--> statement-breakpoint +CREATE INDEX IF NOT EXISTS "dictionary_migration_to_dictionary_id_index" ON "dictionary_migration" ("to_dictionary_id");--> statement-breakpoint +CREATE INDEX IF NOT EXISTS "dictionary_migration_submission_id_index" ON "dictionary_migration" ("submission_id");--> statement-breakpoint DO $$ BEGIN ALTER TABLE "dictionary_migration" ADD CONSTRAINT "dictionary_migration_category_id_dictionary_categories_id_fk" FOREIGN KEY ("category_id") REFERENCES "dictionary_categories"("id") ON DELETE no action ON UPDATE no action; EXCEPTION diff --git a/packages/data-model/migrations/meta/0010_snapshot.json b/packages/data-model/migrations/meta/0010_snapshot.json index 0ec0ca66..b81a5f40 100644 --- a/packages/data-model/migrations/meta/0010_snapshot.json +++ b/packages/data-model/migrations/meta/0010_snapshot.json @@ -607,4 +607,4 @@ "schemas": {}, "tables": {} } -} +} \ No newline at end of file diff --git a/packages/data-model/migrations/meta/0013_snapshot.json b/packages/data-model/migrations/meta/0013_snapshot.json index 6d030257..add9e4e8 100644 --- a/packages/data-model/migrations/meta/0013_snapshot.json +++ b/packages/data-model/migrations/meta/0013_snapshot.json @@ -1,5 +1,5 @@ { - "id": "f628dbb2-7f75-4fa9-9fa0-768f3272b7c3", + "id": "a4119de0-eeda-4b64-b7de-4c2831eb5df8", "prevId": "d5327542-f8b1-4ba9-b6bc-4f7c56687076", "version": "5", "dialect": "pg", @@ -365,7 +365,36 @@ "notNull": false } }, - "indexes": {}, + "indexes": { + "dictionary_migration_category_id_index": { + "name": "dictionary_migration_category_id_index", + "columns": [ + "category_id" + ], + "isUnique": false + }, + "dictionary_migration_from_dictionary_id_index": { + "name": "dictionary_migration_from_dictionary_id_index", + "columns": [ + "from_dictionary_id" + ], + "isUnique": false + }, + "dictionary_migration_to_dictionary_id_index": { + "name": "dictionary_migration_to_dictionary_id_index", + "columns": [ + "to_dictionary_id" + ], + "isUnique": false + }, + "dictionary_migration_submission_id_index": { + "name": "dictionary_migration_submission_id_index", + "columns": [ + "submission_id" + ], + "isUnique": false + } + }, "foreignKeys": { "dictionary_migration_category_id_dictionary_categories_id_fk": { "name": "dictionary_migration_category_id_dictionary_categories_id_fk", diff --git a/packages/data-model/migrations/meta/_journal.json b/packages/data-model/migrations/meta/_journal.json index cccf1ee9..961783fd 100644 --- a/packages/data-model/migrations/meta/_journal.json +++ b/packages/data-model/migrations/meta/_journal.json @@ -96,7 +96,7 @@ { "idx": 13, "version": "5", - "when": 1775072210342, + "when": 1775594188664, "tag": "0013_dictionary_migration", "breakpoints": true } diff --git a/packages/data-model/src/models/dictionary_migration.ts b/packages/data-model/src/models/dictionary_migration.ts index 17f7f0ce..1c0ab444 100644 --- a/packages/data-model/src/models/dictionary_migration.ts +++ b/packages/data-model/src/models/dictionary_migration.ts @@ -1,5 +1,5 @@ import { relations } from 'drizzle-orm'; -import { integer, pgEnum, pgTable, serial, timestamp, varchar } from 'drizzle-orm/pg-core'; +import { index, integer, pgEnum, pgTable, serial, timestamp, varchar } from 'drizzle-orm/pg-core'; import { dictionaries } from './dictionaries.js'; import { dictionaryCategories } from './dictionary_categories.js'; @@ -27,7 +27,14 @@ export const dictionaryMigration = pgTable('dictionary_migration', { createdBy: varchar('created_by'), updatedAt: timestamp('updated_at'), updatedBy: varchar('updated_by'), -}); +},(table) => { + return { + categoryIndex: index('dictionary_migration_category_id_index').on(table.categoryId), + fromDictionaryIndex: index('dictionary_migration_from_dictionary_id_index').on(table.fromDictionaryId), + toDictionaryIndex: index('dictionary_migration_to_dictionary_id_index').on(table.toDictionaryId), + submissionIndex: index('dictionary_migration_submission_id_index').on(table.submissionId), + }; + },); export const dictionaryMigrationRelations = relations(dictionaryMigration, ({ one }) => ({ category: one(dictionaryCategories, { From d351b2e1fe061f979306fe283a1fb1d233f79df5 Mon Sep 17 00:00:00 2001 From: Leonardo Rivera Date: Mon, 13 Apr 2026 14:10:57 -0400 Subject: [PATCH 12/33] fix typos and logs --- .../src/services/migrationService.ts | 22 +++++++-------- .../submission/submissionProcessor.ts | 27 ++++++++++--------- 2 files changed, 25 insertions(+), 24 deletions(-) diff --git a/packages/data-provider/src/services/migrationService.ts b/packages/data-provider/src/services/migrationService.ts index 3e933121..c603b837 100644 --- a/packages/data-provider/src/services/migrationService.ts +++ b/packages/data-provider/src/services/migrationService.ts @@ -2,7 +2,7 @@ import type { DictionaryMigration, NewDictionaryMigration } from '@overture-stac import type { BaseDependencies } from '../config/config.js'; import categoryRepository from '../repository/categoryRepository.js'; -import migrationRepository from '../repository/dictionaryMigrationRepository.js'; +import createMigrationRepository from '../repository/dictionaryMigrationRepository.js'; import submittedDataRepository from '../repository/submittedRepository.js'; import type { MigrationStatus } from '../utils/types.js'; import submissionProcessorFactory from './submission/submissionProcessor.js'; @@ -11,7 +11,7 @@ import submissionService from './submission/submissionService.js'; const migrationService = (dependencies: BaseDependencies) => { const LOG_MODULE = 'MIGRATION_SERVICE'; const { logger, onFinishCommit } = dependencies; - const migrationRepo = migrationRepository(dependencies); + const migrationRepository = createMigrationRepository(dependencies); const submissionProcessor = submissionProcessorFactory.create(dependencies); /** @@ -29,7 +29,7 @@ const migrationService = (dependencies: BaseDependencies) => { userName: string; }): Promise => { try { - const updatedMigration = await migrationRepo.update(migrationId, { + const updatedMigration = await migrationRepository.update(migrationId, { status, updatedAt: new Date(), updatedBy: userName, @@ -49,7 +49,7 @@ const migrationService = (dependencies: BaseDependencies) => { */ const getActiveMigrationByCategoryId = async (categoryId: number): Promise => { try { - const migrations = await migrationRepo.getMigrationsByCategoryId( + const migrations = await migrationRepository.getMigrationsByCategoryId( categoryId, { page: 1, pageSize: 1 }, { status: 'IN-PROGRESS' }, @@ -86,7 +86,7 @@ const migrationService = (dependencies: BaseDependencies) => { }): Promise => { const { getOrCreateActiveSubmission } = submissionService(dependencies); try { - const existingMigrationResult = await migrationRepo.getMigrationsByCategoryId( + const findMigrationResult = await migrationRepository.getMigrationsByCategoryId( categoryId, { page: 1, pageSize: 1 }, { fromDictionaryId, toDictionaryId }, @@ -96,13 +96,13 @@ const migrationService = (dependencies: BaseDependencies) => { let submissionId: number; // Migration already exists, update retries count - if (existingMigrationResult.length > 0) { - const migration = existingMigrationResult[0]; - const updatedRetriesCount = migration.retries + 1; + if (findMigrationResult.length > 0) { + const existingMigration = findMigrationResult[0]; + const updatedRetriesCount = existingMigration.retries + 1; - submissionId = migration.submissionId; + submissionId = existingMigration.submissionId; - migrationId = await migrationRepo.update(migration.id, { + migrationId = await migrationRepository.update(existingMigration.id, { retries: updatedRetriesCount, updatedBy: userName, updatedAt: new Date(), @@ -130,7 +130,7 @@ const migrationService = (dependencies: BaseDependencies) => { createdAt: new Date(), }; - migrationId = await migrationRepo.save(newMigration); + migrationId = await migrationRepository.save(newMigration); logger.info(LOG_MODULE, `Creating migration record for categoryId '${categoryId}'`); } diff --git a/packages/data-provider/src/services/submission/submissionProcessor.ts b/packages/data-provider/src/services/submission/submissionProcessor.ts index 19696bf0..225f2788 100644 --- a/packages/data-provider/src/services/submission/submissionProcessor.ts +++ b/packages/data-provider/src/services/submission/submissionProcessor.ts @@ -62,7 +62,7 @@ import createSubmittedDataRelationsSearch from '../submittedData/searchDataRelat const createSubmissionProcessor = (dependencies: BaseDependencies) => { const LOG_MODULE = 'SUBMISSION_PROCESSOR_SERVICE'; - const categoryRepositry = createCategoryRepository(dependencies); + const categoryRepository = createCategoryRepository(dependencies); const dictionaryRepository = createDictionaryRepository(dependencies); const submissionRepository = createSubmissionRepository(dependencies); const submittedDataRepository = createSubmittedDataRepository(dependencies); @@ -359,7 +359,7 @@ const createSubmissionProcessor = (dependencies: BaseDependencies) => { if (record.id) { const oldIsValid = record.isValid; - const update: Partial = {}; + const inputUpdate: Partial = {}; const submisionUpdateData = dataToValidate.updates?.[record.systemId]; if (submisionUpdateData) { @@ -367,27 +367,27 @@ const createSubmissionProcessor = (dependencies: BaseDependencies) => { LOG_MODULE, `Updating submittedData system ID '${record.systemId}' in entity '${entityName}'`, ); - update.data = record.data; + inputUpdate.data = record.data; } if (oldIsValid !== newIsValid) { - update.isValid = newIsValid; + inputUpdate.isValid = newIsValid; if (newIsValid) { - update.lastValidSchemaId = dictionary.id; + inputUpdate.lastValidSchemaId = dictionary.id; } } - if (Object.keys(update).length === 0) { + if (Object.keys(inputUpdate).length === 0) { return; } - update.updatedBy = username; + inputUpdate.updatedBy = username; if (newIsValid) { - update.lastValidSchemaId = dictionary.id; + inputUpdate.lastValidSchemaId = dictionary.id; } updatesToSave.push({ submittedDataId: record.id, - data: update, + data: inputUpdate, audit: { dataDiff: { old: submisionUpdateData?.old ?? {}, new: submisionUpdateData?.new ?? {} }, errors: errors, @@ -398,7 +398,7 @@ const createSubmissionProcessor = (dependencies: BaseDependencies) => { }); // Check if either 'data' or 'isValid' keys has been updated - if ('data' in update || 'isValid' in update) { + if ('data' in inputUpdate || 'isValid' in inputUpdate) { resultCommit.updates.push({ data: record.data, entityName, @@ -435,7 +435,7 @@ const createSubmissionProcessor = (dependencies: BaseDependencies) => { deletesToProcess.push({ submissionId: submission.id, - systemId: systemId, + systemId, diff: computeDataDiff(data, null), username, }); @@ -496,7 +496,7 @@ const createSubmissionProcessor = (dependencies: BaseDependencies) => { * @returns {Promise} ID of the Submission updated */ const performDataValidation = async (submissionId: number): Promise => { - const { getActiveDictionaryByCategory } = categoryRepositry; + const { getActiveDictionaryByCategory } = categoryRepository; const { getSubmittedDataByCategoryIdAndOrganization } = submittedDataRepository; const { getSubmissionDetailsById } = submissionRepository; @@ -825,7 +825,7 @@ const createSubmissionProcessor = (dependencies: BaseDependencies) => { ), ) .join(', '); - logger.info(`Processing files: ${fileSummaries}`); + logger.info(LOG_MODULE, `Processing files: ${fileSummaries}`); // TODO: This only gets a summary, we need to insert data into an active submission so we need all the insert statements. @@ -865,6 +865,7 @@ const createSubmissionProcessor = (dependencies: BaseDependencies) => { logger.error(`There was an error processing submitted files: ${fileSummaries}`, JSON.stringify(error)); } logger.info( + LOG_MODULE, `Finished addFilesToSubmissionAsync for active submission in category "${params.categoryId}" for organization "${params.organization}" submitted by user "${params.username}"`, ); }; From fb0526385592e5ca523baed2796848b527f4f37b Mon Sep 17 00:00:00 2001 From: Leonardo Rivera Date: Mon, 13 Apr 2026 16:47:45 -0400 Subject: [PATCH 13/33] migration audit and logs --- .../data-provider/src/repository/submittedRepository.ts | 6 +----- .../src/services/submission/submissionProcessor.ts | 7 ++++++- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/packages/data-provider/src/repository/submittedRepository.ts b/packages/data-provider/src/repository/submittedRepository.ts index f0a54b9d..7b626b57 100644 --- a/packages/data-provider/src/repository/submittedRepository.ts +++ b/packages/data-provider/src/repository/submittedRepository.ts @@ -445,11 +445,7 @@ const repository = (dependencies: BaseDependencies) => { .returning(); updatedRecords.push(updated[0]); - if ( - features?.audit?.enabled && - Object.keys(u.audit.dataDiff.new).length && - Object.keys(u.audit.dataDiff.old).length - ) { + if (features?.audit?.enabled) { await auditUpdateSubmittedData( { dataDiff: u.audit.dataDiff, diff --git a/packages/data-provider/src/services/submission/submissionProcessor.ts b/packages/data-provider/src/services/submission/submissionProcessor.ts index 225f2788..8231ee30 100644 --- a/packages/data-provider/src/services/submission/submissionProcessor.ts +++ b/packages/data-provider/src/services/submission/submissionProcessor.ts @@ -468,6 +468,11 @@ const createSubmissionProcessor = (dependencies: BaseDependencies) => { }, tx, ); + + logger.info( + LOG_MODULE, + `Finished processing data changes for submission '${submission.id}', updating submission status to 'COMMITTED'.`, + ); }); return { @@ -483,7 +488,7 @@ const createSubmissionProcessor = (dependencies: BaseDependencies) => { `Unable to complete performCommitSubmissionAsync for submission ${params.submissionId}, an error was thrown during execution`, message, ); - logger.error(error); + logger.error(LOG_MODULE, error); throw error; } }; From b5d67567a02bee541b6a7c3e7e240c4ba78d44e0 Mon Sep 17 00:00:00 2001 From: Leonardo Rivera Date: Tue, 14 Apr 2026 11:14:25 -0400 Subject: [PATCH 14/33] migration on worker thread --- .../src/services/migrationService.ts | 38 +++++++++++-------- .../src/workers/dictionaryMigrationWorker.ts | 24 ++++++++++++ packages/data-provider/src/workers/types.ts | 16 ++++++++ .../src/workers/workerContext.ts | 3 ++ .../src/workers/workerPoolManager.ts | 11 ++++++ .../data-provider/src/workers/workerpool.ts | 15 +++++++- 6 files changed, 91 insertions(+), 16 deletions(-) create mode 100644 packages/data-provider/src/workers/dictionaryMigrationWorker.ts diff --git a/packages/data-provider/src/services/migrationService.ts b/packages/data-provider/src/services/migrationService.ts index c603b837..277178d9 100644 --- a/packages/data-provider/src/services/migrationService.ts +++ b/packages/data-provider/src/services/migrationService.ts @@ -69,7 +69,7 @@ const migrationService = (dependencies: BaseDependencies) => { /** * Creates a Migration record or update retries if one exists. - * It starts running migration asynchronously + * Then, it starts running migration in a worker thread * @param param0 * @returns The ID of the initiated or updated migration */ @@ -135,15 +135,11 @@ const migrationService = (dependencies: BaseDependencies) => { logger.info(LOG_MODULE, `Creating migration record for categoryId '${categoryId}'`); } - // Start migration asynchronously - performMigrationValidation({ categoryId, submissionId, userName }) - .then(() => { - finalizeMigration({ migrationId, status: 'COMPLETED', userName }); - }) - .catch(async (error) => { - logger.error(LOG_MODULE, `Error during migration validation for categoryId '${categoryId}'`, error); - finalizeMigration({ migrationId, status: 'FAILED', userName }); - }); + // Perform dictionary migration in a worker thread + dependencies.workerPool.dictionaryMigration({ + migrationId, + userName, + }); logger.info(LOG_MODULE, `Migration initiated for categoryId '${categoryId}'`); return migrationId; @@ -153,14 +149,19 @@ const migrationService = (dependencies: BaseDependencies) => { } }; - /** Execute submitted data validation for the migration */ + /** + * **This function is designed to be executed in a worker thread.** + * Performs the Submitted data validation for a given migration, + * it iterates over all organizations and validates the submitted data for each of them. + * @param migrationId The ID of the migration to perform + * @param userName The name of the user that initiated the migration (for audit purposes) + * @returns void + */ const performMigrationValidation = async ({ - categoryId, - submissionId, + migrationId, userName, }: { - categoryId: number; - submissionId: number; + migrationId: number; userName: string; }): Promise => { const { getAllOrganizationsByCategoryId, getSubmittedDataByCategoryIdAndOrganization } = @@ -168,6 +169,13 @@ const migrationService = (dependencies: BaseDependencies) => { const { getActiveDictionaryByCategory } = categoryRepository(dependencies); const { performCommitSubmissionAsync } = submissionProcessor; + const migration = await migrationRepository.getMigrationById(migrationId); + if (!migration) { + throw new Error(`Migration with id '${migrationId}' not found`); + } + + const { categoryId, submissionId } = migration; + const dictionary = await getActiveDictionaryByCategory(categoryId); if (!dictionary) { throw new Error(`Dictionary in category '${categoryId}' not found`); diff --git a/packages/data-provider/src/workers/dictionaryMigrationWorker.ts b/packages/data-provider/src/workers/dictionaryMigrationWorker.ts new file mode 100644 index 00000000..7e867b8a --- /dev/null +++ b/packages/data-provider/src/workers/dictionaryMigrationWorker.ts @@ -0,0 +1,24 @@ +import createMigrationService from '../services/migrationService.js'; +import type { DictionaryMigrationWorkerInput } from './types.js'; +import { getWorkerDependencies } from './workerContext.js'; + +export const performDictionaryMigration = async ({ migrationId, userName }: DictionaryMigrationWorkerInput) => { + const dependencies = getWorkerDependencies(); + const migrationService = createMigrationService(dependencies); + + try { + await migrationService.performMigrationValidation({ migrationId, userName }); + migrationService.finalizeMigration({ + migrationId, + status: 'COMPLETED', + userName, + }); + } catch (error) { + console.error('Error performing migration validation:', error); + migrationService.finalizeMigration({ + migrationId, + status: 'FAILED', + userName, + }); + } +}; diff --git a/packages/data-provider/src/workers/types.ts b/packages/data-provider/src/workers/types.ts index a166409d..55da73ab 100644 --- a/packages/data-provider/src/workers/types.ts +++ b/packages/data-provider/src/workers/types.ts @@ -10,6 +10,11 @@ export type DataValidationWorkerInput = { submissionId: number; }; +export type DictionaryMigrationWorkerInput = { + migrationId: number; + userName: string; +}; + export type WorkerContext = { dependencies: BaseDependencies; }; @@ -40,6 +45,11 @@ export type WorkerFunctions = { * @returns A void promise that resolves when the data validation process is complete */ dataValidation(input: DataValidationWorkerInput): Promise; + /** + * Uses a worker thread from the pool to execute the dictionary migration process. + * @param input + */ + dictionaryMigration(input: DictionaryMigrationWorkerInput): Promise; }; /** @@ -68,4 +78,10 @@ export type WorkerProxy = { * @returns A promise that resolves the submission ID */ dataValidation: (input: DataValidationWorkerInput) => Promise; + /** + * This function is executed in the worker thread to start the dictionary migration process. + * @param input + * @returns + */ + dictionaryMigration: (input: DictionaryMigrationWorkerInput) => Promise; }; diff --git a/packages/data-provider/src/workers/workerContext.ts b/packages/data-provider/src/workers/workerContext.ts index a706f616..5062baa3 100644 --- a/packages/data-provider/src/workers/workerContext.ts +++ b/packages/data-provider/src/workers/workerContext.ts @@ -37,6 +37,9 @@ export const initializeWorkerContext = async (configData: AppConfig): Promise { throw new Error('Worker pool functions cannot be called from within the worker'); }, + dictionaryMigration: async () => { + throw new Error('Worker pool functions cannot be called from within the worker'); + }, terminate: async () => { throw new Error('Worker pool functions cannot be called from within the worker'); }, diff --git a/packages/data-provider/src/workers/workerPoolManager.ts b/packages/data-provider/src/workers/workerPoolManager.ts index 5386400f..5784a3d2 100644 --- a/packages/data-provider/src/workers/workerPoolManager.ts +++ b/packages/data-provider/src/workers/workerPoolManager.ts @@ -84,6 +84,17 @@ export const createWorkerPool = (configData: AppConfig): WorkerFunctions => { // This ensures the main thread is not affected by worker errors and can continue processing other tasks. } }, + dictionaryMigration: async (input) => { + const proxy = await readyProxy; // wait for worker to initialize before using + try { + await proxy.dictionaryMigration(input); + } catch (error) { + const errMessage = error instanceof Error ? error.message : String(error); + logger.error(LOG_MODULE, `Worker pool execution failed for dictionaryMigration: ${errMessage}`); + // Do not re-throw error since dictionaryMigration in the worker is designed to not throw errors, but log them instead. + // This ensures the main thread is not affected by worker errors and can continue processing other tasks. + } + }, terminate: async (): Promise => { await pool.terminate(); }, diff --git a/packages/data-provider/src/workers/workerpool.ts b/packages/data-provider/src/workers/workerpool.ts index bdcbcdec..498373f1 100644 --- a/packages/data-provider/src/workers/workerpool.ts +++ b/packages/data-provider/src/workers/workerpool.ts @@ -3,7 +3,13 @@ import * as workerpool from 'workerpool'; import type { AppConfig, ResultOnCommit } from '../../index.js'; import { processCommitSubmission } from './commitSubmissionWorker.js'; import { processDataValidation } from './dataValidationWorker.js'; -import type { CommitWorkerInput, DataValidationWorkerInput, WorkerProxy } from './types.js'; +import { performDictionaryMigration } from './dictionaryMigrationWorker.js'; +import type { + CommitWorkerInput, + DataValidationWorkerInput, + DictionaryMigrationWorkerInput, + WorkerProxy, +} from './types.js'; import { initializeWorkerContext } from './workerContext.js'; // Store initialization promise once it has been initiated. @@ -35,6 +41,13 @@ const workerProxy: WorkerProxy = { await initializeWorkerPromise; return await processDataValidation(input); }, + dictionaryMigration: async (input: DictionaryMigrationWorkerInput): Promise => { + if (!initializeWorkerPromise) { + throw new Error('Worker not initialized. Call initializeWorker first.'); + } + await initializeWorkerPromise; + return await performDictionaryMigration(input); + }, }; workerpool.worker(workerProxy); From f5af2365332e5f2eb11053825075a35bb1ed2a88 Mon Sep 17 00:00:00 2001 From: Leonardo Rivera Date: Wed, 15 Apr 2026 16:29:02 -0400 Subject: [PATCH 15/33] GET migration endpoints --- apps/server/src/server.ts | 1 + apps/server/swagger/migration-api.yml | 48 +++++++++ apps/server/swagger/schemas.yml | 72 ++++++++++++++ .../src/models/dictionary_migration.ts | 4 +- packages/data-provider/index.ts | 1 + .../src/controllers/migrationController.ts | 63 ++++++++++++ packages/data-provider/src/core/provider.ts | 8 ++ .../dictionaryMigrationRepository.ts | 98 ++++++++++++++++--- .../src/routers/migrationRouter.ts | 27 +++++ .../src/services/migrationService.ts | 54 ++++++++-- .../data-provider/src/utils/migrationUtils.ts | 20 ++++ packages/data-provider/src/utils/schemas.ts | 29 ++++++ 12 files changed, 402 insertions(+), 23 deletions(-) create mode 100644 apps/server/swagger/migration-api.yml create mode 100644 packages/data-provider/src/controllers/migrationController.ts create mode 100644 packages/data-provider/src/routers/migrationRouter.ts create mode 100644 packages/data-provider/src/utils/migrationUtils.ts diff --git a/apps/server/src/server.ts b/apps/server/src/server.ts index 9e77e682..a57411a2 100644 --- a/apps/server/src/server.ts +++ b/apps/server/src/server.ts @@ -45,6 +45,7 @@ app.use('/audit', lyricProvider.routers.audit); app.use('/category', lyricProvider.routers.category); app.use('/data', lyricProvider.routers.submittedData); app.use('/dictionary', lyricProvider.routers.dictionary); +app.use('/migration', lyricProvider.routers.migration); app.use('/submission', lyricProvider.routers.submission); app.use('/validator', lyricProvider.routers.validator); diff --git a/apps/server/swagger/migration-api.yml b/apps/server/swagger/migration-api.yml new file mode 100644 index 00000000..b51ecddf --- /dev/null +++ b/apps/server/swagger/migration-api.yml @@ -0,0 +1,48 @@ +# Description of Migration API + +/migration/{migrationId}: + get: + summary: Get Migration general information by ID + tags: + - Migration + parameters: + - name: migrationId + in: path + type: string + required: true + responses: + 200: + description: Migration general information + content: + application/json: + schema: + $ref: '#/components/schemas/MigrationResult' + 404: + $ref: '#/components/responses/NotFound' + 500: + $ref: '#/components/responses/ServerError' + 503: + $ref: '#/components/responses/ServiceUnavailableError' + +/migration/category/{categoryId}: + get: + summary: Retrieve the Migrations for a category + tags: + - Migration + parameters: + - $ref: '#/components/parameters/path/CategoryId' + - $ref: '#/components/parameters/query/Page' + - $ref: '#/components/parameters/query/PageSize' + responses: + 200: + description: List of Migrations + content: + application/json: + schema: + $ref: '#/components/schemas/ListMigrationsResult' + 404: + $ref: '#/components/responses/NotFound' + 500: + $ref: '#/components/responses/ServerError' + 503: + $ref: '#/components/responses/ServiceUnavailableError' diff --git a/apps/server/swagger/schemas.yml b/apps/server/swagger/schemas.yml index 74bbf37f..60db5c1b 100644 --- a/apps/server/swagger/schemas.yml +++ b/apps/server/swagger/schemas.yml @@ -438,6 +438,78 @@ components: type: string description: Name of the Category + ListMigrationsResult: + type: object + properties: + pagination: + type: object + properties: + currentPage: + type: number + pageSize: + type: number + totalPages: + type: number + totalRecords: + type: number + records: + type: array + items: + $ref: '#/components/schemas/MigrationResult' + + MigrationResult: + type: object + properties: + id: + type: number + description: ID of the Migration + category: + type: object + description: Category this dictionary belongs in + properties: + id: + type: number + name: + type: string + fromDictionary: + type: object + description: Previous Dictionary used in this category + properties: + name: + type: string + version: + type: string + toDictionary: + type: object + description: New Dictionary used in this category + properties: + name: + type: string + version: + type: string + submissionId: + type: number + description: ID of the Submission used for the migration + status: + type: string + description: Status of the Migration + enum: ['IN-PROGRESS', 'COMPLETED', 'FAILED'] + retries: + type: number + description: Number of retries of the migration + createdAt: + type: string + description: Date and time of creation + createdBy: + type: string + description: User name who created the migration + udpatedAt: + type: string + description: Date and time of latest update + updatedBy: + type: string + description: User name who last updated the migration + RegisterDictionaryResult: type: object properties: diff --git a/packages/data-model/src/models/dictionary_migration.ts b/packages/data-model/src/models/dictionary_migration.ts index 17f7f0ce..d6abe2b0 100644 --- a/packages/data-model/src/models/dictionary_migration.ts +++ b/packages/data-model/src/models/dictionary_migration.ts @@ -34,12 +34,12 @@ export const dictionaryMigrationRelations = relations(dictionaryMigration, ({ on fields: [dictionaryMigration.categoryId], references: [dictionaryCategories.id], }), - fromDictionaryId: one(dictionaries, { + fromDictionary: one(dictionaries, { fields: [dictionaryMigration.fromDictionaryId], references: [dictionaries.id], relationName: 'fromDictionary', }), - toDictionaryId: one(dictionaries, { + toDictionary: one(dictionaries, { fields: [dictionaryMigration.toDictionaryId], references: [dictionaries.id], relationName: 'toDictionary', diff --git a/packages/data-provider/index.ts b/packages/data-provider/index.ts index e428dffe..12502ed7 100644 --- a/packages/data-provider/index.ts +++ b/packages/data-provider/index.ts @@ -12,6 +12,7 @@ export { type DbConfig, migrate } from '@overture-stack/lyric-data-model'; // routes export { default as dictionaryRouters } from './src/routers/dictionaryRouter.js'; +export { default as migrationRouter } from './src/routers/migrationRouter.js'; export { default as submissionRouter } from './src/routers/submissionRouter.js'; export { default as submittedDataRouter } from './src/routers/submittedDataRouter.js'; diff --git a/packages/data-provider/src/controllers/migrationController.ts b/packages/data-provider/src/controllers/migrationController.ts new file mode 100644 index 00000000..a14969cd --- /dev/null +++ b/packages/data-provider/src/controllers/migrationController.ts @@ -0,0 +1,63 @@ +import { BaseDependencies } from '../config/config.js'; +import createMigrationService from '../services/migrationService.js'; +import { NotFound } from '../utils/errors.js'; +import { validateRequest } from '../utils/requestValidation.js'; +import { migrationByIdRequestSchema, migrationsByCategoryIdRequestSchema } from '../utils/schemas.js'; + +const controller = (dependencies: BaseDependencies) => { + const migrationService = createMigrationService(dependencies); + const { logger } = dependencies; + const LOG_MODULE = 'MIGRATION_CONTROLLER'; + const defaultPage = 1; + const defaultPageSize = 20; + + return { + getMigrationById: validateRequest(migrationByIdRequestSchema, async (req, res, next) => { + try { + const migrationId = Number(req.params.migrationId); + + logger.info(LOG_MODULE, `Request Migration id '${migrationId}'`); + + const submission = await migrationService.getMigrationById(migrationId); + + if (!submission) { + throw new NotFound('Submission not found'); + } + return res.send(submission); + } catch (error) { + next(error); + } + }), + getMigrationsByCategoryId: validateRequest(migrationsByCategoryIdRequestSchema, async (req, res, next) => { + try { + const categoryId = Number(req.params.categoryId); + const page = parseInt(String(req.query.page)) || defaultPage; + const pageSize = parseInt(String(req.query.pageSize)) || defaultPageSize; + + logger.info(LOG_MODULE, `Request Migrations by category Id '${categoryId}'`); + + const submissions = await migrationService.getMigrationsByCategoryId(categoryId, { page, pageSize }); + + if (submissions.result.length === 0) { + throw new NotFound('Submissions not found'); + } + + const response = { + pagination: { + currentPage: page, + pageSize: pageSize, + totalPages: Math.ceil(submissions.metadata.totalRecords / pageSize), + totalRecords: submissions.metadata.totalRecords, + }, + records: submissions.result, + }; + + return res.send(response); + } catch (error) { + next(error); + } + }), + }; +}; + +export default controller; diff --git a/packages/data-provider/src/core/provider.ts b/packages/data-provider/src/core/provider.ts index 54fd260d..44f5df04 100644 --- a/packages/data-provider/src/core/provider.ts +++ b/packages/data-provider/src/core/provider.ts @@ -4,23 +4,27 @@ import { getLogger } from '../config/logger.js'; import auditController from '../controllers/auditController.js'; import categoryController from '../controllers/categoryController.js'; import dictionaryController from '../controllers/dictionaryController.js'; +import migrationController from '../controllers/migrationController.js'; import submissionController from '../controllers/submissionController.js'; import submittedDataController from '../controllers/submittedDataController.js'; import validationController from '../controllers/validationController.js'; import submissionRepository from '../repository/activeSubmissionRepository.js'; import auditRepository from '../repository/auditRepository.js'; import categoryRepository from '../repository/categoryRepository.js'; +import migrationRepository from '../repository/dictionaryMigrationRepository.js'; import dictionaryRepository from '../repository/dictionaryRepository.js'; import submittedDataRepository from '../repository/submittedRepository.js'; import auditRouter from '../routers/auditRouter.js'; import categoryRouter from '../routers/categoryRouter.js'; import dictionaryRouter from '../routers/dictionaryRouter.js'; +import migrationRouter from '../routers/migrationRouter.js'; import submissionRouter from '../routers/submissionRouter.js'; import submittedDataRouter from '../routers/submittedDataRouter.js'; import validationRouter from '../routers/validationRouter.js'; import auditService from '../services/auditService.js'; import categoryService from '../services/categoryService.js'; import dictionaryService from '../services/dictionaryService.js'; +import migrationService from '../services/migrationService.js'; import submissionService from '../services/submission/submission.js'; import submittedDataService from '../services/submittedData/submmittedData.js'; import validationService from '../services/validationService.js'; @@ -55,6 +59,7 @@ const provider = (configData: AppConfig) => { audit: auditRouter({ baseDependencies: baseDeps, authConfig: configData.auth }), category: categoryRouter({ baseDependencies: baseDeps, authConfig: configData.auth }), dictionary: dictionaryRouter({ baseDependencies: baseDeps, authConfig: configData.auth }), + migration: migrationRouter({ baseDependencies: baseDeps, authConfig: configData.auth }), submission: submissionRouter({ baseDependencies: baseDeps, authConfig: configData.auth }), submittedData: submittedDataRouter({ baseDependencies: baseDeps, authConfig: configData.auth }), validator: validationRouter({ @@ -67,6 +72,7 @@ const provider = (configData: AppConfig) => { audit: auditController(baseDeps), category: categoryController(baseDeps), dictionary: dictionaryController(baseDeps), + migration: migrationController(baseDeps), submission: submissionController({ baseDependencies: baseDeps, authConfig: { enabled: configData.auth.enabled }, @@ -78,6 +84,7 @@ const provider = (configData: AppConfig) => { audit: auditService(baseDeps), category: categoryService(baseDeps), dictionary: dictionaryService(baseDeps), + migration: migrationService(baseDeps), submission: submissionService(baseDeps), submittedData: submittedDataService(baseDeps), validation: validationService(baseDeps), @@ -86,6 +93,7 @@ const provider = (configData: AppConfig) => { audit: auditRepository(baseDeps), category: categoryRepository(baseDeps), dictionary: dictionaryRepository(baseDeps), + migration: migrationRepository(baseDeps), submission: submissionRepository(baseDeps), submittedData: submittedDataRepository(baseDeps), }, diff --git a/packages/data-provider/src/repository/dictionaryMigrationRepository.ts b/packages/data-provider/src/repository/dictionaryMigrationRepository.ts index 32d933db..b1d1af6e 100644 --- a/packages/data-provider/src/repository/dictionaryMigrationRepository.ts +++ b/packages/data-provider/src/repository/dictionaryMigrationRepository.ts @@ -1,6 +1,7 @@ -import { and, eq } from 'drizzle-orm'; +import { and, count, eq, type SQL } from 'drizzle-orm'; import { + type Dictionary, type DictionaryMigration, dictionaryMigration, type NewDictionaryMigration, @@ -8,12 +9,62 @@ import { import type { BaseDependencies } from '../config/config.js'; import { ServiceUnavailable } from '../utils/errors.js'; -import type { MigrationStatus, PaginationOptions } from '../utils/types.js'; +import type { BooleanTrueObject, MigrationStatus, PaginationOptions } from '../utils/types.js'; + +type MigrationRepositoryRecord = Omit; + +/** + * Defines the structure of a Migration record returned by the repository, + * includes related entities like category and dictionaries. + */ +export type MigrationRecordWithRelations = MigrationRepositoryRecord & { + category: { + id: number; + name: string; + }; + fromDictionary: { + name: string; + version: string; + } | null; + toDictionary: { + name: string; + version: string; + } | null; +}; const repository = (dependencies: BaseDependencies) => { const LOG_MODULE = 'MIGRATION_REPOSITORY'; const { db, logger } = dependencies; + const migrationRepositoryColumns = { + id: true, + status: true, + submissionId: true, + retries: true, + createdAt: true, + createdBy: true, + updatedAt: true, + updatedBy: true, + } as const satisfies Record; + + const dictionarySummaryColumns = { + columns: { + name: true, + version: true, + }, + } as const satisfies { columns: Partial> }; + + const migrationWithRelationsColumns = { + category: { + columns: { + id: true, + name: true, + }, + }, + fromDictionary: dictionarySummaryColumns, + toDictionary: dictionarySummaryColumns, + } as const satisfies Record; + return { /** * Save a new Dictionary Migration in Database @@ -33,7 +84,7 @@ const repository = (dependencies: BaseDependencies) => { } }, /** Update an existing Dictionary Migration in Database */ - update: async (migrationId: number, data: Partial): Promise => { + update: async (migrationId: number, data: Partial): Promise => { try { const updatedMigration = await db .update(dictionaryMigration) @@ -48,9 +99,11 @@ const repository = (dependencies: BaseDependencies) => { } }, /** Retrieve a Dictionary Migration by ID */ - getMigrationById: async (migrationId: number): Promise => { + getMigrationById: async (migrationId: number): Promise => { try { const migration = await db.query.dictionaryMigration.findFirst({ + columns: migrationRepositoryColumns, + with: migrationWithRelationsColumns, where: eq(dictionaryMigration.id, migrationId), }); if (migration) { @@ -69,24 +122,43 @@ const repository = (dependencies: BaseDependencies) => { categoryId: number, paginationOptions: PaginationOptions, filterOptions: { status?: MigrationStatus; fromDictionaryId?: number; toDictionaryId?: number }, - ): Promise => { + ): Promise<{ + result: MigrationRecordWithRelations[]; + metadata: { totalRecords: number; errorMessage?: string }; + }> => { const { page, pageSize } = paginationOptions; const { status, fromDictionaryId, toDictionaryId } = filterOptions; try { + const whereConditions: SQL | undefined = and( + eq(dictionaryMigration.categoryId, categoryId), + status ? eq(dictionaryMigration.status, status) : undefined, + fromDictionaryId ? eq(dictionaryMigration.fromDictionaryId, fromDictionaryId) : undefined, + toDictionaryId ? eq(dictionaryMigration.toDictionaryId, toDictionaryId) : undefined, + ); + const migrations = await db.query.dictionaryMigration.findMany({ - where: and( - eq(dictionaryMigration.categoryId, categoryId), - status ? eq(dictionaryMigration.status, status) : undefined, - fromDictionaryId ? eq(dictionaryMigration.fromDictionaryId, fromDictionaryId) : undefined, - toDictionaryId ? eq(dictionaryMigration.toDictionaryId, toDictionaryId) : undefined, - ), + where: whereConditions, + columns: migrationRepositoryColumns, + with: migrationWithRelationsColumns, orderBy: (dictionaryMigration, { desc }) => desc(dictionaryMigration.createdAt), limit: pageSize, offset: (page - 1) * pageSize, }); - logger.debug(LOG_MODULE, `Fetched ${migrations.length} migrations for categoryId '${categoryId}'`); - return migrations; + + const countMigrations = await db.select({ total: count() }).from(dictionaryMigration).where(whereConditions); + + logger.debug( + LOG_MODULE, + `Fetched ${migrations.length} migrations from a total of ${countMigrations[0].total} for categoryId '${categoryId}'`, + ); + + return { + metadata: { + totalRecords: countMigrations[0].total, + }, + result: migrations, + }; } catch (error) { logger.error(LOG_MODULE, `Failed fetching dictionary migrations for categoryId '${categoryId}'`, error); throw new ServiceUnavailable(); diff --git a/packages/data-provider/src/routers/migrationRouter.ts b/packages/data-provider/src/routers/migrationRouter.ts new file mode 100644 index 00000000..2abd41cc --- /dev/null +++ b/packages/data-provider/src/routers/migrationRouter.ts @@ -0,0 +1,27 @@ +import { json, Router, urlencoded } from 'express'; + +import { BaseDependencies } from '../config/config.js'; +import migrationController from '../controllers/migrationController.js'; +import { type AuthConfig, authMiddleware } from '../middleware/auth.js'; + +const router = ({ + baseDependencies, + authConfig, +}: { + baseDependencies: BaseDependencies; + authConfig: AuthConfig; +}): Router => { + const router = Router(); + router.use(urlencoded({ extended: false })); + router.use(json()); + + router.use(authMiddleware(authConfig)); + + router.get('/:migrationId', migrationController(baseDependencies).getMigrationById); + + router.get('/category/:categoryId', migrationController(baseDependencies).getMigrationsByCategoryId); + + return router; +}; + +export default router; diff --git a/packages/data-provider/src/services/migrationService.ts b/packages/data-provider/src/services/migrationService.ts index 85ffd5cc..5b71ada3 100644 --- a/packages/data-provider/src/services/migrationService.ts +++ b/packages/data-provider/src/services/migrationService.ts @@ -1,12 +1,13 @@ -import type { DictionaryMigration, NewDictionaryMigration } from '@overture-stack/lyric-data-model/models'; +import type { NewDictionaryMigration } from '@overture-stack/lyric-data-model/models'; import type { BaseDependencies } from '../config/config.js'; import categoryRepository from '../repository/categoryRepository.js'; -import migrationRepository from '../repository/dictionaryMigrationRepository.js'; +import migrationRepository, { type MigrationRecordWithRelations } from '../repository/dictionaryMigrationRepository.js'; import submittedDataRepository from '../repository/submittedRepository.js'; -import type { MigrationStatus } from '../utils/types.js'; +import type { MigrationStatus, PaginationOptions } from '../utils/types.js'; import processor from './submission/processor.js'; import submissionService from './submission/submission.js'; +import { formatMigrationRecord } from '../utils/migrationUtils.js'; const migrationService = (dependencies: BaseDependencies) => { const LOG_MODULE = 'MIGRATION_SERVICE'; @@ -46,16 +47,16 @@ const migrationService = (dependencies: BaseDependencies) => { * @param categoryId * @returns */ - const getActiveMigrationByCategoryId = async (categoryId: number): Promise => { + const getActiveMigrationByCategoryId = async (categoryId: number): Promise => { try { const migrations = await migrationRepo.getMigrationsByCategoryId( categoryId, { page: 1, pageSize: 1 }, { status: 'IN-PROGRESS' }, ); - if (migrations.length > 0) { + if (migrations.result.length > 0) { logger.info(LOG_MODULE, `Active migration found for categoryId '${categoryId}'`); - return migrations[0]; + return formatMigrationRecord(migrations.result[0]); } else { logger.info(LOG_MODULE, `No active migration for categoryId '${categoryId}'`); return null; @@ -66,6 +67,41 @@ const migrationService = (dependencies: BaseDependencies) => { } }; + const getMigrationById = async (migrationId: number): Promise => { + try { + const migration = await migrationRepo.getMigrationById(migrationId); + if (migration) { + logger.info(LOG_MODULE, `Migration found for migrationId '${migrationId}'`); + return formatMigrationRecord(migration); + } else { + logger.info(LOG_MODULE, `No migration found for migrationId '${migrationId}'`); + return undefined; + } + } catch (error) { + logger.error(LOG_MODULE, `Error retrieving migration with id '${migrationId}'`, error); + throw error; + } + }; + + const getMigrationsByCategoryId = async ( + categoryId: number, + paginationOptions: PaginationOptions, + ): Promise<{ metadata: { totalRecords: number; errorMessage?: string }; result: MigrationRecordWithRelations[] }> => { + try { + const migrations = await migrationRepo.getMigrationsByCategoryId(categoryId, paginationOptions, {}); + + return { + metadata: { + totalRecords: migrations.metadata.totalRecords, + }, + result: migrations.result.map(formatMigrationRecord), + }; + } catch (error) { + logger.error(LOG_MODULE, `Error retrieving migrations for categoryId '${categoryId}'`, error); + throw error; + } + }; + /** * Creates a Migration record or update retries if one exists. * It starts running migration asynchronously @@ -95,8 +131,8 @@ const migrationService = (dependencies: BaseDependencies) => { let submissionId: number; // Migration already exists, update retries count - if (existingMigrationResult.length > 0) { - const migration = existingMigrationResult[0]; + if (existingMigrationResult.result.length > 0) { + const migration = existingMigrationResult.result[0]; const updatedRetriesCount = migration.retries + 1; submissionId = migration.submissionId; @@ -200,6 +236,8 @@ const migrationService = (dependencies: BaseDependencies) => { return { finalizeMigration, getActiveMigrationByCategoryId, + getMigrationsByCategoryId, + getMigrationById, initiateMigration, performMigrationValidation, }; diff --git a/packages/data-provider/src/utils/migrationUtils.ts b/packages/data-provider/src/utils/migrationUtils.ts new file mode 100644 index 00000000..18d8d79f --- /dev/null +++ b/packages/data-provider/src/utils/migrationUtils.ts @@ -0,0 +1,20 @@ +import type { MigrationRecordWithRelations } from '../repository/dictionaryMigrationRepository.js'; + +/** + * Order the properties of the Migration Record + * @param migration + * @returns + */ +export const formatMigrationRecord = (migration: MigrationRecordWithRelations): MigrationRecordWithRelations => ({ + id: migration.id, + category: migration.category, + fromDictionary: migration.fromDictionary, + toDictionary: migration.toDictionary, + submissionId: migration.submissionId, + status: migration.status, + retries: migration.retries, + createdAt: migration.createdAt, + createdBy: migration.createdBy, + updatedAt: migration.updatedAt, + updatedBy: migration.updatedBy, +}); diff --git a/packages/data-provider/src/utils/schemas.ts b/packages/data-provider/src/utils/schemas.ts index c7b7b1c7..ffb834bb 100644 --- a/packages/data-provider/src/utils/schemas.ts +++ b/packages/data-provider/src/utils/schemas.ts @@ -124,6 +124,15 @@ const submissionIdSchema = z return isValidIdNumber(parsed); }, 'invalid submission ID'); +const migrationIdSchema = z + .string() + .trim() + .min(1) + .refine((value) => { + const parsed = parseInt(value); + return isValidIdNumber(parsed); + }, 'invalid migration ID'); + const stringNotEmpty = z.string().trim().min(1); // Common Category Path Params @@ -157,6 +166,15 @@ const submissionIdPathParamSchema = z.object({ submissionId: submissionIdSchema, }); +// Common Migration Path Params +export interface migrationIdPathParam extends ParamsDictionary { + migrationId: string; +} + +const migrationIdPathParamSchema = z.object({ + migrationId: migrationIdSchema, +}); + // Common Pagination Query Params export interface paginationQueryParams extends ParsedQs { @@ -226,6 +244,17 @@ export const dictionaryRegisterRequestSchema: RequestValidation< }), }; +// Migration Requests +export const migrationByIdRequestSchema: RequestValidation = { + pathParams: migrationIdPathParamSchema, +}; + +export const migrationsByCategoryIdRequestSchema: RequestValidation = + { + pathParams: categoryPathParamsSchema, + query: paginationQuerySchema, + }; + // Submission Requests export interface submissionsByCategoryQueryParams extends paginationQueryParams { From d7d17d073d356de85c31228b671df1c999935dbb Mon Sep 17 00:00:00 2001 From: Leonardo Rivera Date: Wed, 15 Apr 2026 17:02:49 -0400 Subject: [PATCH 16/33] fix unit tests --- .../submission/parseSubmissionSummaryResponse.spec.ts | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/packages/data-provider/test/utils/submission/parseSubmissionSummaryResponse.spec.ts b/packages/data-provider/test/utils/submission/parseSubmissionSummaryResponse.spec.ts index 6bafb950..07797cae 100644 --- a/packages/data-provider/test/utils/submission/parseSubmissionSummaryResponse.spec.ts +++ b/packages/data-provider/test/utils/submission/parseSubmissionSummaryResponse.spec.ts @@ -31,10 +31,13 @@ describe('Submission Utils - Parse a Submission object to a Summary of the Activ inserts: undefined, updates: undefined, deletes: undefined, + total: 0, }, dictionary: { name: 'books', version: '1' }, dictionaryCategory: { name: 'favorite books', id: 1 }, - errors: {}, + errors: { + total: 0, + }, organization: 'oicr', status: SUBMISSION_STATUS.VALID, createdAt: todaysDate.toISOString(), @@ -94,10 +97,13 @@ describe('Submission Utils - Parse a Submission object to a Summary of the Activ recordsCount: 1, }, }, + total: 3, }, dictionary: { name: 'books', version: '1' }, dictionaryCategory: { name: 'favorite books', id: 1 }, - errors: {}, + errors: { + total: 0, + }, organization: 'oicr', status: SUBMISSION_STATUS.VALID, createdAt: todaysDate.toISOString(), From 2758202ca380810c8595d4fd408ad65d129ee3e3 Mon Sep 17 00:00:00 2001 From: Leonardo Rivera Date: Thu, 16 Apr 2026 13:18:30 -0400 Subject: [PATCH 17/33] GET migration records --- apps/server/swagger/audit-api.yml | 2 +- apps/server/swagger/migration-api.yml | 47 +++++++-- apps/server/swagger/parameters.yml | 7 ++ apps/server/swagger/schemas.yml | 77 +++++++-------- .../src/controllers/auditController.ts | 3 +- .../src/controllers/migrationController.ts | 45 ++++++++- .../src/repository/auditRepository.ts | 96 ++++++++++--------- .../dictionaryMigrationRepository.ts | 58 ++++++++++- .../src/routers/migrationRouter.ts | 2 + .../src/services/auditService.ts | 8 +- .../src/services/migrationService.ts | 63 ++++++++++-- .../data-provider/src/utils/migrationUtils.ts | 9 +- packages/data-provider/src/utils/schemas.ts | 17 ++++ packages/data-provider/src/utils/types.ts | 3 + 14 files changed, 317 insertions(+), 120 deletions(-) diff --git a/apps/server/swagger/audit-api.yml b/apps/server/swagger/audit-api.yml index a5e148d3..c087b904 100644 --- a/apps/server/swagger/audit-api.yml +++ b/apps/server/swagger/audit-api.yml @@ -16,7 +16,7 @@ required: false schema: type: string - enum: [update, delete] + enum: [update, delete, migration] - name: systemId description: Filter events by System ID in: query diff --git a/apps/server/swagger/migration-api.yml b/apps/server/swagger/migration-api.yml index b51ecddf..090b68d2 100644 --- a/apps/server/swagger/migration-api.yml +++ b/apps/server/swagger/migration-api.yml @@ -1,22 +1,42 @@ # Description of Migration API +/migration/category/{categoryId}: + get: + summary: Retrieve the Migrations for a category + tags: + - Migration + parameters: + - $ref: '#/components/parameters/path/CategoryId' + - $ref: '#/components/parameters/query/Page' + - $ref: '#/components/parameters/query/PageSize' + responses: + 200: + description: List of Migrations + content: + application/json: + schema: + $ref: '#/components/schemas/ListMigrationsResult' + 404: + $ref: '#/components/responses/NotFound' + 500: + $ref: '#/components/responses/ServerError' + 503: + $ref: '#/components/responses/ServiceUnavailableError' + /migration/{migrationId}: get: summary: Get Migration general information by ID tags: - Migration parameters: - - name: migrationId - in: path - type: string - required: true + - $ref: '#/components/parameters/path/MigrationId' responses: 200: description: Migration general information content: application/json: schema: - $ref: '#/components/schemas/MigrationResult' + $ref: '#/components/schemas/ExtendedMigrationResult' 404: $ref: '#/components/responses/NotFound' 500: @@ -24,22 +44,29 @@ 503: $ref: '#/components/responses/ServiceUnavailableError' -/migration/category/{categoryId}: +/migration/{migrationId}/records: get: - summary: Retrieve the Migrations for a category + summary: Retrieve the records for a migration tags: - Migration parameters: - - $ref: '#/components/parameters/path/CategoryId' + - $ref: '#/components/parameters/path/MigrationId' + - isInvalid: + description: Optional query parameter to filter results to include only invalid records. Default value is false + name: isInvalid + in: query + schema: + type: boolean + required: false - $ref: '#/components/parameters/query/Page' - $ref: '#/components/parameters/query/PageSize' responses: 200: - description: List of Migrations + description: List of Migration records content: application/json: schema: - $ref: '#/components/schemas/ListMigrationsResult' + $ref: '#/components/schemas/ListMigrationRecordsResult' 404: $ref: '#/components/responses/NotFound' 500: diff --git a/apps/server/swagger/parameters.yml b/apps/server/swagger/parameters.yml index 0b6ac81e..98097c3b 100644 --- a/apps/server/swagger/parameters.yml +++ b/apps/server/swagger/parameters.yml @@ -15,6 +15,13 @@ components: schema: type: string required: true + MigrationId: + name: migrationId + in: path + required: true + schema: + type: string + description: The ID of the Migration Organization: name: organization in: path diff --git a/apps/server/swagger/schemas.yml b/apps/server/swagger/schemas.yml index 935766b1..c287fd83 100644 --- a/apps/server/swagger/schemas.yml +++ b/apps/server/swagger/schemas.yml @@ -62,20 +62,23 @@ components: $ref: '#/components/responses/Error' schemas: + Pagination: + type: object + properties: + currentPage: + type: number + pageSize: + type: number + totalPages: + type: number + totalRecords: + type: number + ListSubmissionsResult: type: object properties: pagination: - type: object - properties: - currentPage: - type: number - pageSize: - type: number - totalPages: - type: number - totalRecords: - type: number + $ref: '#/components/schemas/Pagination' records: type: array items: @@ -284,16 +287,7 @@ components: type: object properties: pagination: - type: object - properties: - currentPage: - type: number - pageSize: - type: number - totalPages: - type: number - totalRecords: - type: number + $ref: '#/components/schemas/Pagination' records: type: array items: @@ -308,7 +302,7 @@ components: event: type: string description: Type of event - enum: ['UPDATE', 'DELETE'] + enum: ['UPDATE', 'DELETE', 'MIGRATION'] dataDiff: type: object description: Captures the state of the data before the change as `old` and after the change as `new` @@ -413,16 +407,7 @@ components: type: object properties: pagination: - type: object - properties: - currentPage: - type: number - pageSize: - type: number - totalPages: - type: number - totalRecords: - type: number + $ref: '#/components/schemas/Pagination' records: type: array items: @@ -453,20 +438,21 @@ components: type: string description: Name of the Category + ListMigrationRecordsResult: + type: object + properties: + pagination: + $ref: '#/components/schemas/Pagination' + records: + type: array + items: + $ref: '#/components/schemas/ChangeHistoryRecord' + ListMigrationsResult: type: object properties: pagination: - type: object - properties: - currentPage: - type: number - pageSize: - type: number - totalPages: - type: number - totalRecords: - type: number + $ref: '#/components/schemas/Pagination' records: type: array items: @@ -525,6 +511,15 @@ components: type: string description: User name who last updated the migration + ExtendedMigrationResult: + allOf: + - $ref: '#/components/schemas/MigrationResult' + - type: object + properties: + invalidRecords: + type: number + description: Number of invalid records during the migration + RegisterDictionaryResult: type: object properties: diff --git a/packages/data-provider/src/controllers/auditController.ts b/packages/data-provider/src/controllers/auditController.ts index 091ec9a7..a72b7623 100644 --- a/packages/data-provider/src/controllers/auditController.ts +++ b/packages/data-provider/src/controllers/auditController.ts @@ -26,11 +26,12 @@ const controller = (dependencies: BaseDependencies) => { logger.info(LOG_MODULE, 'Request Audit', `categoryId '${categoryId}' organization '${organization}'`); - const auditRecords = await auditService.byCategoryIdAndOrganization(categoryId, organization, { + const auditRecords = await auditService.byCategoryIdAndOrganization(categoryId, { entityName, eventType, startDate, endDate, + organization, systemId, page, pageSize, diff --git a/packages/data-provider/src/controllers/migrationController.ts b/packages/data-provider/src/controllers/migrationController.ts index a14969cd..b5f29b92 100644 --- a/packages/data-provider/src/controllers/migrationController.ts +++ b/packages/data-provider/src/controllers/migrationController.ts @@ -2,7 +2,11 @@ import { BaseDependencies } from '../config/config.js'; import createMigrationService from '../services/migrationService.js'; import { NotFound } from '../utils/errors.js'; import { validateRequest } from '../utils/requestValidation.js'; -import { migrationByIdRequestSchema, migrationsByCategoryIdRequestSchema } from '../utils/schemas.js'; +import { + migrationByIdRequestSchema, + migrationDataRequestSchema, + migrationsByCategoryIdRequestSchema, +} from '../utils/schemas.js'; const controller = (dependencies: BaseDependencies) => { const migrationService = createMigrationService(dependencies); @@ -52,6 +56,45 @@ const controller = (dependencies: BaseDependencies) => { records: submissions.result, }; + return res.send(response); + } catch (error) { + next(error); + } + }), + getMigrationRecords: validateRequest(migrationDataRequestSchema, async (req, res, next) => { + try { + const migrationId = Number(req.params.migrationId); + + const page = parseInt(String(req.query.page)) || defaultPage; + const pageSize = parseInt(String(req.query.pageSize)) || defaultPageSize; + const entityNames = req.query.entityNames; + const organizations = req.query.organizations; + const isInvalid = req.query.isInvalid === 'true'; + + logger.info(LOG_MODULE, `Request Data Migration id '${migrationId}'`); + + const submissionRecords = await migrationService.getMigrationRecords(migrationId, { + page, + pageSize, + entityNames, + organizations, + isInvalid, + }); + + if (submissionRecords.result.length === 0) { + throw new NotFound('Records not found for the specified migration'); + } + + const response = { + pagination: { + currentPage: page, + pageSize: pageSize, + totalPages: Math.ceil(submissionRecords.metadata.totalRecords / pageSize), + totalRecords: submissionRecords.metadata.totalRecords, + }, + records: submissionRecords.result, + }; + return res.send(response); } catch (error) { next(error); diff --git a/packages/data-provider/src/repository/auditRepository.ts b/packages/data-provider/src/repository/auditRepository.ts index d8e690e9..8623b069 100644 --- a/packages/data-provider/src/repository/auditRepository.ts +++ b/packages/data-provider/src/repository/auditRepository.ts @@ -30,13 +30,19 @@ const repository = (dependencies: BaseDependencies) => { entityName, eventType, endDate, + newIsValid, + organization, startDate, + submissionId, systemId, }: { entityName?: string; eventType?: string; endDate?: string; + newIsValid?: boolean; + organization?: string; startDate?: string; + submissionId?: number; systemId?: string; }): SQL[] => { const filterArray: SQL[] = []; @@ -44,6 +50,18 @@ const repository = (dependencies: BaseDependencies) => { filterArray.push(eq(auditSubmittedData.systemId, systemId)); } + if (organization && !isEmptyString(organization)) { + filterArray.push(eq(auditSubmittedData.organization, organization)); + } + + if (submissionId) { + filterArray.push(eq(auditSubmittedData.submissionId, submissionId)); + } + + if (newIsValid !== undefined) { + filterArray.push(eq(auditSubmittedData.newDataIsValid, newIsValid)); + } + if (entityName && !isEmptyString(entityName)) { filterArray.push(eq(auditSubmittedData.entityName, entityName)); } @@ -69,98 +87,84 @@ const repository = (dependencies: BaseDependencies) => { return { /** * Returns all the records found on the the Audit table matching the Category ID, - * Organization and additional filters - * @param {number} categoryId - * @param {string} organization - * @param {object} filterOptions - * @param {string} filterOptions.entityName - * @param {string} filterOptions.eventType - * @param {string} filterOptions.startDate - * @param {string} filterOptions.endDate - * @param {string} filterOptions.systemId + * and additional filters + * @param {number} categoryId Category ID to filter the Audit records + * @param {object} filterOptions Additional filters and pagination options * @returns */ getRecordsByCategoryIdAndOrganizationPaginated: async ( categoryId: number, - organization: string, filterOptions: AuditFilterOptions, ): Promise => { - const { entityName, eventType, endDate, startDate, systemId, page, pageSize } = filterOptions; + const { + endDate, + entityName, + eventType, + newIsValid, + organization, + page, + pageSize, + startDate, + submissionId, + systemId, + } = filterOptions; try { const optionalFilter = getOptionalFilter({ + endDate, entityName, eventType, - endDate, + newIsValid, + organization, startDate, + submissionId, systemId, }); return await db.query.auditSubmittedData.findMany({ - where: and( - eq(auditSubmittedData.dictionaryCategoryId, categoryId), - eq(auditSubmittedData.organization, organization), - ...optionalFilter, - ), + where: and(eq(auditSubmittedData.dictionaryCategoryId, categoryId), ...optionalFilter), columns: paginatedColumns, orderBy: (auditSubmittedData, { asc }) => [asc(auditSubmittedData.createdAt)], limit: pageSize, offset: (page - 1) * pageSize, }); } catch (error) { - logger.error( - LOG_MODULE, - `Failed querying Audit Records with categoryId '${categoryId}' organization '${organization}'`, - error, - ); + logger.error(LOG_MODULE, `Failed querying Audit Records with categoryId '${categoryId}'`, error); throw new ServiceUnavailable(); } }, /** * Returns the total number of records found on the the Audit table matching the Category ID, - * Organization and additional filters - * @param {number} categoryId - * @param {string} organization - * @param {object} filterOptions - * @param {string} filterOptions.entityName - * @param {string} filterOptions.eventType - * @param {string} filterOptions.startDate - * @param {string} filterOptions.endDate - * @param {string} filterOptions.systemId + * and additional filters + * @param {number} categoryId Category ID to filter the Audit records + * @param {object} filterOptions Additional filters * @returns */ getTotalRecordsByCategoryIdAndOrganization: async ( categoryId: number, - organization: string, filterOptions: AuditFilterOptions, ): Promise => { - const { entityName, eventType, endDate, startDate, systemId } = filterOptions; + const { entityName, eventType, endDate, startDate, systemId, organization, submissionId, newIsValid } = + filterOptions; try { const optionalFilter = getOptionalFilter({ + endDate, entityName, eventType, - endDate, + newIsValid, + organization, startDate, + submissionId, systemId, }); const resultCount = await db .select({ total: count() }) .from(auditSubmittedData) - .where( - and( - eq(auditSubmittedData.dictionaryCategoryId, categoryId), - eq(auditSubmittedData.organization, organization), - ...optionalFilter, - ), - ); + .where(and(eq(auditSubmittedData.dictionaryCategoryId, categoryId), ...optionalFilter)); return resultCount[0].total; } catch (error) { - logger.error( - LOG_MODULE, - `Failed counting Audit Records with categoryId '${categoryId}' organization '${organization}'`, - error, - ); + logger.error(LOG_MODULE, `Failed counting Audit Records with categoryId '${categoryId}'`, error); throw new ServiceUnavailable(); } }, diff --git a/packages/data-provider/src/repository/dictionaryMigrationRepository.ts b/packages/data-provider/src/repository/dictionaryMigrationRepository.ts index b1d1af6e..27e08291 100644 --- a/packages/data-provider/src/repository/dictionaryMigrationRepository.ts +++ b/packages/data-provider/src/repository/dictionaryMigrationRepository.ts @@ -9,7 +9,8 @@ import { import type { BaseDependencies } from '../config/config.js'; import { ServiceUnavailable } from '../utils/errors.js'; -import type { BooleanTrueObject, MigrationStatus, PaginationOptions } from '../utils/types.js'; +import type { AuditRepositoryRecord, BooleanTrueObject, MigrationStatus, PaginationOptions } from '../utils/types.js'; +import createAuditRepository from './auditRepository.js'; type MigrationRepositoryRecord = Omit; @@ -36,6 +37,8 @@ const repository = (dependencies: BaseDependencies) => { const LOG_MODULE = 'MIGRATION_REPOSITORY'; const { db, logger } = dependencies; + const auditRepository = createAuditRepository(dependencies); + const migrationRepositoryColumns = { id: true, status: true, @@ -164,6 +167,59 @@ const repository = (dependencies: BaseDependencies) => { throw new ServiceUnavailable(); } }, + getMigrationRecords: async ( + migrationId: number, + options: { + page: number; + pageSize: number; + entityNames?: string | string[]; + organizations?: string | string[]; + isInvalid?: boolean; + }, + ): Promise<{ + result: AuditRepositoryRecord[]; + metadata: { totalRecords: number; errorMessage?: string }; + }> => { + try { + const migration = await db.query.dictionaryMigration.findFirst({ + where: eq(dictionaryMigration.id, migrationId), + }); + + if (!migration) { + logger.debug(LOG_MODULE, `No migration found with id '${migrationId}' when fetching migration records`); + return { + result: [], + metadata: { totalRecords: 0 }, + }; + } + + const newIsValid = options.isInvalid !== undefined ? !options.isInvalid : undefined; + + const records = await auditRepository.getRecordsByCategoryIdAndOrganizationPaginated(migration.categoryId, { + page: options.page, + pageSize: options.pageSize, + newIsValid, + submissionId: migration.submissionId, + }); + + const totalRecords = await auditRepository.getTotalRecordsByCategoryIdAndOrganization(migration.categoryId, { + page: options.page, + pageSize: options.pageSize, + newIsValid, + submissionId: migration.submissionId, + }); + + return { + metadata: { + totalRecords, + }, + result: records, + }; + } catch (error) { + logger.error(LOG_MODULE, `Failed fetching migration records for migrationId '${migrationId}'`, error); + throw new ServiceUnavailable(); + } + }, }; }; diff --git a/packages/data-provider/src/routers/migrationRouter.ts b/packages/data-provider/src/routers/migrationRouter.ts index 2abd41cc..cf0da63e 100644 --- a/packages/data-provider/src/routers/migrationRouter.ts +++ b/packages/data-provider/src/routers/migrationRouter.ts @@ -21,6 +21,8 @@ const router = ({ router.get('/category/:categoryId', migrationController(baseDependencies).getMigrationsByCategoryId); + router.get('/:migrationId/records', migrationController(baseDependencies).getMigrationRecords); + return router; }; diff --git a/packages/data-provider/src/services/auditService.ts b/packages/data-provider/src/services/auditService.ts index 549fd4b5..6cea8c61 100644 --- a/packages/data-provider/src/services/auditService.ts +++ b/packages/data-provider/src/services/auditService.ts @@ -13,7 +13,6 @@ const auditService = (dependencies: BaseDependencies) => { return { byCategoryIdAndOrganization: async ( categoryId: number, - organization: string, filterOptions: AuditFilterOptions, ): Promise<{ data: AuditDataResponse[]; @@ -29,7 +28,6 @@ const auditService = (dependencies: BaseDependencies) => { const recordsPaginated = await auditRepo.getRecordsByCategoryIdAndOrganizationPaginated( categoryId, - organization, filterOptions, ); @@ -37,11 +35,7 @@ const auditService = (dependencies: BaseDependencies) => { throw new NotFound('No data found'); } - const totalRecords = await auditRepo.getTotalRecordsByCategoryIdAndOrganization( - categoryId, - organization, - filterOptions, - ); + const totalRecords = await auditRepo.getTotalRecordsByCategoryIdAndOrganization(categoryId, filterOptions); logger.info(LOG_MODULE, `Retrieved '${recordsPaginated.length}' Submitted data on categoryId '${categoryId}'`); diff --git a/packages/data-provider/src/services/migrationService.ts b/packages/data-provider/src/services/migrationService.ts index 5e005c71..174624f0 100644 --- a/packages/data-provider/src/services/migrationService.ts +++ b/packages/data-provider/src/services/migrationService.ts @@ -1,20 +1,25 @@ import type { NewDictionaryMigration } from '@overture-stack/lyric-data-model/models'; import type { BaseDependencies } from '../config/config.js'; -import categoryRepository from '../repository/categoryRepository.js'; +import createAuditRepository from '../repository/auditRepository.js'; +import createCategoryRepository from '../repository/categoryRepository.js'; import createMigrationRepository, { type MigrationRecordWithRelations, } from '../repository/dictionaryMigrationRepository.js'; -import submittedDataRepository from '../repository/submittedRepository.js'; -import type { MigrationStatus, PaginationOptions } from '../utils/types.js'; +import createSubmittedDataRepository from '../repository/submittedRepository.js'; +import { parseAuditRecords } from '../utils/auditUtils.js'; +import { formatMigrationSummary } from '../utils/migrationUtils.js'; +import type { AuditDataResponse, MigrationStatus, PaginationOptions } from '../utils/types.js'; import submissionProcessorFactory from './submission/submissionProcessor.js'; import submissionService from './submission/submissionService.js'; -import { formatMigrationRecord } from '../utils/migrationUtils.js'; const migrationService = (dependencies: BaseDependencies) => { const LOG_MODULE = 'MIGRATION_SERVICE'; const { logger, onFinishCommit } = dependencies; const migrationRepository = createMigrationRepository(dependencies); + const auditRepository = createAuditRepository(dependencies); + const categoryRepository = createCategoryRepository(dependencies); + const submittedDataRepository = createSubmittedDataRepository(dependencies); const submissionProcessor = submissionProcessorFactory.create(dependencies); /** @@ -59,7 +64,7 @@ const migrationService = (dependencies: BaseDependencies) => { ); if (migrations.result.length > 0) { logger.info(LOG_MODULE, `Active migration found for categoryId '${categoryId}'`); - return formatMigrationRecord(migrations.result[0]); + return formatMigrationSummary(migrations.result[0]); } else { logger.info(LOG_MODULE, `No active migration for categoryId '${categoryId}'`); return null; @@ -75,7 +80,15 @@ const migrationService = (dependencies: BaseDependencies) => { const migration = await migrationRepository.getMigrationById(migrationId); if (migration) { logger.info(LOG_MODULE, `Migration found for migrationId '${migrationId}'`); - return formatMigrationRecord(migration); + + const invalidRecords = await auditRepository.getTotalRecordsByCategoryIdAndOrganization(migration.category.id, { + page: -1, + pageSize: -1, + newIsValid: false, + submissionId: migration.submissionId, + }); + + return formatMigrationSummary({ ...migration, invalidRecords }); } else { logger.info(LOG_MODULE, `No migration found for migrationId '${migrationId}'`); return undefined; @@ -97,7 +110,7 @@ const migrationService = (dependencies: BaseDependencies) => { metadata: { totalRecords: migrations.metadata.totalRecords, }, - result: migrations.result.map(formatMigrationRecord), + result: migrations.result.map(formatMigrationSummary), }; } catch (error) { logger.error(LOG_MODULE, `Error retrieving migrations for categoryId '${categoryId}'`, error); @@ -105,6 +118,36 @@ const migrationService = (dependencies: BaseDependencies) => { } }; + const getMigrationRecords = async ( + migrationId: number, + options: { + page: number; + pageSize: number; + entityNames?: string | string[]; + organizations?: string | string[]; + isInvalid?: boolean; + }, + ): Promise<{ result: AuditDataResponse[]; metadata: { totalRecords: number; errorMessage?: string } }> => { + try { + const migrationRecords = await migrationRepository.getMigrationRecords(migrationId, options); + logger.info( + LOG_MODULE, + `Migration records retrieved for migrationId '${migrationId}' with options '${JSON.stringify(options)}'`, + ); + return { + result: parseAuditRecords(migrationRecords.result), + metadata: migrationRecords.metadata, + }; + } catch (error) { + logger.error( + LOG_MODULE, + `Error retrieving migration records for migrationId '${migrationId}' with options '${JSON.stringify(options)}'`, + error, + ); + throw error; + } + }; + /** * Creates a Migration record or update retries if one exists. * Then, it starts running migration in a worker thread @@ -202,9 +245,8 @@ const migrationService = (dependencies: BaseDependencies) => { migrationId: number; userName: string; }): Promise => { - const { getAllOrganizationsByCategoryId, getSubmittedDataByCategoryIdAndOrganization } = - submittedDataRepository(dependencies); - const { getActiveDictionaryByCategory } = categoryRepository(dependencies); + const { getAllOrganizationsByCategoryId, getSubmittedDataByCategoryIdAndOrganization } = submittedDataRepository; + const { getActiveDictionaryByCategory } = categoryRepository; const { performCommitSubmissionAsync } = submissionProcessor; const migration = await migrationRepository.getMigrationById(migrationId); @@ -249,6 +291,7 @@ const migrationService = (dependencies: BaseDependencies) => { getActiveMigrationByCategoryId, getMigrationsByCategoryId, getMigrationById, + getMigrationRecords, initiateMigration, performMigrationValidation, }; diff --git a/packages/data-provider/src/utils/migrationUtils.ts b/packages/data-provider/src/utils/migrationUtils.ts index 18d8d79f..d2a06453 100644 --- a/packages/data-provider/src/utils/migrationUtils.ts +++ b/packages/data-provider/src/utils/migrationUtils.ts @@ -1,16 +1,21 @@ import type { MigrationRecordWithRelations } from '../repository/dictionaryMigrationRepository.js'; +export type MigrationSummary = MigrationRecordWithRelations & { + invalidRecords?: number; +}; + /** - * Order the properties of the Migration Record + * Order the properties of the Migration Summary Record * @param migration * @returns */ -export const formatMigrationRecord = (migration: MigrationRecordWithRelations): MigrationRecordWithRelations => ({ +export const formatMigrationSummary = (migration: MigrationSummary): MigrationSummary => ({ id: migration.id, category: migration.category, fromDictionary: migration.fromDictionary, toDictionary: migration.toDictionary, submissionId: migration.submissionId, + invalidRecords: migration.invalidRecords, status: migration.status, retries: migration.retries, createdAt: migration.createdAt, diff --git a/packages/data-provider/src/utils/schemas.ts b/packages/data-provider/src/utils/schemas.ts index cdaf5e33..742887e6 100644 --- a/packages/data-provider/src/utils/schemas.ts +++ b/packages/data-provider/src/utils/schemas.ts @@ -258,6 +258,23 @@ export const migrationsByCategoryIdRequestSchema: RequestValidation = { + pathParams: migrationIdPathParamSchema, + query: z + .object({ + entityNames: z.union([entityNameSchema, entityNameSchema.array()]).optional(), + organizations: z.union([organizationSchema, organizationSchema.array()]).optional(), + isInvalid: booleanSchema.default('false'), + }) + .merge(paginationQuerySchema), +}; + // Submission Requests export interface SubmissionsByCategoryQueryParams extends PaginationQueryParams { diff --git a/packages/data-provider/src/utils/types.ts b/packages/data-provider/src/utils/types.ts index 0f8d4621..4d12c116 100644 --- a/packages/data-provider/src/utils/types.ts +++ b/packages/data-provider/src/utils/types.ts @@ -90,6 +90,9 @@ export type AuditFilterOptions = PaginationOptions & { startDate?: string; endDate?: string; systemId?: string; + organization?: string; + submissionId?: number; + newIsValid?: boolean; }; /** From b8cd96152ec1f6c8680c6e780c3f1933a1290d86 Mon Sep 17 00:00:00 2001 From: Leonardo Rivera Date: Fri, 17 Apr 2026 14:34:34 -0400 Subject: [PATCH 18/33] adding ts config noUncheckedIndexedAccess --- .../repository/activeSubmissionRepository.ts | 10 +++++-- .../src/repository/auditRepository.ts | 2 +- .../src/repository/categoryRepository.ts | 6 ++++ .../dictionaryMigrationRepository.ts | 14 ++++++--- .../src/repository/dictionaryRepository.ts | 7 +++-- .../src/repository/submittedRepository.ts | 29 ++++++++++++++----- .../submission/submissionProcessor.ts | 8 ++--- .../submittedData/searchDataRelations.ts | 5 ++-- .../services/submittedData/submmittedData.ts | 4 ++- .../src/utils/convertSqonToQuery.ts | 6 +++- .../src/utils/dictionarySchemaRelations.ts | 3 ++ .../src/utils/submissionUtils.ts | 4 +-- .../src/utils/submittedDataUtils.ts | 24 ++++++++------- tsconfig.json | 2 +- 14 files changed, 86 insertions(+), 38 deletions(-) diff --git a/packages/data-provider/src/repository/activeSubmissionRepository.ts b/packages/data-provider/src/repository/activeSubmissionRepository.ts index 05f3cd41..eb9a2bdb 100644 --- a/packages/data-provider/src/repository/activeSubmissionRepository.ts +++ b/packages/data-provider/src/repository/activeSubmissionRepository.ts @@ -188,6 +188,9 @@ jsonb_build_object( save: async (data: NewSubmission): Promise => { try { const [savedActiveSubmission] = await db.insert(submissions).values(data).returning({ id: submissions.id }); + if (!savedActiveSubmission) { + throw new Error('Failed to insert Active Submission, no row returned'); + } logger.info(LOG_MODULE, `New Active Submission saved successfully`); return savedActiveSubmission.id; } catch (error) { @@ -319,9 +322,12 @@ jsonb_build_object( .set({ ...newData, updatedAt: new Date() }) .where(eq(submissions.id, submissionId)) .returning({ id: submissions.id }); + if (!resultUpdate) { + throw new Error(`Failed to update Active Submission with id '${submissionId}', no row returned`); + } return resultUpdate.id; } catch (error) { - logger.error(LOG_MODULE, `Failed updating Active Submission`, error); + logger.error(LOG_MODULE, `Failed updating Active Submission with id '${submissionId}'`, error); throw new ServiceUnavailable(); } }, @@ -398,7 +404,7 @@ jsonb_build_object( filterOptions.organization ? eq(submissions.organization, filterOptions.organization) : undefined, ), ); - return resultCount[0].total; + return resultCount[0]?.total ?? 0; } catch (error) { logger.error(LOG_MODULE, `Failed counting Submission with categoryId '${categoryId}'`, error); throw new ServiceUnavailable(); diff --git a/packages/data-provider/src/repository/auditRepository.ts b/packages/data-provider/src/repository/auditRepository.ts index d8e690e9..1c0b5725 100644 --- a/packages/data-provider/src/repository/auditRepository.ts +++ b/packages/data-provider/src/repository/auditRepository.ts @@ -154,7 +154,7 @@ const repository = (dependencies: BaseDependencies) => { ...optionalFilter, ), ); - return resultCount[0].total; + return resultCount[0]?.total ?? 0; } catch (error) { logger.error( LOG_MODULE, diff --git a/packages/data-provider/src/repository/categoryRepository.ts b/packages/data-provider/src/repository/categoryRepository.ts index 85b9d492..0630f5a9 100644 --- a/packages/data-provider/src/repository/categoryRepository.ts +++ b/packages/data-provider/src/repository/categoryRepository.ts @@ -19,6 +19,9 @@ const repository = (dependencies: BaseDependencies) => { save: async (data: NewCategory): Promise => { try { const savedCategory = await db.insert(dictionaryCategories).values(data).returning(); + if (!savedCategory[0]) { + throw new Error(`Failed to insert Category '${data.name}', no row returned`); + } logger.info(LOG_MODULE, `Category '${data.name}' saved successfully`); return savedCategory[0]; } catch (error) { @@ -150,6 +153,9 @@ const repository = (dependencies: BaseDependencies) => { .set({ ...newData, updatedAt: new Date() }) .where(eq(dictionaryCategories.id, categoryId)) .returning(); + if (!updated[0]) { + throw new Error(`Failed to update Category with id '${categoryId}', no row returned`); + } return updated[0]; } catch (error) { logger.error(LOG_MODULE, `Failed updating Category`, error); diff --git a/packages/data-provider/src/repository/dictionaryMigrationRepository.ts b/packages/data-provider/src/repository/dictionaryMigrationRepository.ts index 32d933db..5f549d33 100644 --- a/packages/data-provider/src/repository/dictionaryMigrationRepository.ts +++ b/packages/data-provider/src/repository/dictionaryMigrationRepository.ts @@ -21,12 +21,15 @@ const repository = (dependencies: BaseDependencies) => { **/ save: async (data: NewDictionaryMigration): Promise => { try { - const savedMigration = await db + const [savedMigration] = await db .insert(dictionaryMigration) .values(data) .returning({ id: dictionaryMigration.id }); + if (!savedMigration) { + throw new Error(`Failed to insert Dictionary Migration for categoryId '${data.categoryId}', no row returned`); + } logger.debug(LOG_MODULE, `Dictionary Migration for categoryId '${data.categoryId}' saved successfully`); - return savedMigration[0].id; + return savedMigration.id; } catch (error) { logger.error(LOG_MODULE, `Failed saving dictionary migration for categoryId '${data.categoryId}'`, error); throw new ServiceUnavailable(); @@ -35,13 +38,16 @@ const repository = (dependencies: BaseDependencies) => { /** Update an existing Dictionary Migration in Database */ update: async (migrationId: number, data: Partial): Promise => { try { - const updatedMigration = await db + const [updatedMigration] = await db .update(dictionaryMigration) .set(data) .where(eq(dictionaryMigration.id, migrationId)) .returning({ id: dictionaryMigration.id }); + if (!updatedMigration) { + throw new Error(`Failed to update Dictionary Migration with id '${migrationId}', no row returned`); + } logger.debug(LOG_MODULE, `Dictionary Migration with id '${migrationId}' updated successfully`); - return updatedMigration[0].id; + return updatedMigration.id; } catch (error) { logger.error(LOG_MODULE, `Failed updating dictionary migration with id '${migrationId}'`, error); throw new ServiceUnavailable(); diff --git a/packages/data-provider/src/repository/dictionaryRepository.ts b/packages/data-provider/src/repository/dictionaryRepository.ts index 2aa839bc..9cbca86b 100644 --- a/packages/data-provider/src/repository/dictionaryRepository.ts +++ b/packages/data-provider/src/repository/dictionaryRepository.ts @@ -16,9 +16,12 @@ const repository = (dependencies: BaseDependencies) => { */ save: async (data: NewDictionary): Promise => { try { - const savedDictionary = await db.insert(dictionaries).values(data).returning(); + const [savedDictionary] = await db.insert(dictionaries).values(data).returning(); + if (!savedDictionary) { + throw new Error(`Failed to insert Dictionary '${data.name}' version '${data.version}', no row returned`); + } logger.info(LOG_MODULE, `Dictionary with name '${data.name}' and version '${data.version}' saved successfully`); - return savedDictionary[0]; + return savedDictionary; } catch (error) { logger.error( LOG_MODULE, diff --git a/packages/data-provider/src/repository/submittedRepository.ts b/packages/data-provider/src/repository/submittedRepository.ts index 0db5f824..8014251e 100644 --- a/packages/data-provider/src/repository/submittedRepository.ts +++ b/packages/data-provider/src/repository/submittedRepository.ts @@ -200,7 +200,13 @@ const repository = (dependencies: BaseDependencies) => { savedRecords.push(...savedSubmittedData); } logger.debug(LOG_MODULE, `Submitted ${savedRecords.length} record(s) successfully`); - return Array.isArray(data) ? savedRecords : savedRecords[0]; + if (Array.isArray(data)) { + return savedRecords; + } + if (!savedRecords[0]) { + throw new Error('Failed to insert SubmittedData, no row returned'); + } + return savedRecords[0]; } catch (error) { logger.error(LOG_MODULE, `Failed submitting ${rows.length} record(s)`, error); throw error; @@ -351,7 +357,7 @@ const repository = (dependencies: BaseDependencies) => { filterEntityNameSql, ), ); - return resultCount[0].total; + return resultCount[0]?.total ?? 0; } catch (error) { logger.error( LOG_MODULE, @@ -382,7 +388,7 @@ const repository = (dependencies: BaseDependencies) => { .select({ total: count() }) .from(submittedData) .where(and(eq(submittedData.dictionaryCategoryId, categoryId), filterEntityNameSql, filterOrganizationSql)); - return resultCount[0].total; + return resultCount[0]?.total ?? 0; } catch (error) { logger.error(LOG_MODULE, `Failed counting SubmittedData with categoryId '${categoryId}'`, error); throw new ServiceUnavailable(); @@ -422,17 +428,20 @@ const repository = (dependencies: BaseDependencies) => { try { const updatedRecords: SubmittedData[] = []; for (const u of updates) { - const updated = await (tx || db) + const [updatedRecord] = await (tx || db) .update(submittedData) .set({ ...u.newData, updatedAt: new Date() }) .where(eq(submittedData.id, u.submittedDataId)) .returning(); - updatedRecords.push(updated[0]); + if (!updatedRecord) { + throw new Error(`Failed to update SubmittedData with id '${u.submittedDataId}', no row returned`); + } + updatedRecords.push(updatedRecord); if (features?.audit?.enabled && Object.keys(u.dataDiff.new).length && Object.keys(u.dataDiff.old).length) { await auditUpdateSubmittedData( { - recordUpdated: updated[0], + recordUpdated: updatedRecord, submissionId: u.submissionId, dataDiff: u.dataDiff, oldIsValid: u.oldIsValid, @@ -442,7 +451,13 @@ const repository = (dependencies: BaseDependencies) => { } } - return Array.isArray(params) ? updatedRecords : updatedRecords[0]; + if (Array.isArray(params)) { + return updatedRecords; + } + if (!updatedRecords[0]) { + throw new Error('Failed to update SubmittedData, no row returned'); + } + return updatedRecords[0]; } catch (error) { logger.error(LOG_MODULE, `Failed updating SubmittedData`, error); throw new ServiceUnavailable(); diff --git a/packages/data-provider/src/services/submission/submissionProcessor.ts b/packages/data-provider/src/services/submission/submissionProcessor.ts index 9a3167cb..3eaced40 100644 --- a/packages/data-provider/src/services/submission/submissionProcessor.ts +++ b/packages/data-provider/src/services/submission/submissionProcessor.ts @@ -162,15 +162,13 @@ const createSubmissionProcessor = (dependencies: BaseDependencies) => { const result = await Promise.all( submissionUpdateRecords.map(async (submissionUpdateRecord) => { - if (!Object.prototype.hasOwnProperty.call(dictionaryRelations, submissionUpdateEntityName)) { + const entityRelations = dictionaryRelations[submissionUpdateEntityName]; + if (!entityRelations) { return { submissionUpdateData: submissionUpdateRecord, dependents: {} }; } // Finds if updates are impacting dependant records based on it's foreign keys - const filterDependents = filterRelationsForPrimaryIdUpdate( - dictionaryRelations[submissionUpdateEntityName], - submissionUpdateRecord, - ); + const filterDependents = filterRelationsForPrimaryIdUpdate(entityRelations, submissionUpdateRecord); if (filterDependents.length === 0) { return { submissionUpdateData: submissionUpdateRecord, dependents: {} }; diff --git a/packages/data-provider/src/services/submittedData/searchDataRelations.ts b/packages/data-provider/src/services/submittedData/searchDataRelations.ts index 238d2af3..978fbc57 100644 --- a/packages/data-provider/src/services/submittedData/searchDataRelations.ts +++ b/packages/data-provider/src/services/submittedData/searchDataRelations.ts @@ -37,11 +37,12 @@ const searchDataRelations = (dependencies: BaseDependencies) => { const { getSubmittedDataFiltered } = submittedDataRepository; // Check if entity has children relationships - if (Object.prototype.hasOwnProperty.call(dictionaryRelations, entityName)) { + const entityRelations = dictionaryRelations[entityName]; + if (entityRelations) { // Array that represents the children fields to filter const filterData: { entityName: string; dataField: string; dataValue: string | undefined }[] = Object.values( - dictionaryRelations[entityName], + entityRelations, ) .filter((childNode) => childNode.parent?.fieldName) .map((childNode) => ({ diff --git a/packages/data-provider/src/services/submittedData/submmittedData.ts b/packages/data-provider/src/services/submittedData/submmittedData.ts index 6a94f60f..fafcacf2 100644 --- a/packages/data-provider/src/services/submittedData/submmittedData.ts +++ b/packages/data-provider/src/services/submittedData/submmittedData.ts @@ -466,7 +466,9 @@ const submittedData = (dependencies: BaseDependencies) => { defaultCentricEntity: defaultCentricEntity, }); - recordResponse = convertedRecord; + if (convertedRecord) { + recordResponse = convertedRecord; + } } } diff --git a/packages/data-provider/src/utils/convertSqonToQuery.ts b/packages/data-provider/src/utils/convertSqonToQuery.ts index 05d80f3e..3984c174 100644 --- a/packages/data-provider/src/utils/convertSqonToQuery.ts +++ b/packages/data-provider/src/utils/convertSqonToQuery.ts @@ -116,7 +116,11 @@ const orOperator = (operations: Operator[]): SQL => { }; const notOperator = (operations: Operator[]): SQL => { - return not(iterateOperator(operations[0])); + const operation = operations[0]; + if (!operation) { + throw new BadRequest(`Invalid SQON format. Invalid 'not' operator`); + } + return not(iterateOperator(operation)); }; /** diff --git a/packages/data-provider/src/utils/dictionarySchemaRelations.ts b/packages/data-provider/src/utils/dictionarySchemaRelations.ts index 8f046593..8c355de4 100644 --- a/packages/data-provider/src/utils/dictionarySchemaRelations.ts +++ b/packages/data-provider/src/utils/dictionarySchemaRelations.ts @@ -122,6 +122,9 @@ const buildHierarchyTree = (schema: SchemaDefinition, tree: TreeNode[], order: O // Use the first mapping for parent-child field relationship const mapping = foreignKey.mappings[0]; + if (!mapping) { + return; + } // remove duplicates. skip mapping when schema is already linked if (order === ORDER_TYPE.Values.desc) { diff --git a/packages/data-provider/src/utils/submissionUtils.ts b/packages/data-provider/src/utils/submissionUtils.ts index 216fca3d..b960bbbd 100644 --- a/packages/data-provider/src/utils/submissionUtils.ts +++ b/packages/data-provider/src/utils/submissionUtils.ts @@ -364,8 +364,8 @@ export const groupSchemaErrorsByEntity = (input: { } Object.entries(groupedErrorsByIndex).forEach(([indexBasedOnCrossSchemas, schemaValidationErrors]) => { - const mapping = dataValidated[entityName][Number(indexBasedOnCrossSchemas)]; - if (!determineIfIsSubmission(mapping.reference)) { + const mapping = dataValidated[entityName]?.[Number(indexBasedOnCrossSchemas)]; + if (!mapping || !determineIfIsSubmission(mapping.reference)) { return; } diff --git a/packages/data-provider/src/utils/submittedDataUtils.ts b/packages/data-provider/src/utils/submittedDataUtils.ts index c960ee85..df21f003 100644 --- a/packages/data-provider/src/utils/submittedDataUtils.ts +++ b/packages/data-provider/src/utils/submittedDataUtils.ts @@ -167,7 +167,7 @@ export const groupErrorsByIndex = ( if (!acc[item.recordIndex]) { acc[item.recordIndex] = []; } - acc[item.recordIndex] = acc[item.recordIndex].concat(item.recordErrors); + acc[item.recordIndex] = (acc[item.recordIndex] ?? []).concat(item.recordErrors); return acc; }, {}); @@ -200,7 +200,7 @@ export const groupSchemaDataByEntityName = (data: { let schemaArr = result.schemaDataByEntityName[entityName]; let submittedArr = result.submittedDataByEntityName[entityName]; - if (!schemaArr) { + if (!schemaArr || !submittedArr) { schemaArr = []; submittedArr = []; result.schemaDataByEntityName[entityName] = schemaArr; @@ -216,7 +216,7 @@ export const groupSchemaDataByEntityName = (data: { let schemaArr = result.schemaDataByEntityName[entityName]; let submittedArr = result.submittedDataByEntityName[entityName]; - if (!schemaArr) { + if (!schemaArr || !submittedArr) { schemaArr = []; submittedArr = []; result.schemaDataByEntityName[entityName] = schemaArr; @@ -264,13 +264,17 @@ export const mapAndMergeSubmittedDataToRecordReferences = ({ return {}; } return submittedData.reduce>((acc, entityData) => { - const foundRecordToUpdateIndex = - editSubmittedData && editSubmittedData[entityData.entityName] - ? editSubmittedData[entityData.entityName].findIndex((item) => item.systemId === entityData.systemId) - : -1; + const entityEditData = editSubmittedData?.[entityData.entityName]; + const foundRecordToUpdateIndex = entityEditData + ? entityEditData.findIndex((item) => item.systemId === entityData.systemId) + : -1; let record: DataRecordReference; - if (editSubmittedData && foundRecordToUpdateIndex >= 0) { - const recordToUpdate = editSubmittedData[entityData.entityName][foundRecordToUpdateIndex]; + if (entityEditData && foundRecordToUpdateIndex >= 0) { + const recordToUpdate = entityEditData[foundRecordToUpdateIndex]; + + if (!recordToUpdate) { + return acc; + } const newDataToUpdate = updateEntityData(entityData.data, recordToUpdate); @@ -298,7 +302,7 @@ export const mapAndMergeSubmittedDataToRecordReferences = ({ acc[entityData.entityName] = []; } - acc[entityData.entityName].push(record); + acc[entityData.entityName]?.push(record); return acc; }, {}); }; diff --git a/tsconfig.json b/tsconfig.json index c8dfe6ab..ab26afef 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -96,7 +96,7 @@ // "exactOptionalPropertyTypes": true, /* Interpret optional property types as written, rather than adding 'undefined'. */ // "noImplicitReturns": true, /* Enable error reporting for codepaths that do not explicitly return in a function. */ // "noFallthroughCasesInSwitch": true, /* Enable error reporting for fallthrough cases in switch statements. */ - // "noUncheckedIndexedAccess": true, /* Add 'undefined' to a type when accessed using an index. */ + "noUncheckedIndexedAccess": true /* Add 'undefined' to a type when accessed using an index. */, // "noImplicitOverride": true, /* Ensure overriding members in derived classes are marked with an override modifier. */ // "noPropertyAccessFromIndexSignature": true, /* Enforces using indexed accessors for keys declared using an indexed type. */ // "allowUnusedLabels": true, /* Disable error reporting for unused labels. */ From cdf32cceb1a9ae1c186550f2cadde00c9c4f0676 Mon Sep 17 00:00:00 2001 From: Leonardo Rivera Date: Fri, 17 Apr 2026 14:35:13 -0400 Subject: [PATCH 19/33] rename migration enum status IN_PROGRESS --- packages/data-model/docs/schema.dbml | 2 +- .../migrations/0013_dictionary_migration.sql | 2 +- .../migrations/meta/0013_snapshot.json | 4 +- .../src/models/dictionary_migration.ts | 50 ++++++++++--------- .../docs/dictionary-registration.md | 4 +- packages/data-provider/src/utils/types.ts | 2 +- 6 files changed, 34 insertions(+), 30 deletions(-) diff --git a/packages/data-model/docs/schema.dbml b/packages/data-model/docs/schema.dbml index 866a56e0..cfd1dc6f 100644 --- a/packages/data-model/docs/schema.dbml +++ b/packages/data-model/docs/schema.dbml @@ -5,7 +5,7 @@ enum audit_action { } enum migration_status { - "IN-PROGRESS" + IN_PROGRESS COMPLETED FAILED } diff --git a/packages/data-model/migrations/0013_dictionary_migration.sql b/packages/data-model/migrations/0013_dictionary_migration.sql index 28f8671d..6ede51fe 100644 --- a/packages/data-model/migrations/0013_dictionary_migration.sql +++ b/packages/data-model/migrations/0013_dictionary_migration.sql @@ -1,5 +1,5 @@ DO $$ BEGIN - CREATE TYPE "migration_status" AS ENUM('IN-PROGRESS', 'COMPLETED', 'FAILED'); + CREATE TYPE "migration_status" AS ENUM('IN_PROGRESS', 'COMPLETED', 'FAILED'); EXCEPTION WHEN duplicate_object THEN null; END $$; diff --git a/packages/data-model/migrations/meta/0013_snapshot.json b/packages/data-model/migrations/meta/0013_snapshot.json index add9e4e8..ad1f40d8 100644 --- a/packages/data-model/migrations/meta/0013_snapshot.json +++ b/packages/data-model/migrations/meta/0013_snapshot.json @@ -759,7 +759,7 @@ "migration_status": { "name": "migration_status", "values": { - "IN-PROGRESS": "IN-PROGRESS", + "IN_PROGRESS": "IN_PROGRESS", "COMPLETED": "COMPLETED", "FAILED": "FAILED" } @@ -783,4 +783,4 @@ "schemas": {}, "tables": {} } -} \ No newline at end of file +} diff --git a/packages/data-model/src/models/dictionary_migration.ts b/packages/data-model/src/models/dictionary_migration.ts index 1c0ab444..3819090f 100644 --- a/packages/data-model/src/models/dictionary_migration.ts +++ b/packages/data-model/src/models/dictionary_migration.ts @@ -5,36 +5,40 @@ import { dictionaries } from './dictionaries.js'; import { dictionaryCategories } from './dictionary_categories.js'; import { submissions } from './submissions.js'; -export const migrationStatusEnum = pgEnum('migration_status', ['IN-PROGRESS', 'COMPLETED', 'FAILED']); +export const migrationStatusEnum = pgEnum('migration_status', ['IN_PROGRESS', 'COMPLETED', 'FAILED']); -export const dictionaryMigration = pgTable('dictionary_migration', { - id: serial('id').primaryKey(), - categoryId: integer('category_id') - .references(() => dictionaryCategories.id) - .notNull(), - fromDictionaryId: integer('from_dictionary_id') - .references(() => dictionaries.id) - .notNull(), - toDictionaryId: integer('to_dictionary_id') - .references(() => dictionaries.id) - .notNull(), - submissionId: integer('submission_id') - .references(() => submissions.id) - .notNull(), - status: migrationStatusEnum('status').notNull(), - retries: integer('retries').notNull().default(0), - createdAt: timestamp('created_at'), - createdBy: varchar('created_by'), - updatedAt: timestamp('updated_at'), - updatedBy: varchar('updated_by'), -},(table) => { +export const dictionaryMigration = pgTable( + 'dictionary_migration', + { + id: serial('id').primaryKey(), + categoryId: integer('category_id') + .references(() => dictionaryCategories.id) + .notNull(), + fromDictionaryId: integer('from_dictionary_id') + .references(() => dictionaries.id) + .notNull(), + toDictionaryId: integer('to_dictionary_id') + .references(() => dictionaries.id) + .notNull(), + submissionId: integer('submission_id') + .references(() => submissions.id) + .notNull(), + status: migrationStatusEnum('status').notNull(), + retries: integer('retries').notNull().default(0), + createdAt: timestamp('created_at'), + createdBy: varchar('created_by'), + updatedAt: timestamp('updated_at'), + updatedBy: varchar('updated_by'), + }, + (table) => { return { categoryIndex: index('dictionary_migration_category_id_index').on(table.categoryId), fromDictionaryIndex: index('dictionary_migration_from_dictionary_id_index').on(table.fromDictionaryId), toDictionaryIndex: index('dictionary_migration_to_dictionary_id_index').on(table.toDictionaryId), submissionIndex: index('dictionary_migration_submission_id_index').on(table.submissionId), }; - },); + }, +); export const dictionaryMigrationRelations = relations(dictionaryMigration, ({ one }) => ({ category: one(dictionaryCategories, { diff --git a/packages/data-provider/docs/dictionary-registration.md b/packages/data-provider/docs/dictionary-registration.md index 57821605..4e8195e0 100644 --- a/packages/data-provider/docs/dictionary-registration.md +++ b/packages/data-provider/docs/dictionary-registration.md @@ -76,7 +76,7 @@ sequenceDiagram note over LyricAPI: [Migration] Initiate migration LyricAPI->>+LyricDB: Create a new submission (no data, status=COMMITTED) LyricDB-->>-LyricAPI: Return submission - LyricAPI->>+LyricDB: Create a migration in `category_migration` table
(categoryId, fromDictionaryId, toDictionaryId, submissionId, status=IN-PROGRESS, createdAt, createdBy) + LyricAPI->>+LyricDB: Create a migration in `category_migration` table
(categoryId, fromDictionaryId, toDictionaryId, submissionId, status=IN_PROGRESS, createdAt, createdBy) LyricDB-->>-LyricAPI: Return migrationId LyricAPI->>User: 200 OK: Return Dictionary, Category and migrationId @@ -134,7 +134,7 @@ When a dictionary definition is updated in Lectern, Lyric allows an existing cat - The endpoint returns a `migrationId`, representing a new migration job. -- A migration entry is created with the initial status `IN-PROGRESS`. +- A migration entry is created with the initial status `IN_PROGRESS`. - Since migrations run asynchronously, the status is updated to `COMPLETED` once processing finishes. diff --git a/packages/data-provider/src/utils/types.ts b/packages/data-provider/src/utils/types.ts index cb692e7c..57cfc9b0 100644 --- a/packages/data-provider/src/utils/types.ts +++ b/packages/data-provider/src/utils/types.ts @@ -34,7 +34,7 @@ export const SUBMISSION_STATUS = { } as const; export type SubmissionStatus = ObjectValues; -export const MIGRATION_STATUS = z.enum(['IN-PROGRESS', 'COMPLETED', 'FAILED']); +export const MIGRATION_STATUS = z.enum(['IN_PROGRESS', 'COMPLETED', 'FAILED']); export type MigrationStatus = z.infer; /** From f5994eb9920d5fb222525c38d72752e196968e8d Mon Sep 17 00:00:00 2001 From: Leonardo Rivera Date: Tue, 28 Apr 2026 12:38:21 -0400 Subject: [PATCH 20/33] refactoring migration function with Result --- .../src/services/dictionaryService.ts | 12 +- .../src/services/migrationService.ts | 118 ++++++++++-------- .../submission/submissionProcessor.ts | 2 +- .../src/workers/dictionaryMigrationWorker.ts | 9 +- 4 files changed, 83 insertions(+), 58 deletions(-) diff --git a/packages/data-provider/src/services/dictionaryService.ts b/packages/data-provider/src/services/dictionaryService.ts index a208f707..7d8e65f2 100644 --- a/packages/data-provider/src/services/dictionaryService.ts +++ b/packages/data-provider/src/services/dictionaryService.ts @@ -1,4 +1,4 @@ -import { isEmpty } from 'lodash-es'; +import { isEmpty, result } from 'lodash-es'; import { Dictionary as SchemasDictionary, Schema } from '@overture-stack/lectern-client'; import { Category, Dictionary, NewCategory, NewDictionary } from '@overture-stack/lyric-data-model/models'; @@ -144,19 +144,25 @@ const dictionaryService = (dependencies: BaseDependencies) => { updatedBy: username, }); - const migrationId = await initiateMigration({ + const resultMigration = await initiateMigration({ categoryId: updatedCategory.id, fromDictionaryId: foundCategory.activeDictionaryId, toDictionaryId: savedDictionary.id, userName: username || '', }); + if (!resultMigration.success) { + const errorMessage = `Failed to initiate migration for category '${categoryName}' with error: ${resultMigration.data}`; + logger.error(LOG_MODULE, errorMessage); + throw new Error(errorMessage); + } + logger.info( LOG_MODULE, `Category '${updatedCategory.name}' updated successfully with Dictionary '${savedDictionary.name}' version '${savedDictionary.version}'`, ); - return { dictionary: savedDictionary, category: updatedCategory, migrationId }; + return { dictionary: savedDictionary, category: updatedCategory, migrationId: resultMigration.data }; } else { // Create a new Category const newCategory: NewCategory = { diff --git a/packages/data-provider/src/services/migrationService.ts b/packages/data-provider/src/services/migrationService.ts index 0777febb..9a02c81b 100644 --- a/packages/data-provider/src/services/migrationService.ts +++ b/packages/data-provider/src/services/migrationService.ts @@ -4,6 +4,7 @@ import type { BaseDependencies } from '../config/config.js'; import categoryRepository from '../repository/categoryRepository.js'; import createMigrationRepository from '../repository/dictionaryMigrationRepository.js'; import submittedDataRepository from '../repository/submittedRepository.js'; +import { failure, type Result, success } from '../utils/result.js'; import type { MigrationStatus } from '../utils/types.js'; import submissionProcessorFactory from './submission/submissionProcessor.js'; import submissionService from './submission/submissionService.js'; @@ -16,8 +17,10 @@ const migrationService = (dependencies: BaseDependencies) => { /** * Update the status of the migration to COMPLETED or FAILED - * @param param0 - * @returns The ID of the finalized migration + * @param migrationId The ID of the migration to finalize + * @param status The final status of the migration, either 'COMPLETED' or 'FAILED' + * @param userName The name of the user that is finalizing the migration (for audit purposes) + * @returns A Result object with null data and a string error message in case of failure */ const finalizeMigration = async ({ migrationId, @@ -27,18 +30,24 @@ const migrationService = (dependencies: BaseDependencies) => { migrationId: number; status: Extract; userName: string; - }): Promise => { + }): Promise> => { try { - const updatedMigration = await migrationRepository.update(migrationId, { + const resultUpdate = await migrationRepository.update(migrationId, { status, updatedAt: new Date(), updatedBy: userName, }); + + if (resultUpdate === 0) { + logger.info(LOG_MODULE, `Migration with id '${migrationId}' not found for finalization`); + return failure(`Migration with id '${migrationId}' not found.`); + } logger.info(LOG_MODULE, `Migration finalized for migrationId '${migrationId}' with status '${status}'`); - return updatedMigration; + return success(null); } catch (error) { - logger.error(LOG_MODULE, `Error finalizing migration for migrationId '${migrationId}'`, error); - throw error; + const errorMessage = error instanceof Error ? error.message : String(error); + logger.error(LOG_MODULE, `Error finalizing migration for migrationId '${migrationId}'`, errorMessage); + return failure(`Error finalizing migration for migrationId '${migrationId}'`); } }; @@ -71,7 +80,8 @@ const migrationService = (dependencies: BaseDependencies) => { * Creates a Migration record or update retries if one exists. * Then, it starts running migration in a worker thread * @param param0 - * @returns The ID of the initiated or updated migration + * @returns The result of the migration initiation process, with the migrationId in case of success or + * an error message in case of failure */ const initiateMigration = async ({ categoryId, @@ -83,7 +93,7 @@ const migrationService = (dependencies: BaseDependencies) => { fromDictionaryId: number; toDictionaryId: number; userName: string; - }): Promise => { + }): Promise> => { const { getOrCreateActiveSubmission } = submissionService(dependencies); try { const findMigrationResult = await migrationRepository.getMigrationsByCategoryId( @@ -142,10 +152,11 @@ const migrationService = (dependencies: BaseDependencies) => { }); logger.info(LOG_MODULE, `Migration initiated for categoryId '${categoryId}'`); - return migrationId; + return success(migrationId); } catch (error) { - logger.error(LOG_MODULE, `Error initiating migration for categoryId '${categoryId}'`, error); - throw error; + const errorMessage = error instanceof Error ? error.message : String(error); + logger.error(LOG_MODULE, `Error initiating migration for categoryId '${categoryId}'`, errorMessage); + return failure(`Error initiating migration for categoryId '${categoryId}'`); } }; @@ -155,7 +166,7 @@ const migrationService = (dependencies: BaseDependencies) => { * it iterates over all organizations and validates the submitted data for each of them. * @param migrationId The ID of the migration to perform * @param userName The name of the user that initiated the migration (for audit purposes) - * @returns void + * @returns A Result object with null data and a string error message in case of failure */ const performMigrationValidation = async ({ migrationId, @@ -163,47 +174,54 @@ const migrationService = (dependencies: BaseDependencies) => { }: { migrationId: number; userName: string; - }): Promise => { - const { getAllOrganizationsByCategoryId, getSubmittedDataByCategoryIdAndOrganization } = - submittedDataRepository(dependencies); - const { getActiveDictionaryByCategory } = categoryRepository(dependencies); - const { performCommitSubmissionAsync } = submissionProcessor; - - const migration = await migrationRepository.getMigrationById(migrationId); - if (!migration) { - throw new Error(`Migration with id '${migrationId}' not found`); - } + }): Promise> => { + try { + const { getAllOrganizationsByCategoryId, getSubmittedDataByCategoryIdAndOrganization } = + submittedDataRepository(dependencies); + const { getActiveDictionaryByCategory } = categoryRepository(dependencies); + const { performCommitSubmissionAsync } = submissionProcessor; + + const migration = await migrationRepository.getMigrationById(migrationId); + if (!migration) { + return failure(`Migration with id '${migrationId}' not found`); + } - const { categoryId, submissionId } = migration; + const { categoryId, submissionId } = migration; - const dictionary = await getActiveDictionaryByCategory(categoryId); - if (!dictionary) { - throw new Error(`Dictionary in category '${categoryId}' not found`); - } + const dictionary = await getActiveDictionaryByCategory(categoryId); + if (!dictionary) { + return failure(`Dictionary in category '${categoryId}' not found`); + } - const organizations = await getAllOrganizationsByCategoryId(categoryId); - logger.info(LOG_MODULE, `Starting migration validation for following organizations '${organizations}'`); - for (const organization of organizations) { - const submittedDataToValidate = await getSubmittedDataByCategoryIdAndOrganization(categoryId, organization); - logger.info( - LOG_MODULE, - `Performing migration validation for organization '${organization}' with ${submittedDataToValidate.length} submitted records`, - ); - await performCommitSubmissionAsync({ - dataToValidate: { - inserts: [], - submittedData: submittedDataToValidate, - deletes: [], - updates: {}, - }, - submissionId, - dictionary, - username: userName, - isMigration: true, - onFinishCommit, - }); + const organizations = await getAllOrganizationsByCategoryId(categoryId); + logger.info(LOG_MODULE, `Starting migration validation for following organizations '${organizations}'`); + for (const organization of organizations) { + const submittedDataToValidate = await getSubmittedDataByCategoryIdAndOrganization(categoryId, organization); + logger.info( + LOG_MODULE, + `Performing migration validation for organization '${organization}' with ${submittedDataToValidate.length} submitted records`, + ); + await performCommitSubmissionAsync({ + dataToValidate: { + inserts: [], + submittedData: submittedDataToValidate, + deletes: [], + updates: {}, + }, + submissionId, + dictionary, + username: userName, + isMigration: true, + onFinishCommit, + }); + } + logger.info(LOG_MODULE, `Migration validation completed for submissionId '${submissionId}'`); + return success(null); + } catch (error) { + const errorMessage = error instanceof Error ? error.message : String(error); + logger.error(LOG_MODULE, `Error performing migration validation for migrationId '${migrationId}'`, errorMessage); + return failure(`Error performing migration validation for migrationId '${migrationId}'`); } - logger.info(LOG_MODULE, `Migration validation completed for submissionId '${submissionId}'`); }; return { diff --git a/packages/data-provider/src/services/submission/submissionProcessor.ts b/packages/data-provider/src/services/submission/submissionProcessor.ts index 620c5c10..7f4a3a49 100644 --- a/packages/data-provider/src/services/submission/submissionProcessor.ts +++ b/packages/data-provider/src/services/submission/submissionProcessor.ts @@ -361,7 +361,7 @@ const createSubmissionProcessor = (dependencies: BaseDependencies) => { const submisionUpdateData = dataToValidate.updates?.[record.systemId]; if (submisionUpdateData) { - logger.info( + logger.debug( LOG_MODULE, `Updating submittedData system ID '${record.systemId}' in entity '${entityName}'`, ); diff --git a/packages/data-provider/src/workers/dictionaryMigrationWorker.ts b/packages/data-provider/src/workers/dictionaryMigrationWorker.ts index 7e867b8a..630c394b 100644 --- a/packages/data-provider/src/workers/dictionaryMigrationWorker.ts +++ b/packages/data-provider/src/workers/dictionaryMigrationWorker.ts @@ -6,15 +6,16 @@ export const performDictionaryMigration = async ({ migrationId, userName }: Dict const dependencies = getWorkerDependencies(); const migrationService = createMigrationService(dependencies); - try { - await migrationService.performMigrationValidation({ migrationId, userName }); + const resultMigration = await migrationService.performMigrationValidation({ migrationId, userName }); + + if (resultMigration.success) { migrationService.finalizeMigration({ migrationId, status: 'COMPLETED', userName, }); - } catch (error) { - console.error('Error performing migration validation:', error); + } else { + console.error(`Migration validation failed for migrationId '${migrationId}' with error: ${resultMigration.data}`); migrationService.finalizeMigration({ migrationId, status: 'FAILED', From c682a4793be197171a6a319ef46d438e36b2245b Mon Sep 17 00:00:00 2001 From: Leonardo Rivera Date: Wed, 29 Apr 2026 10:54:36 -0400 Subject: [PATCH 21/33] default constants pagination --- packages/data-provider/src/config/pagination.ts | 2 ++ .../src/controllers/auditController.ts | 7 +++---- .../src/controllers/migrationController.ts | 11 +++++------ .../src/controllers/submissionController.ts | 11 +++++------ .../src/controllers/submittedDataController.ts | 15 +++++++-------- 5 files changed, 22 insertions(+), 24 deletions(-) create mode 100644 packages/data-provider/src/config/pagination.ts diff --git a/packages/data-provider/src/config/pagination.ts b/packages/data-provider/src/config/pagination.ts new file mode 100644 index 00000000..dc6d0653 --- /dev/null +++ b/packages/data-provider/src/config/pagination.ts @@ -0,0 +1,2 @@ +export const DEFAULT_PAGE = 1; +export const DEFAULT_PAGE_SIZE = 20; diff --git a/packages/data-provider/src/controllers/auditController.ts b/packages/data-provider/src/controllers/auditController.ts index a72b7623..4d220003 100644 --- a/packages/data-provider/src/controllers/auditController.ts +++ b/packages/data-provider/src/controllers/auditController.ts @@ -1,4 +1,5 @@ import { BaseDependencies } from '../config/config.js'; +import { DEFAULT_PAGE, DEFAULT_PAGE_SIZE } from '../config/pagination.js'; import auditSvc from '../services/auditService.js'; import { NotFound } from '../utils/errors.js'; import { validateRequest } from '../utils/requestValidation.js'; @@ -9,8 +10,6 @@ const controller = (dependencies: BaseDependencies) => { const auditService = auditSvc(dependencies); const { logger } = dependencies; const LOG_MODULE = 'AUDIT_CONTROLLER'; - const defaultPage = 1; - const defaultPageSize = 20; return { byCategoryIdAndOrganization: validateRequest(auditByCatAndOrgRequestSchema, async (req, res, next) => { try { @@ -18,8 +17,8 @@ const controller = (dependencies: BaseDependencies) => { const organization = req.params.organization; // pagination parameters - const page = parseInt(String(req.query.page)) || defaultPage; - const pageSize = parseInt(String(req.query.pageSize)) || defaultPageSize; + const page = parseInt(String(req.query.page)) || DEFAULT_PAGE; + const pageSize = parseInt(String(req.query.pageSize)) || DEFAULT_PAGE_SIZE; // optional query parameters const { entityName, eventType, startDate, endDate, systemId } = req.query; diff --git a/packages/data-provider/src/controllers/migrationController.ts b/packages/data-provider/src/controllers/migrationController.ts index b5f29b92..d7c1dbba 100644 --- a/packages/data-provider/src/controllers/migrationController.ts +++ b/packages/data-provider/src/controllers/migrationController.ts @@ -1,4 +1,5 @@ import { BaseDependencies } from '../config/config.js'; +import { DEFAULT_PAGE, DEFAULT_PAGE_SIZE } from '../config/pagination.js'; import createMigrationService from '../services/migrationService.js'; import { NotFound } from '../utils/errors.js'; import { validateRequest } from '../utils/requestValidation.js'; @@ -12,8 +13,6 @@ const controller = (dependencies: BaseDependencies) => { const migrationService = createMigrationService(dependencies); const { logger } = dependencies; const LOG_MODULE = 'MIGRATION_CONTROLLER'; - const defaultPage = 1; - const defaultPageSize = 20; return { getMigrationById: validateRequest(migrationByIdRequestSchema, async (req, res, next) => { @@ -35,8 +34,8 @@ const controller = (dependencies: BaseDependencies) => { getMigrationsByCategoryId: validateRequest(migrationsByCategoryIdRequestSchema, async (req, res, next) => { try { const categoryId = Number(req.params.categoryId); - const page = parseInt(String(req.query.page)) || defaultPage; - const pageSize = parseInt(String(req.query.pageSize)) || defaultPageSize; + const page = parseInt(String(req.query.page)) || DEFAULT_PAGE; + const pageSize = parseInt(String(req.query.pageSize)) || DEFAULT_PAGE_SIZE; logger.info(LOG_MODULE, `Request Migrations by category Id '${categoryId}'`); @@ -65,8 +64,8 @@ const controller = (dependencies: BaseDependencies) => { try { const migrationId = Number(req.params.migrationId); - const page = parseInt(String(req.query.page)) || defaultPage; - const pageSize = parseInt(String(req.query.pageSize)) || defaultPageSize; + const page = parseInt(String(req.query.page)) || DEFAULT_PAGE; + const pageSize = parseInt(String(req.query.pageSize)) || DEFAULT_PAGE_SIZE; const entityNames = req.query.entityNames; const organizations = req.query.organizations; const isInvalid = req.query.isInvalid === 'true'; diff --git a/packages/data-provider/src/controllers/submissionController.ts b/packages/data-provider/src/controllers/submissionController.ts index 2d063897..8cfb848d 100644 --- a/packages/data-provider/src/controllers/submissionController.ts +++ b/packages/data-provider/src/controllers/submissionController.ts @@ -2,6 +2,7 @@ import type { Response } from 'express'; import { isEmpty } from 'lodash-es'; import { BaseDependencies } from '../config/config.js'; +import { DEFAULT_PAGE, DEFAULT_PAGE_SIZE } from '../config/pagination.js'; import { type AuthConfig, shouldBypassAuth } from '../middleware/auth.js'; import { getSubmittedFileType } from '../services/submission/submissionFile.js'; import createSubmissionService from '../services/submission/submissionService.js'; @@ -42,8 +43,6 @@ const controller = ({ const submissionService = createSubmissionService(baseDependencies); const dataService = createSubmittedDataService(baseDependencies); const { logger } = baseDependencies; - const defaultPage = 1; - const defaultPageSize = 20; const LOG_MODULE = 'SUBMISSION_CONTROLLER'; return { commit: validateRequest(submissionCommitRequestSchema, async (req, res, next) => { @@ -228,8 +227,8 @@ const controller = ({ const categoryId = Number(req.params.categoryId); const onlyActive = req.query.onlyActive?.toLowerCase() === 'true'; const organization = req.query.organization; - const page = parseInt(String(req.query.page)) || defaultPage; - const pageSize = parseInt(String(req.query.pageSize)) || defaultPageSize; + const page = parseInt(String(req.query.page)) || DEFAULT_PAGE; + const pageSize = parseInt(String(req.query.pageSize)) || DEFAULT_PAGE_SIZE; const username = req.query.username; logger.info( @@ -287,8 +286,8 @@ const controller = ({ const actionTypes = parseSubmissionActionTypes(req.query.actionTypes || SUBMISSION_ACTION_TYPE.options); // query params - const page = parseInt(String(req.query.page)) || defaultPage; - const pageSize = parseInt(String(req.query.pageSize)) || defaultPageSize; + const page = parseInt(String(req.query.page)) || DEFAULT_PAGE; + const pageSize = parseInt(String(req.query.pageSize)) || DEFAULT_PAGE_SIZE; logger.info(LOG_MODULE, `Request Submission Details by ID '${submissionId}'`); diff --git a/packages/data-provider/src/controllers/submittedDataController.ts b/packages/data-provider/src/controllers/submittedDataController.ts index 1fd76130..430f11b8 100644 --- a/packages/data-provider/src/controllers/submittedDataController.ts +++ b/packages/data-provider/src/controllers/submittedDataController.ts @@ -1,6 +1,7 @@ import * as _ from 'lodash-es'; import { BaseDependencies } from '../config/config.js'; +import { DEFAULT_PAGE, DEFAULT_PAGE_SIZE } from '../config/pagination.js'; import { type AuthConfig, shouldBypassAuth } from '../middleware/auth.js'; import submittedDataService from '../services/submittedData/submmittedData.js'; import { getUserReadableOrganizations, hasUserReadAccess } from '../utils/authUtils.js'; @@ -27,8 +28,6 @@ const controller = ({ const service = submittedDataService(baseDependencies); const { logger } = baseDependencies; const LOG_MODULE = 'SUBMITTED_DATA_CONTROLLER'; - const defaultPage = 1; - const defaultPageSize = 20; const defaultView = VIEW_TYPE.Values.flat; return { @@ -38,8 +37,8 @@ const controller = ({ // query params const entityName = asArray(req.query.entityName || []); - const page = parseInt(String(req.query.page)) || defaultPage; - const pageSize = parseInt(String(req.query.pageSize)) || defaultPageSize; + const page = parseInt(String(req.query.page)) || DEFAULT_PAGE; + const pageSize = parseInt(String(req.query.pageSize)) || DEFAULT_PAGE_SIZE; const view = convertToViewType(req.query.view) || defaultView; const user = req.user; @@ -85,8 +84,8 @@ const controller = ({ // query parameters const entityName = asArray(req.query.entityName || []); - const page = parseInt(String(req.query.page)) || defaultPage; - const pageSize = parseInt(String(req.query.pageSize)) || defaultPageSize; + const page = parseInt(String(req.query.page)) || DEFAULT_PAGE; + const pageSize = parseInt(String(req.query.pageSize)) || DEFAULT_PAGE_SIZE; const view = convertToViewType(String(req.query.view)) || defaultView; const user = req.user; @@ -139,8 +138,8 @@ const controller = ({ // query parameters const entityName = asArray(req.query.entityName || []); - const page = parseInt(String(req.query.page)) || defaultPage; - const pageSize = parseInt(String(req.query.pageSize)) || defaultPageSize; + const page = parseInt(String(req.query.page)) || DEFAULT_PAGE; + const pageSize = parseInt(String(req.query.pageSize)) || DEFAULT_PAGE_SIZE; const user = req.user; logger.info( From b41d1adcc601dad6c75a3c5c3a3efc5b3d5de391 Mon Sep 17 00:00:00 2001 From: Leonardo Rivera Date: Wed, 29 Apr 2026 16:31:30 -0400 Subject: [PATCH 22/33] change logs and response NotFound --- .../src/controllers/migrationController.ts | 41 ++++++++++++------- 1 file changed, 26 insertions(+), 15 deletions(-) diff --git a/packages/data-provider/src/controllers/migrationController.ts b/packages/data-provider/src/controllers/migrationController.ts index d7c1dbba..e1ba9ac3 100644 --- a/packages/data-provider/src/controllers/migrationController.ts +++ b/packages/data-provider/src/controllers/migrationController.ts @@ -1,5 +1,6 @@ import { BaseDependencies } from '../config/config.js'; import { DEFAULT_PAGE, DEFAULT_PAGE_SIZE } from '../config/pagination.js'; +import createCategoryService from '../services/categoryService.js'; import createMigrationService from '../services/migrationService.js'; import { NotFound } from '../utils/errors.js'; import { validateRequest } from '../utils/requestValidation.js'; @@ -11,6 +12,7 @@ import { const controller = (dependencies: BaseDependencies) => { const migrationService = createMigrationService(dependencies); + const categoryService = createCategoryService(dependencies); const { logger } = dependencies; const LOG_MODULE = 'MIGRATION_CONTROLLER'; @@ -21,12 +23,14 @@ const controller = (dependencies: BaseDependencies) => { logger.info(LOG_MODULE, `Request Migration id '${migrationId}'`); - const submission = await migrationService.getMigrationById(migrationId); + const migrationResult = await migrationService.getMigrationById(migrationId); - if (!submission) { - throw new NotFound('Submission not found'); + if (!migrationResult) { + const message = `Migration with id '${migrationId}' not found`; + logger.info(LOG_MODULE, message); + throw new NotFound(message); } - return res.send(submission); + return res.send(migrationResult); } catch (error) { next(error); } @@ -39,20 +43,23 @@ const controller = (dependencies: BaseDependencies) => { logger.info(LOG_MODULE, `Request Migrations by category Id '${categoryId}'`); - const submissions = await migrationService.getMigrationsByCategoryId(categoryId, { page, pageSize }); - - if (submissions.result.length === 0) { - throw new NotFound('Submissions not found'); + const categoryExists = await categoryService.getDetails(categoryId); + if (!categoryExists) { + const message = `Category with id '${categoryId}' not found`; + logger.info(LOG_MODULE, message); + throw new NotFound(message); } + const migrationsResult = await migrationService.getMigrationsByCategoryId(categoryId, { page, pageSize }); + const response = { pagination: { currentPage: page, pageSize: pageSize, - totalPages: Math.ceil(submissions.metadata.totalRecords / pageSize), - totalRecords: submissions.metadata.totalRecords, + totalPages: Math.ceil(migrationsResult.metadata.totalRecords / pageSize), + totalRecords: migrationsResult.metadata.totalRecords, }, - records: submissions.result, + records: migrationsResult.result, }; return res.send(response); @@ -72,6 +79,14 @@ const controller = (dependencies: BaseDependencies) => { logger.info(LOG_MODULE, `Request Data Migration id '${migrationId}'`); + const migrationResult = await migrationService.getMigrationById(migrationId); + + if (!migrationResult) { + const message = `Migration with id '${migrationId}' not found`; + logger.info(LOG_MODULE, message); + throw new NotFound(message); + } + const submissionRecords = await migrationService.getMigrationRecords(migrationId, { page, pageSize, @@ -80,10 +95,6 @@ const controller = (dependencies: BaseDependencies) => { isInvalid, }); - if (submissionRecords.result.length === 0) { - throw new NotFound('Records not found for the specified migration'); - } - const response = { pagination: { currentPage: page, From 6f231ce3c630c3bd75f0f7e318c17f4099e09d16 Mon Sep 17 00:00:00 2001 From: Leonardo Rivera Date: Wed, 29 Apr 2026 16:31:56 -0400 Subject: [PATCH 23/33] fix swagger docs --- apps/server/swagger/schemas.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/server/swagger/schemas.yml b/apps/server/swagger/schemas.yml index c287fd83..1bc1dd67 100644 --- a/apps/server/swagger/schemas.yml +++ b/apps/server/swagger/schemas.yml @@ -497,14 +497,14 @@ components: enum: ['IN-PROGRESS', 'COMPLETED', 'FAILED'] retries: type: number - description: Number of retries of the migration + description: Number of times the migration has run. When a migration fails, it can be retried by registering again the same dictionary. createdAt: type: string description: Date and time of creation createdBy: type: string description: User name who created the migration - udpatedAt: + updatedAt: type: string description: Date and time of latest update updatedBy: From 4f7a01b5a3e3eb7507aef368e28f1cae6dfea526 Mon Sep 17 00:00:00 2001 From: Leonardo Rivera Date: Wed, 29 Apr 2026 23:36:18 -0400 Subject: [PATCH 24/33] include errors or migration changes --- apps/server/swagger/migration-api.yml | 2 +- apps/server/swagger/schemas.yml | 40 ++++++++++++++++- .../src/controllers/migrationController.ts | 2 +- .../src/repository/auditRepository.ts | 6 +-- .../dictionaryMigrationRepository.ts | 45 ++++++++++--------- .../src/services/dictionaryService.ts | 2 +- .../src/services/migrationService.ts | 9 ++-- .../data-provider/src/utils/auditUtils.ts | 1 + .../data-provider/src/utils/migrationUtils.ts | 13 ++++++ packages/data-provider/src/utils/types.ts | 19 ++++++++ .../test/unit/utils/auditUtils.spec.ts | 2 + 11 files changed, 108 insertions(+), 33 deletions(-) diff --git a/apps/server/swagger/migration-api.yml b/apps/server/swagger/migration-api.yml index 090b68d2..fb281ee7 100644 --- a/apps/server/swagger/migration-api.yml +++ b/apps/server/swagger/migration-api.yml @@ -52,7 +52,7 @@ parameters: - $ref: '#/components/parameters/path/MigrationId' - isInvalid: - description: Optional query parameter to filter results to include only invalid records. Default value is false + description: Optional query parameter to filter results to include only invalid records. Default is to include all records. name: isInvalid in: query schema: diff --git a/apps/server/swagger/schemas.yml b/apps/server/swagger/schemas.yml index 1bc1dd67..9e1c2cbe 100644 --- a/apps/server/swagger/schemas.yml +++ b/apps/server/swagger/schemas.yml @@ -303,6 +303,11 @@ components: type: string description: Type of event enum: ['UPDATE', 'DELETE', 'MIGRATION'] + errors: + type: array + description: List of validation errors related to the change if applicable + items: + $ref: '#/components/schemas/ValidationError' dataDiff: type: object description: Captures the state of the data before the change as `old` and after the change as `new` @@ -446,7 +451,7 @@ components: records: type: array items: - $ref: '#/components/schemas/ChangeHistoryRecord' + $ref: '#/components/schemas/MigrationChangeHistoryRecord' ListMigrationsResult: type: object @@ -458,6 +463,39 @@ components: items: $ref: '#/components/schemas/MigrationResult' + MigrationChangeHistoryRecord: + type: object + properties: + entityName: + type: string + description: Name of the Entity + errors: + type: array + description: List of validation errors related to the change if applicable + items: + $ref: '#/components/schemas/ValidationError' + dataDiff: + type: object + description: Captures the state of the data before the change as `old` and after the change as `new` + newIsValid: + type: boolean + description: New data isValid value + oldIsValid: + type: boolean + description: Old data isValid value + organization: + type: string + description: Name of the Organization + systemId: + type: string + description: The unique identifier of the record changed + createdAt: + type: string + description: Date and time of creation + createdBy: + type: string + description: User name who updated the submission + MigrationResult: type: object properties: diff --git a/packages/data-provider/src/controllers/migrationController.ts b/packages/data-provider/src/controllers/migrationController.ts index e1ba9ac3..6ba29f3e 100644 --- a/packages/data-provider/src/controllers/migrationController.ts +++ b/packages/data-provider/src/controllers/migrationController.ts @@ -75,7 +75,7 @@ const controller = (dependencies: BaseDependencies) => { const pageSize = parseInt(String(req.query.pageSize)) || DEFAULT_PAGE_SIZE; const entityNames = req.query.entityNames; const organizations = req.query.organizations; - const isInvalid = req.query.isInvalid === 'true'; + const isInvalid = req.query.isInvalid === undefined ? undefined : req.query.isInvalid === 'true'; logger.info(LOG_MODULE, `Request Data Migration id '${migrationId}'`); diff --git a/packages/data-provider/src/repository/auditRepository.ts b/packages/data-provider/src/repository/auditRepository.ts index a9422532..abcff80c 100644 --- a/packages/data-provider/src/repository/auditRepository.ts +++ b/packages/data-provider/src/repository/auditRepository.ts @@ -6,13 +6,13 @@ import { BaseDependencies } from '../config/config.js'; import { convertToAuditEvent } from '../utils/auditUtils.js'; import { ServiceUnavailable } from '../utils/errors.js'; import { isEmptyString, isValidDateFormat } from '../utils/formatUtils.js'; -import { AuditFilterOptions, AuditRepositoryRecord, BooleanTrueObject } from '../utils/types.js'; +import { AuditFilterOptions, AuditRepositoryRecord } from '../utils/types.js'; const repository = (dependencies: BaseDependencies) => { const LOG_MODULE = 'AUDIT_REPOSITORY'; const { db, logger } = dependencies; - const paginatedColumns: BooleanTrueObject = { + const paginatedColumns = { entityName: true, action: true, dataDiff: true, @@ -24,7 +24,7 @@ const repository = (dependencies: BaseDependencies) => { systemId: true, createdAt: true, createdBy: true, - }; + } as const satisfies Record; const getOptionalFilter = ({ entityName, diff --git a/packages/data-provider/src/repository/dictionaryMigrationRepository.ts b/packages/data-provider/src/repository/dictionaryMigrationRepository.ts index 0b9a9ec7..88e27072 100644 --- a/packages/data-provider/src/repository/dictionaryMigrationRepository.ts +++ b/packages/data-provider/src/repository/dictionaryMigrationRepository.ts @@ -1,6 +1,7 @@ -import { and, count, eq, type SQL } from 'drizzle-orm'; +import { and, count, eq } from 'drizzle-orm'; import { + type Category, type Dictionary, type DictionaryMigration, dictionaryMigration, @@ -9,7 +10,14 @@ import { import type { BaseDependencies } from '../config/config.js'; import { ServiceUnavailable } from '../utils/errors.js'; -import type { AuditRepositoryRecord, BooleanTrueObject, MigrationStatus, PaginationOptions } from '../utils/types.js'; +import { formatMigrationAuditRecord } from '../utils/migrationUtils.js'; +import type { + MigrationAuditRecord, + MigrationStatus, + PaginationOptions, + PartialColumns, + WithColumns, +} from '../utils/types.js'; import createAuditRepository from './auditRepository.js'; type MigrationRepositoryRecord = Omit; @@ -19,18 +27,9 @@ type MigrationRepositoryRecord = Omit; + fromDictionary: Pick | null; + toDictionary: Pick | null; }; const repository = (dependencies: BaseDependencies) => { @@ -48,14 +47,14 @@ const repository = (dependencies: BaseDependencies) => { createdBy: true, updatedAt: true, updatedBy: true, - } as const satisfies Record; + } as const satisfies PartialColumns; const dictionarySummaryColumns = { columns: { name: true, version: true, }, - } as const satisfies { columns: Partial> }; + } as const satisfies WithColumns; const migrationWithRelationsColumns = { category: { @@ -66,7 +65,11 @@ const repository = (dependencies: BaseDependencies) => { }, fromDictionary: dictionarySummaryColumns, toDictionary: dictionarySummaryColumns, - } as const satisfies Record; + } as const satisfies { + category: WithColumns; + fromDictionary: WithColumns; + toDictionary: WithColumns; + }; return { /** @@ -139,7 +142,7 @@ const repository = (dependencies: BaseDependencies) => { const { status, fromDictionaryId, toDictionaryId } = filterOptions; try { - const whereConditions: SQL | undefined = and( + const whereConditions = and( eq(dictionaryMigration.categoryId, categoryId), status ? eq(dictionaryMigration.status, status) : undefined, fromDictionaryId ? eq(dictionaryMigration.fromDictionaryId, fromDictionaryId) : undefined, @@ -174,7 +177,7 @@ const repository = (dependencies: BaseDependencies) => { throw new ServiceUnavailable(); } }, - getMigrationRecords: async ( + getMigrationAuditRecords: async ( migrationId: number, options: { page: number; @@ -184,7 +187,7 @@ const repository = (dependencies: BaseDependencies) => { isInvalid?: boolean; }, ): Promise<{ - result: AuditRepositoryRecord[]; + result: MigrationAuditRecord[]; metadata: { totalRecords: number; errorMessage?: string }; }> => { try { @@ -220,7 +223,7 @@ const repository = (dependencies: BaseDependencies) => { metadata: { totalRecords, }, - result: records, + result: records.map(formatMigrationAuditRecord), }; } catch (error) { logger.error(LOG_MODULE, `Failed fetching migration records for migrationId '${migrationId}'`, error); diff --git a/packages/data-provider/src/services/dictionaryService.ts b/packages/data-provider/src/services/dictionaryService.ts index 7d8e65f2..f16ccc5a 100644 --- a/packages/data-provider/src/services/dictionaryService.ts +++ b/packages/data-provider/src/services/dictionaryService.ts @@ -1,4 +1,4 @@ -import { isEmpty, result } from 'lodash-es'; +import { isEmpty } from 'lodash-es'; import { Dictionary as SchemasDictionary, Schema } from '@overture-stack/lectern-client'; import { Category, Dictionary, NewCategory, NewDictionary } from '@overture-stack/lyric-data-model/models'; diff --git a/packages/data-provider/src/services/migrationService.ts b/packages/data-provider/src/services/migrationService.ts index 1f1a3855..b1869aaa 100644 --- a/packages/data-provider/src/services/migrationService.ts +++ b/packages/data-provider/src/services/migrationService.ts @@ -7,10 +7,9 @@ import createMigrationRepository, { type MigrationRecordWithRelations, } from '../repository/dictionaryMigrationRepository.js'; import createSubmittedDataRepository from '../repository/submittedRepository.js'; -import { parseAuditRecords } from '../utils/auditUtils.js'; import { formatMigrationSummary } from '../utils/migrationUtils.js'; import { failure, type Result, success } from '../utils/result.js'; -import type { AuditDataResponse, MigrationStatus, PaginationOptions } from '../utils/types.js'; +import type { MigrationAuditRecord, MigrationStatus, PaginationOptions } from '../utils/types.js'; import submissionProcessorFactory from './submission/submissionProcessor.js'; import submissionService from './submission/submissionService.js'; @@ -137,15 +136,15 @@ const migrationService = (dependencies: BaseDependencies) => { organizations?: string | string[]; isInvalid?: boolean; }, - ): Promise<{ result: AuditDataResponse[]; metadata: { totalRecords: number; errorMessage?: string } }> => { + ): Promise<{ result: MigrationAuditRecord[]; metadata: { totalRecords: number; errorMessage?: string } }> => { try { - const migrationRecords = await migrationRepository.getMigrationRecords(migrationId, options); + const migrationRecords = await migrationRepository.getMigrationAuditRecords(migrationId, options); logger.info( LOG_MODULE, `Migration records retrieved for migrationId '${migrationId}' with options '${JSON.stringify(options)}'`, ); return { - result: parseAuditRecords(migrationRecords.result), + result: migrationRecords.result, metadata: migrationRecords.metadata, }; } catch (error) { diff --git a/packages/data-provider/src/utils/auditUtils.ts b/packages/data-provider/src/utils/auditUtils.ts index 88e29a38..aaad3324 100644 --- a/packages/data-provider/src/utils/auditUtils.ts +++ b/packages/data-provider/src/utils/auditUtils.ts @@ -44,6 +44,7 @@ export const parseAuditRecords = (data: AuditRepositoryRecord[]): AuditDataRespo return data.map((record) => ({ entityName: record.entityName, event: record.action, + errors: record.errors, dataDiff: record.dataDiff, newIsValid: record.newDataIsValid, oldIsValid: record.oldDataIsValid, diff --git a/packages/data-provider/src/utils/migrationUtils.ts b/packages/data-provider/src/utils/migrationUtils.ts index d2a06453..63dd95e3 100644 --- a/packages/data-provider/src/utils/migrationUtils.ts +++ b/packages/data-provider/src/utils/migrationUtils.ts @@ -1,4 +1,5 @@ import type { MigrationRecordWithRelations } from '../repository/dictionaryMigrationRepository.js'; +import type { AuditRepositoryRecord, MigrationAuditRecord } from './types.js'; export type MigrationSummary = MigrationRecordWithRelations & { invalidRecords?: number; @@ -23,3 +24,15 @@ export const formatMigrationSummary = (migration: MigrationSummary): MigrationSu updatedAt: migration.updatedAt, updatedBy: migration.updatedBy, }); + +export const formatMigrationAuditRecord = (record: AuditRepositoryRecord): MigrationAuditRecord => ({ + entityName: record.entityName, + dataDiff: record.dataDiff, + errors: record.errors, + newDataIsValid: record.newDataIsValid, + oldDataIsValid: record.oldDataIsValid, + organization: record.organization, + systemId: record.systemId, + createdAt: record.createdAt, + createdBy: record.createdBy, +}); diff --git a/packages/data-provider/src/utils/types.ts b/packages/data-provider/src/utils/types.ts index f53d152a..57e0dd65 100644 --- a/packages/data-provider/src/utils/types.ts +++ b/packages/data-provider/src/utils/types.ts @@ -4,6 +4,7 @@ import { type DataRecord, type DataRecordValue, Dictionary as SchemasDictionary, + type DictionaryValidationRecordErrorDetails, type Schema, } from '@overture-stack/lectern-client'; import { @@ -50,6 +51,7 @@ export type AuditRepositoryRecord = { entityName: string; action: AuditAction; dataDiff: DataDiff | null; + errors: DictionaryValidationRecordErrorDetails[] | null; newDataIsValid: boolean; oldDataIsValid: boolean; organization: string; @@ -66,6 +68,7 @@ export type AuditDataResponse = { entityName: string; event: AuditAction; dataDiff: DataDiff | null; + errors: DictionaryValidationRecordErrorDetails[] | null; newIsValid: boolean; oldIsValid: boolean; organization: string; @@ -156,6 +159,8 @@ export type RegisterDictionaryResult = { migrationId?: number; }; +export type MigrationAuditRecord = Omit; + export type { Schema, SchemasDictionary }; /** @@ -223,6 +228,20 @@ export type BooleanTrueObject = { [key: string]: true; }; +/** + * Specifies which columns of a table to select in a Drizzle query + * Used in the `columns` property of a Drizzle query + */ +export type PartialColumns = Partial>; + +/** + * Specifies additional columns to select in a Drizzle query for a related table. + * Used in the `with` property of a Drizzle query + */ +export type WithColumns = { + columns: PartialColumns; +}; + /** * Pagination Query Params */ diff --git a/packages/data-provider/test/unit/utils/auditUtils.spec.ts b/packages/data-provider/test/unit/utils/auditUtils.spec.ts index 72e6a41f..dc4d865c 100644 --- a/packages/data-provider/test/unit/utils/auditUtils.spec.ts +++ b/packages/data-provider/test/unit/utils/auditUtils.spec.ts @@ -54,6 +54,7 @@ describe('Audit utils', () => { action: 'DELETE', entityName: 'sport', dataDiff: null, + errors: null, newDataIsValid: false, oldDataIsValid: false, organization: 'fifa', @@ -67,6 +68,7 @@ describe('Audit utils', () => { entityName: 'sport', event: 'DELETE', dataDiff: null, + errors: null, newIsValid: false, oldIsValid: false, organization: 'fifa', From 57cf05b3c1cacadf7da0587799d099a85c6ebc45 Mon Sep 17 00:00:00 2001 From: Leonardo Rivera Date: Thu, 30 Apr 2026 00:21:24 -0400 Subject: [PATCH 25/33] type paginated result --- .../src/controllers/auditController.ts | 4 +-- .../src/routers/migrationRouter.ts | 10 ++++--- .../src/services/auditService.ts | 8 ++---- .../src/services/migrationService.ts | 28 ++++++++++++++----- .../services/submission/submissionService.ts | 6 ++-- .../services/submittedData/submmittedData.ts | 8 ++---- packages/data-provider/src/utils/result.ts | 10 +++++++ 7 files changed, 47 insertions(+), 27 deletions(-) diff --git a/packages/data-provider/src/controllers/auditController.ts b/packages/data-provider/src/controllers/auditController.ts index 4d220003..a77ea7a3 100644 --- a/packages/data-provider/src/controllers/auditController.ts +++ b/packages/data-provider/src/controllers/auditController.ts @@ -36,7 +36,7 @@ const controller = (dependencies: BaseDependencies) => { pageSize, }); - if (auditRecords.data.length === 0) { + if (auditRecords.result.length === 0) { throw new NotFound('No Records found'); } @@ -47,7 +47,7 @@ const controller = (dependencies: BaseDependencies) => { totalPages: Math.ceil(auditRecords.metadata.totalRecords / pageSize), totalRecords: auditRecords.metadata.totalRecords, }, - records: auditRecords.data, + records: auditRecords.result, }; return res.status(200).send(responsePaginated); } catch (error) { diff --git a/packages/data-provider/src/routers/migrationRouter.ts b/packages/data-provider/src/routers/migrationRouter.ts index cf0da63e..fdd9afdb 100644 --- a/packages/data-provider/src/routers/migrationRouter.ts +++ b/packages/data-provider/src/routers/migrationRouter.ts @@ -1,7 +1,7 @@ import { json, Router, urlencoded } from 'express'; import { BaseDependencies } from '../config/config.js'; -import migrationController from '../controllers/migrationController.js'; +import createMigrationController from '../controllers/migrationController.js'; import { type AuthConfig, authMiddleware } from '../middleware/auth.js'; const router = ({ @@ -17,11 +17,13 @@ const router = ({ router.use(authMiddleware(authConfig)); - router.get('/:migrationId', migrationController(baseDependencies).getMigrationById); + const migrationController = createMigrationController(baseDependencies); - router.get('/category/:categoryId', migrationController(baseDependencies).getMigrationsByCategoryId); + router.get('/:migrationId', migrationController.getMigrationById); - router.get('/:migrationId/records', migrationController(baseDependencies).getMigrationRecords); + router.get('/category/:categoryId', migrationController.getMigrationsByCategoryId); + + router.get('/:migrationId/records', migrationController.getMigrationRecords); return router; }; diff --git a/packages/data-provider/src/services/auditService.ts b/packages/data-provider/src/services/auditService.ts index 6cea8c61..91bdf20b 100644 --- a/packages/data-provider/src/services/auditService.ts +++ b/packages/data-provider/src/services/auditService.ts @@ -3,6 +3,7 @@ import auditRepository from '../repository/auditRepository.js'; import categoryRepository from '../repository/categoryRepository.js'; import { parseAuditRecords } from '../utils/auditUtils.js'; import { BadRequest, NotFound } from '../utils/errors.js'; +import type { PaginatedResult } from '../utils/result.js'; import { AuditDataResponse, AuditFilterOptions } from '../utils/types.js'; const auditService = (dependencies: BaseDependencies) => { @@ -14,10 +15,7 @@ const auditService = (dependencies: BaseDependencies) => { byCategoryIdAndOrganization: async ( categoryId: number, filterOptions: AuditFilterOptions, - ): Promise<{ - data: AuditDataResponse[]; - metadata: { totalRecords: number; errorMessage?: string }; - }> => { + ): Promise> => { logger.debug(LOG_MODULE, `Get category Details`); const isValidCategory = await categoryRepo.categoryIdExists(categoryId); @@ -40,7 +38,7 @@ const auditService = (dependencies: BaseDependencies) => { logger.info(LOG_MODULE, `Retrieved '${recordsPaginated.length}' Submitted data on categoryId '${categoryId}'`); return { - data: parseAuditRecords(recordsPaginated), + result: parseAuditRecords(recordsPaginated), metadata: { totalRecords, }, diff --git a/packages/data-provider/src/services/migrationService.ts b/packages/data-provider/src/services/migrationService.ts index b1869aaa..af60cfa8 100644 --- a/packages/data-provider/src/services/migrationService.ts +++ b/packages/data-provider/src/services/migrationService.ts @@ -8,7 +8,7 @@ import createMigrationRepository, { } from '../repository/dictionaryMigrationRepository.js'; import createSubmittedDataRepository from '../repository/submittedRepository.js'; import { formatMigrationSummary } from '../utils/migrationUtils.js'; -import { failure, type Result, success } from '../utils/result.js'; +import { failure, type PaginatedResult, type Result, success } from '../utils/result.js'; import type { MigrationAuditRecord, MigrationStatus, PaginationOptions } from '../utils/types.js'; import submissionProcessorFactory from './submission/submissionProcessor.js'; import submissionService from './submission/submissionService.js'; @@ -60,8 +60,9 @@ const migrationService = (dependencies: BaseDependencies) => { /** * Find the active migration by category ID - * @param categoryId - * @returns + * @param categoryId The ID of the category to find the active migration for + * @returns The active migration for the given category ID, or null if no active migration exists + * @throws Will throw an error if there is an issue retrieving the migration from the repository */ const getActiveMigrationByCategoryId = async (categoryId: number): Promise => { try { @@ -84,7 +85,13 @@ const migrationService = (dependencies: BaseDependencies) => { } }; - const getMigrationById = async (migrationId: number): Promise => { + /** + * Find a migration by its ID + * @param migrationId The ID of the migration to find + * @returns The migration with the given ID, or null if no migration exists + * @throws Will throw an error if there is an issue retrieving the migration from the repository + */ + const getMigrationById = async (migrationId: number): Promise => { try { const migration = await migrationRepository.getMigrationById(migrationId); if (migration) { @@ -100,7 +107,7 @@ const migrationService = (dependencies: BaseDependencies) => { return formatMigrationSummary({ ...migration, invalidRecords }); } else { logger.info(LOG_MODULE, `No migration found for migrationId '${migrationId}'`); - return undefined; + return null; } } catch (error) { logger.error(LOG_MODULE, `Error retrieving migration with id '${migrationId}'`, error); @@ -108,10 +115,17 @@ const migrationService = (dependencies: BaseDependencies) => { } }; + /** + * Find migrations by category ID with pagination options + * @param categoryId The ID of the category to find migrations for + * @param paginationOptions The pagination options to apply + * @returns An object containing the metadata and the list of migrations + * @throws Will throw an error if there is an issue retrieving the migrations from the repository + */ const getMigrationsByCategoryId = async ( categoryId: number, paginationOptions: PaginationOptions, - ): Promise<{ metadata: { totalRecords: number; errorMessage?: string }; result: MigrationRecordWithRelations[] }> => { + ): Promise> => { try { const migrations = await migrationRepository.getMigrationsByCategoryId(categoryId, paginationOptions, {}); @@ -136,7 +150,7 @@ const migrationService = (dependencies: BaseDependencies) => { organizations?: string | string[]; isInvalid?: boolean; }, - ): Promise<{ result: MigrationAuditRecord[]; metadata: { totalRecords: number; errorMessage?: string } }> => { + ): Promise> => { try { const migrationRecords = await migrationRepository.getMigrationAuditRecords(migrationId, options); logger.info( diff --git a/packages/data-provider/src/services/submission/submissionService.ts b/packages/data-provider/src/services/submission/submissionService.ts index 9cbe4f1b..1ca243dc 100644 --- a/packages/data-provider/src/services/submission/submissionService.ts +++ b/packages/data-provider/src/services/submission/submissionService.ts @@ -8,6 +8,7 @@ import createSubmissionRepository from '../../repository/activeSubmissionReposit import createCategoryRepository from '../../repository/categoryRepository.js'; import { getSchemaByName } from '../../utils/dictionaryUtils.js'; import { BadRequest, InternalServerError, StatusConflict } from '../../utils/errors.js'; +import type { PaginatedResult } from '../../utils/result.js'; import type { FilenameEntityPair } from '../../utils/schemas.js'; import { filterAndPaginateSubmissionData, type FlattenedSubmissionData } from '../../utils/submissionResponseParser.js'; import { @@ -236,10 +237,7 @@ const submissionService = (dependencies: BaseDependencies) => { username?: string; organization?: string; }, - ): Promise<{ - result: SubmissionSummary[]; - metadata: { totalRecords: number; errorMessage?: string }; - }> => { + ): Promise> => { const recordsPaginated = await submissionRepository.getSubmissionsByCategory( categoryId, paginationOptions, diff --git a/packages/data-provider/src/services/submittedData/submmittedData.ts b/packages/data-provider/src/services/submittedData/submmittedData.ts index fafcacf2..fa95240a 100644 --- a/packages/data-provider/src/services/submittedData/submmittedData.ts +++ b/packages/data-provider/src/services/submittedData/submmittedData.ts @@ -10,6 +10,7 @@ import submittedRepository from '../../repository/submittedRepository.js'; import { convertSqonToQuery } from '../../utils/convertSqonToQuery.js'; import { getDictionarySchemaRelations } from '../../utils/dictionarySchemaRelations.js'; import { InternalServerError, StatusConflict } from '../../utils/errors.js'; +import type { PaginatedResult } from '../../utils/result.js'; import { filterUpdatesFromDeletes, mergeDeleteRecords } from '../../utils/submissionUtils.js'; import { fetchDataErrorResponse, @@ -273,10 +274,7 @@ const submittedData = (dependencies: BaseDependencies) => { categoryId: number, paginationOptions: PaginationOptions, filterOptions: { entityName?: string[]; view: ViewType; organizations?: string[] }, - ): Promise<{ - result: SubmittedDataResponse[]; - metadata: { totalRecords: number; errorMessage?: string }; - }> => { + ): Promise> => { const { getSubmittedDataByCategoryIdPaginated, getTotalRecordsByCategoryId } = submittedDataRepo; const { getCategoryById } = categoryRepository(dependencies); @@ -345,7 +343,7 @@ const submittedData = (dependencies: BaseDependencies) => { organization: string, paginationOptions: PaginationOptions, filterOptions: { sqon?: SQON; entityName?: string[]; view: ViewType }, - ): Promise<{ result: SubmittedDataResponse[]; metadata: { totalRecords: number; errorMessage?: string } }> => { + ): Promise> => { const { getSubmittedDataByCategoryIdAndOrganizationPaginated, getTotalRecordsByCategoryIdAndOrganization } = submittedDataRepo; const { getCategoryById } = categoryRepository(dependencies); diff --git a/packages/data-provider/src/utils/result.ts b/packages/data-provider/src/utils/result.ts index 3b669d7f..1931907d 100644 --- a/packages/data-provider/src/utils/result.ts +++ b/packages/data-provider/src/utils/result.ts @@ -40,3 +40,13 @@ export const failure = (data: T): Failure => { data, }; }; + +/** + * Represents a paginated result array of type T + * and metadata with the total number of records + * and an optional error message + */ +export type PaginatedResult = { + metadata: { totalRecords: number; errorMessage?: string }; + result: T[]; +}; From a1c4d8fdf7d65256ee95c65dee6f0c3ba0447623 Mon Sep 17 00:00:00 2001 From: Leonardo Rivera Date: Thu, 30 Apr 2026 00:39:58 -0400 Subject: [PATCH 26/33] using type paginated result --- .../src/repository/dictionaryMigrationRepository.ts | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/packages/data-provider/src/repository/dictionaryMigrationRepository.ts b/packages/data-provider/src/repository/dictionaryMigrationRepository.ts index 88e27072..c7980400 100644 --- a/packages/data-provider/src/repository/dictionaryMigrationRepository.ts +++ b/packages/data-provider/src/repository/dictionaryMigrationRepository.ts @@ -11,6 +11,7 @@ import { import type { BaseDependencies } from '../config/config.js'; import { ServiceUnavailable } from '../utils/errors.js'; import { formatMigrationAuditRecord } from '../utils/migrationUtils.js'; +import type { PaginatedResult } from '../utils/result.js'; import type { MigrationAuditRecord, MigrationStatus, @@ -134,10 +135,7 @@ const repository = (dependencies: BaseDependencies) => { categoryId: number, paginationOptions: PaginationOptions, filterOptions: { status?: MigrationStatus; fromDictionaryId?: number; toDictionaryId?: number }, - ): Promise<{ - result: MigrationRecordWithRelations[]; - metadata: { totalRecords: number; errorMessage?: string }; - }> => { + ): Promise> => { const { page, pageSize } = paginationOptions; const { status, fromDictionaryId, toDictionaryId } = filterOptions; @@ -186,10 +184,7 @@ const repository = (dependencies: BaseDependencies) => { organizations?: string | string[]; isInvalid?: boolean; }, - ): Promise<{ - result: MigrationAuditRecord[]; - metadata: { totalRecords: number; errorMessage?: string }; - }> => { + ): Promise> => { try { const migration = await db.query.dictionaryMigration.findFirst({ where: eq(dictionaryMigration.id, migrationId), From e588d0b82e07e0258c8f880bafed18572d2f39d3 Mon Sep 17 00:00:00 2001 From: Leonardo Rivera Date: Thu, 30 Apr 2026 14:38:35 -0400 Subject: [PATCH 27/33] unit test migration formatter functions --- .../src/controllers/migrationController.ts | 5 +- .../src/repository/auditRepository.ts | 2 +- .../dictionaryMigrationRepository.ts | 4 +- .../src/services/migrationService.ts | 31 ++++-- ...Utils.ts => migrationResponseFormatter.ts} | 0 .../utils/migrationResponseFormatter.spec.ts | 97 +++++++++++++++++++ 6 files changed, 123 insertions(+), 16 deletions(-) rename packages/data-provider/src/utils/{migrationUtils.ts => migrationResponseFormatter.ts} (100%) create mode 100644 packages/data-provider/test/unit/utils/migrationResponseFormatter.spec.ts diff --git a/packages/data-provider/src/controllers/migrationController.ts b/packages/data-provider/src/controllers/migrationController.ts index 6ba29f3e..39c0ab4b 100644 --- a/packages/data-provider/src/controllers/migrationController.ts +++ b/packages/data-provider/src/controllers/migrationController.ts @@ -3,6 +3,7 @@ import { DEFAULT_PAGE, DEFAULT_PAGE_SIZE } from '../config/pagination.js'; import createCategoryService from '../services/categoryService.js'; import createMigrationService from '../services/migrationService.js'; import { NotFound } from '../utils/errors.js'; +import { formatMigrationSummary } from '../utils/migrationResponseFormatter.js'; import { validateRequest } from '../utils/requestValidation.js'; import { migrationByIdRequestSchema, @@ -30,7 +31,7 @@ const controller = (dependencies: BaseDependencies) => { logger.info(LOG_MODULE, message); throw new NotFound(message); } - return res.send(migrationResult); + return res.send(formatMigrationSummary(migrationResult)); } catch (error) { next(error); } @@ -59,7 +60,7 @@ const controller = (dependencies: BaseDependencies) => { totalPages: Math.ceil(migrationsResult.metadata.totalRecords / pageSize), totalRecords: migrationsResult.metadata.totalRecords, }, - records: migrationsResult.result, + records: migrationsResult.result.map(formatMigrationSummary), }; return res.send(response); diff --git a/packages/data-provider/src/repository/auditRepository.ts b/packages/data-provider/src/repository/auditRepository.ts index abcff80c..bcfd424a 100644 --- a/packages/data-provider/src/repository/auditRepository.ts +++ b/packages/data-provider/src/repository/auditRepository.ts @@ -142,7 +142,7 @@ const repository = (dependencies: BaseDependencies) => { */ getTotalRecordsByCategoryIdAndOrganization: async ( categoryId: number, - filterOptions: AuditFilterOptions, + filterOptions: Omit, ): Promise => { const { entityName, eventType, endDate, startDate, systemId, organization, submissionId, newIsValid } = filterOptions; diff --git a/packages/data-provider/src/repository/dictionaryMigrationRepository.ts b/packages/data-provider/src/repository/dictionaryMigrationRepository.ts index c7980400..9d134cf9 100644 --- a/packages/data-provider/src/repository/dictionaryMigrationRepository.ts +++ b/packages/data-provider/src/repository/dictionaryMigrationRepository.ts @@ -10,7 +10,7 @@ import { import type { BaseDependencies } from '../config/config.js'; import { ServiceUnavailable } from '../utils/errors.js'; -import { formatMigrationAuditRecord } from '../utils/migrationUtils.js'; +import { formatMigrationAuditRecord } from '../utils/migrationResponseFormatter.js'; import type { PaginatedResult } from '../utils/result.js'; import type { MigrationAuditRecord, @@ -208,8 +208,6 @@ const repository = (dependencies: BaseDependencies) => { }); const totalRecords = await auditRepository.getTotalRecordsByCategoryIdAndOrganization(migration.categoryId, { - page: options.page, - pageSize: options.pageSize, newIsValid, submissionId: migration.submissionId, }); diff --git a/packages/data-provider/src/services/migrationService.ts b/packages/data-provider/src/services/migrationService.ts index af60cfa8..600e8bea 100644 --- a/packages/data-provider/src/services/migrationService.ts +++ b/packages/data-provider/src/services/migrationService.ts @@ -7,7 +7,7 @@ import createMigrationRepository, { type MigrationRecordWithRelations, } from '../repository/dictionaryMigrationRepository.js'; import createSubmittedDataRepository from '../repository/submittedRepository.js'; -import { formatMigrationSummary } from '../utils/migrationUtils.js'; +import { type MigrationSummary } from '../utils/migrationResponseFormatter.js'; import { failure, type PaginatedResult, type Result, success } from '../utils/result.js'; import type { MigrationAuditRecord, MigrationStatus, PaginationOptions } from '../utils/types.js'; import submissionProcessorFactory from './submission/submissionProcessor.js'; @@ -74,7 +74,7 @@ const migrationService = (dependencies: BaseDependencies) => { const activeMigration = migrations.result[0]; if (activeMigration) { logger.info(LOG_MODULE, `Active migration found for categoryId '${categoryId}'`); - return formatMigrationSummary(activeMigration); + return activeMigration; } else { logger.info(LOG_MODULE, `No active migration for categoryId '${categoryId}'`); return null; @@ -91,20 +91,18 @@ const migrationService = (dependencies: BaseDependencies) => { * @returns The migration with the given ID, or null if no migration exists * @throws Will throw an error if there is an issue retrieving the migration from the repository */ - const getMigrationById = async (migrationId: number): Promise => { + const getMigrationById = async (migrationId: number): Promise => { try { const migration = await migrationRepository.getMigrationById(migrationId); if (migration) { logger.info(LOG_MODULE, `Migration found for migrationId '${migrationId}'`); const invalidRecords = await auditRepository.getTotalRecordsByCategoryIdAndOrganization(migration.category.id, { - page: -1, - pageSize: -1, newIsValid: false, submissionId: migration.submissionId, }); - return formatMigrationSummary({ ...migration, invalidRecords }); + return { ...migration, invalidRecords }; } else { logger.info(LOG_MODULE, `No migration found for migrationId '${migrationId}'`); return null; @@ -125,15 +123,28 @@ const migrationService = (dependencies: BaseDependencies) => { const getMigrationsByCategoryId = async ( categoryId: number, paginationOptions: PaginationOptions, - ): Promise> => { + ): Promise> => { try { - const migrations = await migrationRepository.getMigrationsByCategoryId(categoryId, paginationOptions, {}); + const migrationsOnCategory = await migrationRepository.getMigrationsByCategoryId( + categoryId, + paginationOptions, + {}, + ); + + const resultsWithInvalidRecords = await Promise.all( + migrationsOnCategory.result.map(async (migrationSummary): Promise => { + const invalidRecords = await auditRepository.getTotalRecordsByCategoryIdAndOrganization(categoryId, { + submissionId: migrationSummary.submissionId, + }); + return { ...migrationSummary, invalidRecords }; + }), + ); return { metadata: { - totalRecords: migrations.metadata.totalRecords, + totalRecords: migrationsOnCategory.metadata.totalRecords, }, - result: migrations.result.map(formatMigrationSummary), + result: resultsWithInvalidRecords, }; } catch (error) { logger.error(LOG_MODULE, `Error retrieving migrations for categoryId '${categoryId}'`, error); diff --git a/packages/data-provider/src/utils/migrationUtils.ts b/packages/data-provider/src/utils/migrationResponseFormatter.ts similarity index 100% rename from packages/data-provider/src/utils/migrationUtils.ts rename to packages/data-provider/src/utils/migrationResponseFormatter.ts diff --git a/packages/data-provider/test/unit/utils/migrationResponseFormatter.spec.ts b/packages/data-provider/test/unit/utils/migrationResponseFormatter.spec.ts new file mode 100644 index 00000000..5fed3b82 --- /dev/null +++ b/packages/data-provider/test/unit/utils/migrationResponseFormatter.spec.ts @@ -0,0 +1,97 @@ +import { expect } from 'chai'; +import { describe, it } from 'mocha'; + +import { + formatMigrationAuditRecord, + formatMigrationSummary, + type MigrationSummary, +} from '../../../src/utils/migrationResponseFormatter.js'; +import type { AuditRepositoryRecord, MigrationAuditRecord } from '../../../src/utils/types.js'; + +describe('Migration Utils', () => { + describe('formatMigrationSummary', () => { + it('should return migration summary with properties ordered as defined by the formatter function', () => { + const now = new Date('2026-04-30T10:00:00.000Z'); + const expectedPropertyOrder = [ + 'id', + 'category', + 'fromDictionary', + 'toDictionary', + 'submissionId', + 'invalidRecords', + 'status', + 'retries', + 'createdAt', + 'createdBy', + 'updatedAt', + 'updatedBy', + ]; + const migrationSummary: MigrationSummary = { + id: 10, + submissionId: 25, + status: 'IN_PROGRESS', + retries: 1, + createdAt: now, + createdBy: 'test-user', + updatedAt: now, + updatedBy: 'test-user', + category: { id: 3, name: 'birds' }, + fromDictionary: { name: 'animals', version: '1.0.0' }, + toDictionary: { name: 'animals', version: '2.0.0' }, + invalidRecords: 2, + }; + + const result = formatMigrationSummary(migrationSummary); + + expect(Object.keys(result)).to.eql(expectedPropertyOrder); + + expect(result).to.eql({ + id: 10, + category: { id: 3, name: 'birds' }, + fromDictionary: { name: 'animals', version: '1.0.0' }, + toDictionary: { name: 'animals', version: '2.0.0' }, + submissionId: 25, + invalidRecords: 2, + status: 'IN_PROGRESS', + retries: 1, + createdAt: now, + createdBy: 'test-user', + updatedAt: now, + updatedBy: 'test-user', + }); + }); + }); + + describe('formatMigrationAuditRecord', () => { + it('should convert repository audit record to migration audit response shape', () => { + const now = new Date('2026-04-30T11:00:00.000Z'); + const repositoryRecord: AuditRepositoryRecord = { + action: 'MIGRATION', + entityName: 'sport', + dataDiff: null, + errors: null, + newDataIsValid: false, + oldDataIsValid: true, + organization: 'lyric-org', + submissionId: 44, + systemId: 'sys-001', + createdAt: now, + createdBy: 'test-user', + }; + + const result: MigrationAuditRecord = formatMigrationAuditRecord(repositoryRecord); + + expect(result).to.eql({ + entityName: 'sport', + dataDiff: null, + errors: null, + newDataIsValid: false, + oldDataIsValid: true, + organization: 'lyric-org', + systemId: 'sys-001', + createdAt: now, + createdBy: 'test-user', + }); + }); + }); +}); From d8b930d2fe2bd387519aec7072e7343b2d7945c9 Mon Sep 17 00:00:00 2001 From: Leonardo Rivera Date: Mon, 4 May 2026 08:47:18 -0400 Subject: [PATCH 28/33] remove unhandled error thrown --- .../src/services/migrationService.ts | 142 ++++++++---------- .../src/services/submission/submissionFile.ts | 6 +- packages/data-provider/src/utils/result.ts | 5 + 3 files changed, 67 insertions(+), 86 deletions(-) diff --git a/packages/data-provider/src/services/migrationService.ts b/packages/data-provider/src/services/migrationService.ts index 600e8bea..e88fb107 100644 --- a/packages/data-provider/src/services/migrationService.ts +++ b/packages/data-provider/src/services/migrationService.ts @@ -8,7 +8,7 @@ import createMigrationRepository, { } from '../repository/dictionaryMigrationRepository.js'; import createSubmittedDataRepository from '../repository/submittedRepository.js'; import { type MigrationSummary } from '../utils/migrationResponseFormatter.js'; -import { failure, type PaginatedResult, type Result, success } from '../utils/result.js'; +import { type AsyncResult, failure, type PaginatedResult, success } from '../utils/result.js'; import type { MigrationAuditRecord, MigrationStatus, PaginationOptions } from '../utils/types.js'; import submissionProcessorFactory from './submission/submissionProcessor.js'; import submissionService from './submission/submissionService.js'; @@ -37,7 +37,7 @@ const migrationService = (dependencies: BaseDependencies) => { migrationId: number; status: Extract; userName: string; - }): Promise> => { + }): AsyncResult => { try { const resultUpdate = await migrationRepository.update(migrationId, { status, @@ -65,23 +65,18 @@ const migrationService = (dependencies: BaseDependencies) => { * @throws Will throw an error if there is an issue retrieving the migration from the repository */ const getActiveMigrationByCategoryId = async (categoryId: number): Promise => { - try { - const migrations = await migrationRepository.getMigrationsByCategoryId( - categoryId, - { page: 1, pageSize: 1 }, - { status: 'IN_PROGRESS' }, - ); - const activeMigration = migrations.result[0]; - if (activeMigration) { - logger.info(LOG_MODULE, `Active migration found for categoryId '${categoryId}'`); - return activeMigration; - } else { - logger.info(LOG_MODULE, `No active migration for categoryId '${categoryId}'`); - return null; - } - } catch (error) { - logger.error(LOG_MODULE, `Error retrieving active migration for categoryId '${categoryId}'`, error); - throw error; + const migrations = await migrationRepository.getMigrationsByCategoryId( + categoryId, + { page: 1, pageSize: 1 }, + { status: 'IN_PROGRESS' }, + ); + const activeMigration = migrations.result[0]; + if (activeMigration) { + logger.info(LOG_MODULE, `Active migration found for categoryId '${categoryId}'`); + return activeMigration; + } else { + logger.info(LOG_MODULE, `No active migration for categoryId '${categoryId}'`); + return null; } }; @@ -92,25 +87,21 @@ const migrationService = (dependencies: BaseDependencies) => { * @throws Will throw an error if there is an issue retrieving the migration from the repository */ const getMigrationById = async (migrationId: number): Promise => { - try { - const migration = await migrationRepository.getMigrationById(migrationId); - if (migration) { - logger.info(LOG_MODULE, `Migration found for migrationId '${migrationId}'`); - - const invalidRecords = await auditRepository.getTotalRecordsByCategoryIdAndOrganization(migration.category.id, { - newIsValid: false, - submissionId: migration.submissionId, - }); + const migration = await migrationRepository.getMigrationById(migrationId); - return { ...migration, invalidRecords }; - } else { - logger.info(LOG_MODULE, `No migration found for migrationId '${migrationId}'`); - return null; - } - } catch (error) { - logger.error(LOG_MODULE, `Error retrieving migration with id '${migrationId}'`, error); - throw error; + if (!migration) { + logger.info(LOG_MODULE, `No migration found for migrationId '${migrationId}'`); + return null; } + + logger.info(LOG_MODULE, `Migration found for migrationId '${migrationId}'`); + + const invalidRecords = await auditRepository.getTotalRecordsByCategoryIdAndOrganization(migration.category.id, { + newIsValid: false, + submissionId: migration.submissionId, + }); + + return { ...migration, invalidRecords }; }; /** @@ -124,34 +115,33 @@ const migrationService = (dependencies: BaseDependencies) => { categoryId: number, paginationOptions: PaginationOptions, ): Promise> => { - try { - const migrationsOnCategory = await migrationRepository.getMigrationsByCategoryId( - categoryId, - paginationOptions, - {}, - ); + const migrationsOnCategory = await migrationRepository.getMigrationsByCategoryId(categoryId, paginationOptions, {}); - const resultsWithInvalidRecords = await Promise.all( - migrationsOnCategory.result.map(async (migrationSummary): Promise => { - const invalidRecords = await auditRepository.getTotalRecordsByCategoryIdAndOrganization(categoryId, { - submissionId: migrationSummary.submissionId, - }); - return { ...migrationSummary, invalidRecords }; - }), - ); + const resultsWithInvalidRecords = await Promise.all( + migrationsOnCategory.result.map(async (migrationSummary): Promise => { + const invalidRecords = await auditRepository.getTotalRecordsByCategoryIdAndOrganization(categoryId, { + submissionId: migrationSummary.submissionId, + }); + return { ...migrationSummary, invalidRecords }; + }), + ); - return { - metadata: { - totalRecords: migrationsOnCategory.metadata.totalRecords, - }, - result: resultsWithInvalidRecords, - }; - } catch (error) { - logger.error(LOG_MODULE, `Error retrieving migrations for categoryId '${categoryId}'`, error); - throw error; - } + return { + metadata: { + totalRecords: migrationsOnCategory.metadata.totalRecords, + }, + result: resultsWithInvalidRecords, + }; }; + /** + * Retrieve the records impacted by a migration. + * Accepts pagination options and filters by entity names, organizations and validity of the records. + * @param migrationId The ID of the migration to retrieve records for + * @param options The options to filter and paginate the records + * @returns A paginated result of migration audit records + * @throws Will throw an error if there is an issue retrieving the records from the repository + */ const getMigrationRecords = async ( migrationId: number, options: { @@ -162,30 +152,18 @@ const migrationService = (dependencies: BaseDependencies) => { isInvalid?: boolean; }, ): Promise> => { - try { - const migrationRecords = await migrationRepository.getMigrationAuditRecords(migrationId, options); - logger.info( - LOG_MODULE, - `Migration records retrieved for migrationId '${migrationId}' with options '${JSON.stringify(options)}'`, - ); - return { - result: migrationRecords.result, - metadata: migrationRecords.metadata, - }; - } catch (error) { - logger.error( - LOG_MODULE, - `Error retrieving migration records for migrationId '${migrationId}' with options '${JSON.stringify(options)}'`, - error, - ); - throw error; - } + const migrationRecords = await migrationRepository.getMigrationAuditRecords(migrationId, options); + logger.info( + LOG_MODULE, + `Migration records retrieved for migrationId '${migrationId}' with options '${JSON.stringify(options)}'`, + ); + return migrationRecords; }; /** * Creates a Migration record or update retries if one exists. * Then, it starts running migration in a worker thread - * @param param0 + * @param param0 The parameters for initiating the migration * @returns The result of the migration initiation process, with the migrationId in case of success or * an error message in case of failure */ @@ -199,7 +177,7 @@ const migrationService = (dependencies: BaseDependencies) => { fromDictionaryId: number; toDictionaryId: number; userName: string; - }): Promise> => { + }): AsyncResult => { const { getOrCreateActiveSubmission } = submissionService(dependencies); try { const findMigrationResult = await migrationRepository.getMigrationsByCategoryId( @@ -216,8 +194,6 @@ const migrationService = (dependencies: BaseDependencies) => { if (existingMigration) { const updatedRetriesCount = existingMigration.retries + 1; - submissionId = existingMigration.submissionId; - migrationId = await migrationRepository.update(existingMigration.id, { retries: updatedRetriesCount, updatedBy: userName, @@ -280,7 +256,7 @@ const migrationService = (dependencies: BaseDependencies) => { }: { migrationId: number; userName: string; - }): Promise> => { + }): AsyncResult => { const { getAllOrganizationsByCategoryId, getSubmittedDataByCategoryIdAndOrganization } = submittedDataRepository; const { getActiveDictionaryByCategory } = categoryRepository; const { performCommitSubmissionAsync } = submissionProcessor; diff --git a/packages/data-provider/src/services/submission/submissionFile.ts b/packages/data-provider/src/services/submission/submissionFile.ts index 85689f4d..72c2210a 100644 --- a/packages/data-provider/src/services/submission/submissionFile.ts +++ b/packages/data-provider/src/services/submission/submissionFile.ts @@ -126,14 +126,14 @@ export function getSubmittedFileEntity(params: { case 'UNKNOWN_ENTITY': { // Mapped to an unknown entity, return failure return failure({ - code: 'UNKNOWN_ENTITY', + code: SUBMITTED_FILE_ERROR_CODES.UNKNOWN_ENTITY, message: `Provided File-Entity map indicated the file "${file.originalname}" maps to an entity named "${mapResult.data.entityName}", which does not match any of the available Schema names.`, }); } case 'MULTIPLE_MATCHES': { // Multiple mappings found, cannot map file to an entity, return failure return failure({ - code: 'UNKNOWN_ENTITY', + code: SUBMITTED_FILE_ERROR_CODES.UNKNOWN_ENTITY, message: `Provided File-Entity map has multiple matches for the file ${file.originalname}: ${mapResult.data.entityNames.join(', ')}`, }); } @@ -146,7 +146,7 @@ export function getSubmittedFileEntity(params: { } return failure({ - code: 'UNKNOWN_ENTITY', + code: SUBMITTED_FILE_ERROR_CODES.UNKNOWN_ENTITY, message: `The file named "${file.originalname}" cannot be mapped to an entity.`, }); } diff --git a/packages/data-provider/src/utils/result.ts b/packages/data-provider/src/utils/result.ts index 1931907d..5bd4aeda 100644 --- a/packages/data-provider/src/utils/result.ts +++ b/packages/data-provider/src/utils/result.ts @@ -17,6 +17,11 @@ export type Failure = { * Optionally, a data type can be provided for the failure case. */ export type Result = Success | Failure; + +/** + * Async Reporesentation of Result type, where the result is wrapped in a Promise. + */ +export type AsyncResult = Promise>; /** * Create a successful response for a Result or Either type, with data of the success type * @param {T} data From a7cfcf1aa1e5772012ab91e6dc48ad3754c91600 Mon Sep 17 00:00:00 2001 From: Leonardo Rivera Date: Mon, 4 May 2026 12:35:21 -0400 Subject: [PATCH 29/33] retry dictionary registration --- apps/server/swagger/dictionary-api.yml | 14 +++++++-- .../src/controllers/dictionaryController.ts | 2 ++ .../src/services/dictionaryService.ts | 30 +++++++++++++++++-- .../src/services/migrationService.ts | 4 +-- packages/data-provider/src/utils/schemas.ts | 9 +++++- 5 files changed, 51 insertions(+), 8 deletions(-) diff --git a/apps/server/swagger/dictionary-api.yml b/apps/server/swagger/dictionary-api.yml index a77c0294..46f46afb 100644 --- a/apps/server/swagger/dictionary-api.yml +++ b/apps/server/swagger/dictionary-api.yml @@ -2,9 +2,17 @@ /dictionary/register: post: - summary: Register new dictionary + summary: Register a dictionary to a category. Creates the category if it does not exist, or updates its active dictionary and triggers a data migration to validate and update existing data against the new dictionary. tags: - Dictionary + parameters: + - name: force + description: Re-registers the current active dictionary on the category and retries the data migration. + in: query + required: false + schema: + type: boolean + default: false requestBody: content: application/json: @@ -38,8 +46,8 @@ $ref: '#/components/responses/BadRequest' 401: $ref: '#/components/responses/UnauthorizedError' - 404: - $ref: '#/components/responses/NotFound' + 409: + $ref: '#/components/responses/StatusConflict' 500: $ref: '#/components/responses/ServerError' 503: diff --git a/packages/data-provider/src/controllers/dictionaryController.ts b/packages/data-provider/src/controllers/dictionaryController.ts index 16d9d221..5435fab4 100644 --- a/packages/data-provider/src/controllers/dictionaryController.ts +++ b/packages/data-provider/src/controllers/dictionaryController.ts @@ -22,6 +22,7 @@ const controller = (dependencies: BaseDependencies) => { const dictionaryName = req.body.dictionaryName; const dictionaryVersion = req.body.dictionaryVersion; const defaultCentricEntity = req.body.defaultCentricEntity; + const forceRegistration = req.query.force?.toLowerCase() === 'true'; const user = req.user; logger.info( @@ -35,6 +36,7 @@ const controller = (dependencies: BaseDependencies) => { dictionaryVersion, defaultCentricEntity, username: user?.username, + forceRegistration, }); logger.info(LOG_MODULE, `Register Dictionary completed!`); diff --git a/packages/data-provider/src/services/dictionaryService.ts b/packages/data-provider/src/services/dictionaryService.ts index f16ccc5a..17d7fa97 100644 --- a/packages/data-provider/src/services/dictionaryService.ts +++ b/packages/data-provider/src/services/dictionaryService.ts @@ -7,7 +7,7 @@ import { BaseDependencies } from '../config/config.js'; import lecternClient from '../external/lecternClient.js'; import categoryRepository from '../repository/categoryRepository.js'; import dictionaryRepository from '../repository/dictionaryRepository.js'; -import { BadRequest } from '../utils/errors.js'; +import { BadRequest, StatusConflict } from '../utils/errors.js'; import migrationService from './migrationService.js'; const dictionaryService = (dependencies: BaseDependencies) => { @@ -99,12 +99,14 @@ const dictionaryService = (dependencies: BaseDependencies) => { dictionaryVersion, defaultCentricEntity, username, + forceRegistration = false, }: { categoryName: string; dictionaryName: string; dictionaryVersion: string; defaultCentricEntity?: string; username?: string; + forceRegistration?: boolean; }): Promise<{ dictionary: Dictionary; category: Category; migrationId?: number }> => { logger.debug( LOG_MODULE, @@ -135,7 +137,31 @@ const dictionaryService = (dependencies: BaseDependencies) => { // Dictionary and Category already exists logger.info(LOG_MODULE, `Dictionary and Category already exists`); - return { dictionary: savedDictionary, category: foundCategory }; + if (forceRegistration) { + logger.info( + LOG_MODULE, + `Force flag is true, initiating migration for Category '${foundCategory.name}' + with Dictionary '${savedDictionary.name}' version '${savedDictionary.version}'`, + ); + + const resultMigration = await initiateMigration({ + categoryId: foundCategory.id, + toDictionaryId: savedDictionary.id, + userName: username || '', + }); + + if (!resultMigration.success) { + const errorMessage = `Failed to initiate migration for category '${categoryName}' with error: ${resultMigration.data}`; + logger.error(LOG_MODULE, errorMessage); + throw new Error(errorMessage); + } + + return { dictionary: savedDictionary, category: foundCategory, migrationId: resultMigration.data }; + } + + throw new StatusConflict( + `Category '${categoryName}' with Dictionary '${savedDictionary.name}' version '${savedDictionary.version}' already exists`, + ); } else if (foundCategory && foundCategory.activeDictionaryId !== savedDictionary.id) { // Update the dictionary on existing Category const updatedCategory = await categoryRepo.update(foundCategory.id, { diff --git a/packages/data-provider/src/services/migrationService.ts b/packages/data-provider/src/services/migrationService.ts index e88fb107..229f40b2 100644 --- a/packages/data-provider/src/services/migrationService.ts +++ b/packages/data-provider/src/services/migrationService.ts @@ -174,7 +174,7 @@ const migrationService = (dependencies: BaseDependencies) => { userName, }: { categoryId: number; - fromDictionaryId: number; + fromDictionaryId?: number; toDictionaryId: number; userName: string; }): AsyncResult => { @@ -214,7 +214,7 @@ const migrationService = (dependencies: BaseDependencies) => { const newMigration: NewDictionaryMigration = { categoryId, - fromDictionaryId, + fromDictionaryId: fromDictionaryId ?? toDictionaryId, toDictionaryId, submissionId, status: 'IN_PROGRESS', diff --git a/packages/data-provider/src/utils/schemas.ts b/packages/data-provider/src/utils/schemas.ts index 742887e6..18985295 100644 --- a/packages/data-provider/src/utils/schemas.ts +++ b/packages/data-provider/src/utils/schemas.ts @@ -234,9 +234,13 @@ export interface DictionaryRegisterBodyParams { defaultCentricEntity?: string; } +export interface DictionaryRegisterQueryParams extends ParsedQs { + force?: string; +} + export const dictionaryRegisterRequestSchema: RequestValidation< DictionaryRegisterBodyParams, - ParsedQs, + DictionaryRegisterQueryParams, ParamsDictionary > = { body: z.object({ @@ -245,6 +249,9 @@ export const dictionaryRegisterRequestSchema: RequestValidation< dictionaryVersion: stringNotEmpty, defaultCentricEntity: entityNameSchema.or(z.literal('')).optional(), }), + query: z.object({ + force: booleanSchema.default('false'), + }), }; // Migration Requests From a2b05e858d30a10144eaff5d26bc5f039c2207d3 Mon Sep 17 00:00:00 2001 From: Leonardo Rivera Date: Mon, 4 May 2026 13:09:03 -0400 Subject: [PATCH 30/33] updating docs --- packages/data-provider/docs/dictionary-registration.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/packages/data-provider/docs/dictionary-registration.md b/packages/data-provider/docs/dictionary-registration.md index 4e8195e0..e6ea01e7 100644 --- a/packages/data-provider/docs/dictionary-registration.md +++ b/packages/data-provider/docs/dictionary-registration.md @@ -108,6 +108,10 @@ A Category is uniquely identified by its case-sensitive `categoryName`. Category groups data that is related and shares the same data structure, for that reason, a category must be associated to a registered dictionary. Over time, if the dictionary requires an update, the category needs to be updates accordingly, See [Dictionary Migration](#dictionary-migration) for more details. +If a category is already using the same dictionary name and version, registration returns `409 Conflict`. + +Set the `force` query parameter to `true` to allow re-registration and trigger a migration (or retry) to revalidate existing category data against that dictionary. + ## Centric entity Some dictionaries define a centric entity, representing the root of the data model hierarchy (used on compound views). From feb65f56cb3c45fed0a90f5a9598ef9cd3db796b Mon Sep 17 00:00:00 2001 From: Leonardo Rivera Date: Fri, 29 May 2026 11:42:51 -0400 Subject: [PATCH 31/33] update documentation --- apps/server/swagger/dictionary-api.yml | 2 +- packages/data-provider/docs/dictionary-registration.md | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/apps/server/swagger/dictionary-api.yml b/apps/server/swagger/dictionary-api.yml index 46f46afb..5f863d0c 100644 --- a/apps/server/swagger/dictionary-api.yml +++ b/apps/server/swagger/dictionary-api.yml @@ -7,7 +7,7 @@ - Dictionary parameters: - name: force - description: Re-registers the current active dictionary on the category and retries the data migration. + description: Runs dictionary registration and migration again for this category, even if the same dictionary is already registered. Use this only when a previous migration ended unexpectedly and you must rerun both steps. in: query required: false schema: diff --git a/packages/data-provider/docs/dictionary-registration.md b/packages/data-provider/docs/dictionary-registration.md index e6ea01e7..0a7f75cd 100644 --- a/packages/data-provider/docs/dictionary-registration.md +++ b/packages/data-provider/docs/dictionary-registration.md @@ -108,9 +108,7 @@ A Category is uniquely identified by its case-sensitive `categoryName`. Category groups data that is related and shares the same data structure, for that reason, a category must be associated to a registered dictionary. Over time, if the dictionary requires an update, the category needs to be updates accordingly, See [Dictionary Migration](#dictionary-migration) for more details. -If a category is already using the same dictionary name and version, registration returns `409 Conflict`. - -Set the `force` query parameter to `true` to allow re-registration and trigger a migration (or retry) to revalidate existing category data against that dictionary. +If a category is already using the same dictionary name and version, registration returns `409 Conflict` by default. Set the `force` query parameter to `true` only when you need to override this and rerun registration for the same dictionary. This also triggers migration validation for existing category data, which is mainly useful when a previous registration or migration ended unexpectedly. ## Centric entity From ec4a58040fe6993ad2972fa915deb283d1773743 Mon Sep 17 00:00:00 2001 From: Leonardo Rivera Date: Fri, 29 May 2026 11:43:23 -0400 Subject: [PATCH 32/33] DRY refactor --- .../src/services/dictionaryService.ts | 48 ++++++++++++------- .../src/services/migrationService.ts | 4 +- 2 files changed, 32 insertions(+), 20 deletions(-) diff --git a/packages/data-provider/src/services/dictionaryService.ts b/packages/data-provider/src/services/dictionaryService.ts index 17d7fa97..d3992e14 100644 --- a/packages/data-provider/src/services/dictionaryService.ts +++ b/packages/data-provider/src/services/dictionaryService.ts @@ -133,6 +133,31 @@ const dictionaryService = (dependencies: BaseDependencies) => { // Check if Category exist const foundCategory = await categoryRepo.getCategoryByName(categoryName); + const initiateMigrationOrThrow = async ({ + categoryId, + fromDictionaryId, + toDictionaryId, + }: { + categoryId: number; + fromDictionaryId: number; + toDictionaryId: number; + }): Promise => { + const resultMigration = await initiateMigration({ + categoryId, + fromDictionaryId, + toDictionaryId, + userName: username || '', + }); + + if (!resultMigration.success) { + const errorMessage = `Failed to initiate migration for category '${categoryName}' with error: ${resultMigration.data}`; + logger.error(LOG_MODULE, errorMessage); + throw new Error(errorMessage); + } + + return resultMigration.data; + }; + if (foundCategory && foundCategory.activeDictionaryId === savedDictionary.id) { // Dictionary and Category already exists logger.info(LOG_MODULE, `Dictionary and Category already exists`); @@ -144,19 +169,13 @@ const dictionaryService = (dependencies: BaseDependencies) => { with Dictionary '${savedDictionary.name}' version '${savedDictionary.version}'`, ); - const resultMigration = await initiateMigration({ + const migrationId = await initiateMigrationOrThrow({ categoryId: foundCategory.id, + fromDictionaryId: foundCategory.activeDictionaryId, toDictionaryId: savedDictionary.id, - userName: username || '', }); - if (!resultMigration.success) { - const errorMessage = `Failed to initiate migration for category '${categoryName}' with error: ${resultMigration.data}`; - logger.error(LOG_MODULE, errorMessage); - throw new Error(errorMessage); - } - - return { dictionary: savedDictionary, category: foundCategory, migrationId: resultMigration.data }; + return { dictionary: savedDictionary, category: foundCategory, migrationId }; } throw new StatusConflict( @@ -170,25 +189,18 @@ const dictionaryService = (dependencies: BaseDependencies) => { updatedBy: username, }); - const resultMigration = await initiateMigration({ + const migrationId = await initiateMigrationOrThrow({ categoryId: updatedCategory.id, fromDictionaryId: foundCategory.activeDictionaryId, toDictionaryId: savedDictionary.id, - userName: username || '', }); - if (!resultMigration.success) { - const errorMessage = `Failed to initiate migration for category '${categoryName}' with error: ${resultMigration.data}`; - logger.error(LOG_MODULE, errorMessage); - throw new Error(errorMessage); - } - logger.info( LOG_MODULE, `Category '${updatedCategory.name}' updated successfully with Dictionary '${savedDictionary.name}' version '${savedDictionary.version}'`, ); - return { dictionary: savedDictionary, category: updatedCategory, migrationId: resultMigration.data }; + return { dictionary: savedDictionary, category: updatedCategory, migrationId }; } else { // Create a new Category const newCategory: NewCategory = { diff --git a/packages/data-provider/src/services/migrationService.ts b/packages/data-provider/src/services/migrationService.ts index 229f40b2..e88fb107 100644 --- a/packages/data-provider/src/services/migrationService.ts +++ b/packages/data-provider/src/services/migrationService.ts @@ -174,7 +174,7 @@ const migrationService = (dependencies: BaseDependencies) => { userName, }: { categoryId: number; - fromDictionaryId?: number; + fromDictionaryId: number; toDictionaryId: number; userName: string; }): AsyncResult => { @@ -214,7 +214,7 @@ const migrationService = (dependencies: BaseDependencies) => { const newMigration: NewDictionaryMigration = { categoryId, - fromDictionaryId: fromDictionaryId ?? toDictionaryId, + fromDictionaryId, toDictionaryId, submissionId, status: 'IN_PROGRESS', From 3871c5983b2c016b0332c57a5db0b08499b76ea3 Mon Sep 17 00:00:00 2001 From: Leonardo Rivera Date: Tue, 9 Jun 2026 16:38:05 -0400 Subject: [PATCH 33/33] re-run migration only when. prev has failed --- apps/server/swagger/dictionary-api.yml | 2 +- .../data-provider/docs/dictionary-registration.md | 2 +- .../data-provider/src/services/dictionaryService.ts | 12 ++++++++++-- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/apps/server/swagger/dictionary-api.yml b/apps/server/swagger/dictionary-api.yml index 5f863d0c..b70dbe43 100644 --- a/apps/server/swagger/dictionary-api.yml +++ b/apps/server/swagger/dictionary-api.yml @@ -7,7 +7,7 @@ - Dictionary parameters: - name: force - description: Runs dictionary registration and migration again for this category, even if the same dictionary is already registered. Use this only when a previous migration ended unexpectedly and you must rerun both steps. + description: Retries migration for this category only when the same dictionary is already registered and the most recent migration previously failed. If there is no previous failed migration, this flag is ignored. in: query required: false schema: diff --git a/packages/data-provider/docs/dictionary-registration.md b/packages/data-provider/docs/dictionary-registration.md index 0a7f75cd..46713e8e 100644 --- a/packages/data-provider/docs/dictionary-registration.md +++ b/packages/data-provider/docs/dictionary-registration.md @@ -108,7 +108,7 @@ A Category is uniquely identified by its case-sensitive `categoryName`. Category groups data that is related and shares the same data structure, for that reason, a category must be associated to a registered dictionary. Over time, if the dictionary requires an update, the category needs to be updates accordingly, See [Dictionary Migration](#dictionary-migration) for more details. -If a category is already using the same dictionary name and version, registration returns `409 Conflict` by default. Set the `force` query parameter to `true` only when you need to override this and rerun registration for the same dictionary. This also triggers migration validation for existing category data, which is mainly useful when a previous registration or migration ended unexpectedly. +If a category is already using the same dictionary name and version, registration returns `409 Conflict` by default. If the `force` query parameter is set to `true`, Lyric retries only when the same dictionary is already registered and the latest migration previously failed. If no prior failed migration exists, the `force` flag is ignored. ## Centric entity diff --git a/packages/data-provider/src/services/dictionaryService.ts b/packages/data-provider/src/services/dictionaryService.ts index d3992e14..c8e2bf02 100644 --- a/packages/data-provider/src/services/dictionaryService.ts +++ b/packages/data-provider/src/services/dictionaryService.ts @@ -114,7 +114,7 @@ const dictionaryService = (dependencies: BaseDependencies) => { ); const categoryRepo = categoryRepository(dependencies); - const { initiateMigration } = migrationService(dependencies); + const { initiateMigration, getMigrationsByCategoryId } = migrationService(dependencies); const dictionary = await fetchDictionaryByVersion(dictionaryName, dictionaryVersion); @@ -162,7 +162,15 @@ const dictionaryService = (dependencies: BaseDependencies) => { // Dictionary and Category already exists logger.info(LOG_MODULE, `Dictionary and Category already exists`); - if (forceRegistration) { + // check last migration of this category to find if it failed, if it failed and forceRegistration is true, + // we will re-initiate the migration with the new dictionary + const activeMigration = await getMigrationsByCategoryId(foundCategory.id, { pageSize: 1, page: 1 }); + + if ( + forceRegistration && + activeMigration.result.length === 1 && + activeMigration.result.at(0)?.status === 'FAILED' + ) { logger.info( LOG_MODULE, `Force flag is true, initiating migration for Category '${foundCategory.name}'