From 91bf7f7212cac0fa86dd6cfdf31223ba4df60321 Mon Sep 17 00:00:00 2001 From: Alexander Onnikov Date: Mon, 26 May 2025 13:57:03 +0700 Subject: [PATCH 1/3] UBERF-11178 Migrate datalake workspace id Signed-off-by: Alexander Onnikov --- .../datalake/pod-datalake/src/datalake/db.ts | 21 ++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/services/datalake/pod-datalake/src/datalake/db.ts b/services/datalake/pod-datalake/src/datalake/db.ts index 7d45011c285..4ea130c46be 100644 --- a/services/datalake/pod-datalake/src/datalake/db.ts +++ b/services/datalake/pod-datalake/src/datalake/db.ts @@ -560,7 +560,7 @@ export function escape (value: any): string { } function getMigrations (): [string, string][] { - return [migrationV1()] + return [migrationV1(), migrationV2()] } function migrationV1 (): [string, string] { @@ -599,3 +599,22 @@ function migrationV1 (): [string, string] { ` return ['init_tables_01', sql] } + +function migrationV2 (): [string, string] { + const sql = ` + ALTER TABLE blob.meta DROP CONSTRAINT IF EXISTS fk_blob; + + UPDATE blob.blob + SET workspace = w.uuid + FROM global_account.workspace w + WHERE workspace = w.data_id; + + UPDATE blob.meta + SET workspace = w.uuid + FROM global_account.workspace w + WHERE workspace = w.data_id; + + ALTER TABLE blob.meta ADD CONSTRAINT fk_blob FOREIGN KEY (workspace, name) REFERENCES blob.blob (workspace, name); + ` + return ['migrate_workspaces_02', sql] +} From 363a6c1c99336e133f982391ae7a6261ea35290d Mon Sep 17 00:00:00 2001 From: Alexander Onnikov Date: Thu, 17 Jul 2025 12:38:50 +0700 Subject: [PATCH 2/3] fix: change foreign key name Signed-off-by: Alexander Onnikov --- services/datalake/pod-datalake/src/datalake/db.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/datalake/pod-datalake/src/datalake/db.ts b/services/datalake/pod-datalake/src/datalake/db.ts index 4ea130c46be..186dc420e1f 100644 --- a/services/datalake/pod-datalake/src/datalake/db.ts +++ b/services/datalake/pod-datalake/src/datalake/db.ts @@ -614,7 +614,7 @@ function migrationV2 (): [string, string] { FROM global_account.workspace w WHERE workspace = w.data_id; - ALTER TABLE blob.meta ADD CONSTRAINT fk_blob FOREIGN KEY (workspace, name) REFERENCES blob.blob (workspace, name); + ALTER TABLE blob.meta ADD CONSTRAINT fk_blob_meta FOREIGN KEY (workspace, name) REFERENCES blob.blob (workspace, name); ` return ['migrate_workspaces_02', sql] } From 3c6ba8ca2f6d8477267e20613e07cd224b5cbdf0 Mon Sep 17 00:00:00 2001 From: Alexander Onnikov Date: Thu, 17 Jul 2025 15:01:44 +0700 Subject: [PATCH 3/3] move datalake storage config to overrides Signed-off-by: Alexander Onnikov --- tests/.env | 3 ++- tests/docker-compose.override.yaml | 35 ++++++++++++++++++++++++++++++ tests/docker-compose.yaml | 26 +--------------------- tests/tool.sh | 2 +- 4 files changed, 39 insertions(+), 27 deletions(-) diff --git a/tests/.env b/tests/.env index 1672ce40acb..7c1f4743864 100644 --- a/tests/.env +++ b/tests/.env @@ -1,4 +1,5 @@ -STORAGE_CONFIG="datalake|http://datalake:4031" +STORAGE_CONFIG="minio|minio?accessKey=minioadmin&secretKey=minioadmin" +DATALAKE_STORAGE_CONFIG="datalake|http://datalake:4031" BACKUP_STORAGE_CONFIG="minio|minio?accessKey=minioadmin&secretKey=minioadmin" BACKUP_BUCKET_NAME=dev-backups diff --git a/tests/docker-compose.override.yaml b/tests/docker-compose.override.yaml index c16574744be..f732728a7ad 100644 --- a/tests/docker-compose.override.yaml +++ b/tests/docker-compose.override.yaml @@ -2,12 +2,47 @@ services: account: environment: - DB_URL=${DB_PG_URL} + - STORAGE_CONFIG=${DATALAKE_STORAGE_CONFIG} transactor: environment: - DB_URL=${DB_PG_URL} + - STORAGE_CONFIG=${DATALAKE_STORAGE_CONFIG} workspace: environment: - DB_URL=${DB_PG_URL} + - STORAGE_CONFIG=${DATALAKE_STORAGE_CONFIG} fulltext: environment: - DB_URL=${DB_PG_URL} + - STORAGE_CONFIG=${DATALAKE_STORAGE_CONFIG} + front: + environment: + - STORAGE_CONFIG=${DATALAKE_STORAGE_CONFIG} + - FILES_URL=http://localhost:4031/blob/:workspace/:blobId/:filename + - UPLOAD_URL=http://localhost:4031/upload/form-data/:workspace + - PREVIEW_CONFIG=http://localhost:4031/image/fit=cover,width=:width,height=:height,dpr=:dpr/:workspace/:blobId + collaborator: + environment: + - STORAGE_CONFIG=${DATALAKE_STORAGE_CONFIG} + datalake: + image: hardcoreeng/datalake + depends_on: + minio: + condition: service_healthy + cockroach: + condition: service_started + stats: + condition: service_started + account: + condition: service_started + ports: + - 4031:4031 + environment: + - PORT=4031 + - SECRET=secret + - ACCOUNTS_URL=http://account:3003 + - STATS_URL=http://stats:4901 + - STREAM_URL=http://localhost:1081/recording + - DB_URL=${DB_PG_URL} + - BUCKETS=blobs,eu|http://minio:9000?accessKey=minioadmin&secretKey=minioadmin + restart: unless-stopped \ No newline at end of file diff --git a/tests/docker-compose.yaml b/tests/docker-compose.yaml index 71a7a1de768..c004c1cea95 100644 --- a/tests/docker-compose.yaml +++ b/tests/docker-compose.yaml @@ -173,9 +173,7 @@ services: - SERVER_SECRET=secret - ACCOUNTS_URL_INTERNAL=http://account:3003 - ACCOUNTS_URL=http://localhost:3003 - - FILES_URL=http://localhost:4031/blob/:workspace/:blobId/:filename - - UPLOAD_URL=http://localhost:4031/upload/form-data/:workspace - - PREVIEW_CONFIG=http://localhost:4031/image/fit=cover,width=:width,height=:height,dpr=:dpr/:workspace/:blobId + - UPLOAD_URL=/files - ELASTIC_URL=http://elastic:9200 - GMAIL_URL=http://localhost:8088 - CALENDAR_URL=http://localhost:8095 @@ -275,28 +273,6 @@ services: - REKONI_URL=http://rekoni:4007 - ACCOUNTS_URL=http://account:3003 - STATS_URL=http://stats:4901 - datalake: - image: hardcoreeng/datalake - depends_on: - minio: - condition: service_healthy - cockroach: - condition: service_started - stats: - condition: service_started - account: - condition: service_started - ports: - - 4031:4031 - environment: - - PORT=4031 - - SECRET=secret - - ACCOUNTS_URL=http://account:3003 - - STATS_URL=http://stats:4901 - - STREAM_URL=http://localhost:1081/recording - - DB_URL=${DB_PG_URL} - - BUCKETS=blobs,eu|http://minio:9000?accessKey=minioadmin&secretKey=minioadmin - restart: unless-stopped stats: image: hardcoreeng/stats ports: diff --git a/tests/tool.sh b/tests/tool.sh index e69ffec1654..82f26588bdd 100755 --- a/tests/tool.sh +++ b/tests/tool.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash export MODEL_VERSION=$(node ../common/scripts/show_version.js) -export STORAGE_CONFIG="datalake|http://localhost:4031" +export STORAGE_CONFIG="minio|minio?accessKey=minioadmin&secretKey=minioadmin" export ACCOUNTS_URL=http://localhost:3003 export TRANSACTOR_URL=ws://localhost:3334 export ACCOUNT_DB_URL=mongodb://localhost:27018